whisper

mindnlp.transformers.models.whisper.modeling_whisper

MindSpore Whisper model.

mindnlp.transformers.models.whisper.modeling_whisper.WhisperAttention

Bases: Cell

Multi-headed attention from 'Attention Is All You Need' paper

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperAttention(nn.Cell):
    """Multi-headed attention from 'Attention Is All You Need' paper"""
    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[WhisperConfig] = None,
    ):
        """
        Initializes the WhisperAttention class.

        Args:
            self: The instance of the class.
            embed_dim (int): The dimension of the input embeddings.
            num_heads (int): The number of attention heads.
            dropout (float, optional): The dropout probability. Default is 0.0.
            is_decoder (bool, optional): Indicates whether the attention mechanism is used as a decoder.
                Default is False.
            bias (bool, optional): Indicates whether the linear layers have bias terms. Default is True.
            is_causal (bool, optional): Indicates whether the attention is causal. Default is False.
            config (Optional[WhisperConfig], optional): The configuration for WhisperAttention. Default is None.

        Returns:
            None.

        Raises:
            ValueError: If the embed_dim is not divisible by num_heads.
        """
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Dense(embed_dim, embed_dim, has_bias=False)
        self.v_proj = nn.Dense(embed_dim, embed_dim, has_bias=bias)
        self.q_proj = nn.Dense(embed_dim, embed_dim, has_bias=bias)
        self.out_proj = nn.Dense(embed_dim, embed_dim, has_bias=bias)

    # Copied from transformers.models.bart.modeling_bart.BartAttention._shape with BART->whisper
    def _shape(self, tensor: mindspore.Tensor, seq_len: int, bsz: int):
        """
        Reshapes the input tensor for attention computation.

        Args:
            self (WhisperAttention): An instance of the WhisperAttention class.
            tensor (mindspore.Tensor): The input tensor to be reshaped.
                It should have shape (bsz * seq_len, self.embed_dim).
            seq_len (int): The length of the sequence.
            bsz (int): The batch size.

        Returns:
            None.

        Raises:
            None.
        """
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).swapaxes(1, 2)

    # Copied from transformers.models.bart.modeling_bart.BartAttention.forward with BART->whisper
    def construct(
        self,
        hidden_states: mindspore.Tensor,
        key_value_states: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        layer_head_mask: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        """Input shape: Batch x Time x Channel"""
        # if key_value_states are provided this layer is used as a cross-attention layer
        # for the decoder
        is_cross_attention = key_value_states is not None

        bsz, tgt_len, _ = hidden_states.shape

        # get query proj
        query_states = self.q_proj(hidden_states) * self.scaling
        # get key, value proj
        # `past_key_value[0].shape[2] == key_value_states.shape[1]`
        # is checking that the `sequence_length` of the `past_key_value` is the same as
        # the provided `key_value_states` to support prefix tuning
        if (
            is_cross_attention
            and past_key_value is not None
            and past_key_value[0].shape[2] == key_value_states.shape[1]
        ):
            # reuse k,v, cross_attentions
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            # cross_attentions
            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
        elif past_key_value is not None:
            # reuse k, v, self_attention
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = ops.cat([past_key_value[0], key_states], axis=2)
            value_states = ops.cat([past_key_value[1], value_states], axis=2)
        else:
            # self_attention
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
        key_states = key_states.reshape(*proj_shape)
        value_states = value_states.reshape(*proj_shape)

        src_len = key_states.shape[1]
        attn_weights = ops.bmm(query_states, key_states.swapaxes(1, 2))

        if attn_weights.shape != (bsz * self.num_heads, tgt_len, src_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {attn_weights.shape}"
            )

        if attention_mask is not None:
            if attention_mask.shape != (bsz, 1, tgt_len, src_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.shape}"
                )
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        attn_weights = ops.softmax(attn_weights, axis=-1)

        if layer_head_mask is not None:
            if layer_head_mask.shape != (self.num_heads,):
                raise ValueError(
                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                    f" {layer_head_mask.shape}"
                )
            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        if output_attentions:
            # this operation is a bit awkward, but it's required to
            # make sure that attn_weights keeps its gradient.
            # In order to do so, attn_weights have to be reshaped
            # twice and have to be reused in the following
            attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
        else:
            attn_weights_reshaped = None

        attn_probs = ops.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = ops.bmm(attn_probs, value_states)
        if attn_output.shape != (bsz * self.num_heads, tgt_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {attn_output.shape}"
            )

        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
        attn_output = attn_output.swapaxes(1, 2)

        # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
        # partitioned across GPUs when using tensor-parallelism.
        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)
        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights_reshaped, past_key_value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperAttention.__init__(embed_dim, num_heads, dropout=0.0, is_decoder=False, bias=True, is_causal=False, config=None)

Initializes the WhisperAttention class.

PARAMETER	DESCRIPTION
self	The instance of the class.
embed_dim	The dimension of the input embeddings. TYPE: int
num_heads	The number of attention heads. TYPE: int
dropout	The dropout probability. Default is 0.0. TYPE: float DEFAULT: 0.0
is_decoder	Indicates whether the attention mechanism is used as a decoder. Default is False. TYPE: bool DEFAULT: False
bias	Indicates whether the linear layers have bias terms. Default is True. TYPE: bool DEFAULT: True
is_causal	Indicates whether the attention is causal. Default is False. TYPE: bool DEFAULT: False
config	The configuration for WhisperAttention. Default is None. TYPE: Optional[WhisperConfig] DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the embed_dim is not divisible by num_heads.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(
    self,
    embed_dim: int,
    num_heads: int,
    dropout: float = 0.0,
    is_decoder: bool = False,
    bias: bool = True,
    is_causal: bool = False,
    config: Optional[WhisperConfig] = None,
):
    """
    Initializes the WhisperAttention class.

    Args:
        self: The instance of the class.
        embed_dim (int): The dimension of the input embeddings.
        num_heads (int): The number of attention heads.
        dropout (float, optional): The dropout probability. Default is 0.0.
        is_decoder (bool, optional): Indicates whether the attention mechanism is used as a decoder.
            Default is False.
        bias (bool, optional): Indicates whether the linear layers have bias terms. Default is True.
        is_causal (bool, optional): Indicates whether the attention is causal. Default is False.
        config (Optional[WhisperConfig], optional): The configuration for WhisperAttention. Default is None.

    Returns:
        None.

    Raises:
        ValueError: If the embed_dim is not divisible by num_heads.
    """
    super().__init__()
    self.embed_dim = embed_dim
    self.num_heads = num_heads
    self.dropout = dropout
    self.head_dim = embed_dim // num_heads
    self.config = config

    if (self.head_dim * num_heads) != self.embed_dim:
        raise ValueError(
            f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
            f" and `num_heads`: {num_heads})."
        )
    self.scaling = self.head_dim**-0.5
    self.is_decoder = is_decoder
    self.is_causal = is_causal

    self.k_proj = nn.Dense(embed_dim, embed_dim, has_bias=False)
    self.v_proj = nn.Dense(embed_dim, embed_dim, has_bias=bias)
    self.q_proj = nn.Dense(embed_dim, embed_dim, has_bias=bias)
    self.out_proj = nn.Dense(embed_dim, embed_dim, has_bias=bias)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperAttention.construct(hidden_states, key_value_states=None, past_key_value=None, attention_mask=None, layer_head_mask=None, output_attentions=False)

Input shape: Batch x Time x Channel

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    hidden_states: mindspore.Tensor,
    key_value_states: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    layer_head_mask: Optional[mindspore.Tensor] = None,
    output_attentions: bool = False,
) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
    """Input shape: Batch x Time x Channel"""
    # if key_value_states are provided this layer is used as a cross-attention layer
    # for the decoder
    is_cross_attention = key_value_states is not None

    bsz, tgt_len, _ = hidden_states.shape

    # get query proj
    query_states = self.q_proj(hidden_states) * self.scaling
    # get key, value proj
    # `past_key_value[0].shape[2] == key_value_states.shape[1]`
    # is checking that the `sequence_length` of the `past_key_value` is the same as
    # the provided `key_value_states` to support prefix tuning
    if (
        is_cross_attention
        and past_key_value is not None
        and past_key_value[0].shape[2] == key_value_states.shape[1]
    ):
        # reuse k,v, cross_attentions
        key_states = past_key_value[0]
        value_states = past_key_value[1]
    elif is_cross_attention:
        # cross_attentions
        key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
        value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
    elif past_key_value is not None:
        # reuse k, v, self_attention
        key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
        key_states = ops.cat([past_key_value[0], key_states], axis=2)
        value_states = ops.cat([past_key_value[1], value_states], axis=2)
    else:
        # self_attention
        key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_states, value_states)

    proj_shape = (bsz * self.num_heads, -1, self.head_dim)
    query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
    key_states = key_states.reshape(*proj_shape)
    value_states = value_states.reshape(*proj_shape)

    src_len = key_states.shape[1]
    attn_weights = ops.bmm(query_states, key_states.swapaxes(1, 2))

    if attn_weights.shape != (bsz * self.num_heads, tgt_len, src_len):
        raise ValueError(
            f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
            f" {attn_weights.shape}"
        )

    if attention_mask is not None:
        if attention_mask.shape != (bsz, 1, tgt_len, src_len):
            raise ValueError(
                f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.shape}"
            )
        attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
        attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

    attn_weights = ops.softmax(attn_weights, axis=-1)

    if layer_head_mask is not None:
        if layer_head_mask.shape != (self.num_heads,):
            raise ValueError(
                f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                f" {layer_head_mask.shape}"
            )
        attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
        attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

    if output_attentions:
        # this operation is a bit awkward, but it's required to
        # make sure that attn_weights keeps its gradient.
        # In order to do so, attn_weights have to be reshaped
        # twice and have to be reused in the following
        attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
        attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
    else:
        attn_weights_reshaped = None

    attn_probs = ops.dropout(attn_weights, p=self.dropout, training=self.training)

    attn_output = ops.bmm(attn_probs, value_states)
    if attn_output.shape != (bsz * self.num_heads, tgt_len, self.head_dim):
        raise ValueError(
            f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
            f" {attn_output.shape}"
        )

    attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
    attn_output = attn_output.swapaxes(1, 2)

    # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
    # partitioned across GPUs when using tensor-parallelism.
    attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)
    attn_output = self.out_proj(attn_output)

    return attn_output, attn_weights_reshaped, past_key_value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder

Bases: WhisperPreTrainedModel

Transformer decoder consisting of config.decoder_layers layers. Each layer is a [WhisperDecoderLayer]

PARAMETER	DESCRIPTION
config	WhisperConfig TYPE: WhisperConfig

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperDecoder(WhisperPreTrainedModel):
    """
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`WhisperDecoderLayer`]

    Args:
        config: WhisperConfig
    """
    main_input_name = "input_ids"

    def __init__(self, config: WhisperConfig):
        """
        Initializes the WhisperDecoder class.

        Args:
            self: The instance of the class.
            config (WhisperConfig):
                An instance of WhisperConfig containing the configuration parameters for the decoder.

                - dropout (float): The dropout probability.
                - decoder_layerdrop (float): The layer dropout probability for the decoder.
                - pad_token_id (int): The token id used for padding.
                - max_target_positions (int): The maximum target sequence length.
                - max_source_positions (int): The maximum source sequence length.
                - d_model (int): The dimensionality of the model.
                - scale_embedding (bool): Indicates whether to scale the embeddings.
                - vocab_size (int): The size of the vocabulary.
                - decoder_layers (int): The number of decoder layers.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.max_target_positions = config.max_target_positions
        self.max_source_positions = config.max_source_positions
        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

        self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, padding_idx=self.padding_idx)
        self.embed_positions = WhisperPositionalEmbedding(self.max_target_positions, config.d_model)

        self.layers = nn.CellList([WhisperDecoderLayer(config) for _ in range(config.decoder_layers)])

        self.layer_norm = nn.LayerNorm([config.d_model])

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        Returns the input embeddings for the WhisperDecoder class.

        Args:
            self: An instance of the WhisperDecoder class.

        Returns:
            embed_tokens: This method returns the input embeddings for the decoder.

        Raises:
            None.
        """
        return self.embed_tokens

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperDecoder class.

        Args:
            self (WhisperDecoder): The instance of the WhisperDecoder class.
            value: The input embeddings to be set for the WhisperDecoder.
                This parameter should be of the appropriate type and format required for input embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.embed_tokens = value

    def construct(
        self,
        input_ids=None,
        attention_mask=None,
        encoder_hidden_states=None,
        head_mask=None,
        cross_attn_head_mask=None,
        past_key_values=None,
        inputs_embeds=None,
        use_cache=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it.

                Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                of the decoder.
            head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

            cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention
                on hidden heads. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

            past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
                Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

                Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
                cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
                that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
                all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            inputs_embeds (`mindspore.Tensor` of
                shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
                `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
                control over how to convert `input_ids` indices into associated vectors than the model's internal
                embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        if input_ids is not None:
            input_shape = input_ids.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        if getattr(self.config, "_flash_attn_2_enabled", False):
            # 2d mask is passed through the layers
            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
        else:
            # 4d mask is passed through the layers
            attention_mask = _prepare_4d_causal_attention_mask(
                attention_mask, input_shape, inputs_embeds, past_key_values_length
            )

        # embed positions
        if input_ids is not None:
            positions = self.embed_positions(input_ids, past_key_values_length=past_key_values_length)
        else:
            positions = self.embed_positions(inputs_embeds, past_key_values_length=past_key_values_length)

        hidden_states = inputs_embeds + positions
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..."
                )
                use_cache = False
        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
        next_decoder_cache = () if use_cache else None

        # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired
        for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
            if attn_mask is not None:
                assert attn_mask.shape[0] == (len(self.layers)), (
                    f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                    f" {head_mask.shape[0]}."
                )
        for idx, decoder_layer in enumerate(self.layers):
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = ops.rand((1,))
                if dropout_probability < self.layerdrop:
                    continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None
            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                encoder_hidden_states=encoder_hidden_states,
                layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                cross_attn_layer_head_mask=(
                    cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
                ),
                past_key_value=past_key_value,
                output_attentions=output_attentions,
                use_cache=use_cache,
            )
            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)

            if output_attentions:
                all_self_attns += (layer_outputs[1],)

                if encoder_hidden_states is not None:
                    all_cross_attentions += (layer_outputs[2],)

        hidden_states = self.layer_norm(hidden_states)
        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.__init__(config)

Initializes the WhisperDecoder class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An instance of WhisperConfig containing the configuration parameters for the decoder. dropout (float): The dropout probability. decoder_layerdrop (float): The layer dropout probability for the decoder. pad_token_id (int): The token id used for padding. max_target_positions (int): The maximum target sequence length. max_source_positions (int): The maximum source sequence length. d_model (int): The dimensionality of the model. scale_embedding (bool): Indicates whether to scale the embeddings. vocab_size (int): The size of the vocabulary. decoder_layers (int): The number of decoder layers. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes the WhisperDecoder class.

    Args:
        self: The instance of the class.
        config (WhisperConfig):
            An instance of WhisperConfig containing the configuration parameters for the decoder.

            - dropout (float): The dropout probability.
            - decoder_layerdrop (float): The layer dropout probability for the decoder.
            - pad_token_id (int): The token id used for padding.
            - max_target_positions (int): The maximum target sequence length.
            - max_source_positions (int): The maximum source sequence length.
            - d_model (int): The dimensionality of the model.
            - scale_embedding (bool): Indicates whether to scale the embeddings.
            - vocab_size (int): The size of the vocabulary.
            - decoder_layers (int): The number of decoder layers.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.dropout = config.dropout
    self.layerdrop = config.decoder_layerdrop
    self.padding_idx = config.pad_token_id
    self.max_target_positions = config.max_target_positions
    self.max_source_positions = config.max_source_positions
    self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

    self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, padding_idx=self.padding_idx)
    self.embed_positions = WhisperPositionalEmbedding(self.max_target_positions, config.d_model)

    self.layers = nn.CellList([WhisperDecoderLayer(config) for _ in range(config.decoder_layers)])

    self.layer_norm = nn.LayerNorm([config.d_model])

    self.gradient_checkpointing = False
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.construct(input_ids=None, attention_mask=None, encoder_hidden_states=None, head_mask=None, cross_attn_head_mask=None, past_key_values=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [WhisperTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
encoder_hidden_states	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional* DEFAULT: None
head_mask	Mask to nullify selected heads of the attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
cross_attn_head_mask	Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention on hidden heads. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
past_key_values	Tuple of tuple(mindspore.Tensor) of length config.n_layers, with each tuple having 2 tensors of shape (batch_size, num_heads, sequence_length, embed_size_per_head)) and 2 additional tensors of shape (batch_size, num_heads, encoder_sequence_length, embed_size_per_head). Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see past_key_values input) to speed up sequential decoding. If past_key_values are used, the user can optionally input only the last decoder_input_ids (those that don't have their past key value states given to this model) of shape (batch_size, 1) instead of all decoder_input_ids of shape (batch_size, sequence_length). TYPE: `tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    input_ids=None,
    attention_mask=None,
    encoder_hidden_states=None,
    head_mask=None,
    cross_attn_head_mask=None,
    past_key_values=None,
    inputs_embeds=None,
    use_cache=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
):
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it.

            Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention
            on hidden heads. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
            shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
            cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
            that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
            all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`mindspore.Tensor` of
            shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
            `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
            control over how to convert `input_ids` indices into associated vectors than the model's internal
            embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
    if input_ids is not None:
        input_shape = input_ids.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    if getattr(self.config, "_flash_attn_2_enabled", False):
        # 2d mask is passed through the layers
        attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
    else:
        # 4d mask is passed through the layers
        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

    # embed positions
    if input_ids is not None:
        positions = self.embed_positions(input_ids, past_key_values_length=past_key_values_length)
    else:
        positions = self.embed_positions(inputs_embeds, past_key_values_length=past_key_values_length)

    hidden_states = inputs_embeds + positions
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

    if self.gradient_checkpointing and self.training:
        if use_cache:
            logger.warning_once(
                "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..."
            )
            use_cache = False
    # decoder layers
    all_hidden_states = () if output_hidden_states else None
    all_self_attns = () if output_attentions else None
    all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
    next_decoder_cache = () if use_cache else None

    # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired
    for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
        if attn_mask is not None:
            assert attn_mask.shape[0] == (len(self.layers)), (
                f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                f" {head_mask.shape[0]}."
            )
    for idx, decoder_layer in enumerate(self.layers):
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        if output_hidden_states:
            all_hidden_states += (hidden_states,)
        if self.training:
            dropout_probability = ops.rand((1,))
            if dropout_probability < self.layerdrop:
                continue

        past_key_value = past_key_values[idx] if past_key_values is not None else None
        layer_outputs = decoder_layer(
            hidden_states,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            layer_head_mask=(head_mask[idx] if head_mask is not None else None),
            cross_attn_layer_head_mask=(
                cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
            ),
            past_key_value=past_key_value,
            output_attentions=output_attentions,
            use_cache=use_cache,
        )
        hidden_states = layer_outputs[0]

        if use_cache:
            next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)

        if output_attentions:
            all_self_attns += (layer_outputs[1],)

            if encoder_hidden_states is not None:
                all_cross_attentions += (layer_outputs[2],)

    hidden_states = self.layer_norm(hidden_states)
    # add hidden states from the last decoder layer
    if output_hidden_states:
        all_hidden_states += (hidden_states,)

    next_cache = next_decoder_cache if use_cache else None
    if not return_dict:
        return tuple(
            v
            for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attns,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.get_input_embeddings()

Returns the input embeddings for the WhisperDecoder class.

PARAMETER	DESCRIPTION
self	An instance of the WhisperDecoder class.

RETURNS	DESCRIPTION
embed_tokens	This method returns the input embeddings for the decoder.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self):
    """
    Returns the input embeddings for the WhisperDecoder class.

    Args:
        self: An instance of the WhisperDecoder class.

    Returns:
        embed_tokens: This method returns the input embeddings for the decoder.

    Raises:
        None.
    """
    return self.embed_tokens

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.set_input_embeddings(value)

Sets the input embeddings for the WhisperDecoder class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperDecoder class. TYPE: WhisperDecoder
value	The input embeddings to be set for the WhisperDecoder. This parameter should be of the appropriate type and format required for input embeddings.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperDecoder class.

    Args:
        self (WhisperDecoder): The instance of the WhisperDecoder class.
        value: The input embeddings to be set for the WhisperDecoder.
            This parameter should be of the appropriate type and format required for input embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.embed_tokens = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderLayer

Bases: Cell

The WhisperDecoderLayer class represents a single layer of the Whisper decoder model, which includes self-attention and cross-attention mechanisms. This class is designed to be used within the WhisperTransformer model for sequence-to-sequence tasks.

This class inherits from nn.Cell and contains methods for initializing the layer and performing computations on input tensors. The layer consists of self-attention, encoder attention, feedforward neural network, and layer normalization modules.

The init method sets up the layer with parameters such as embedding dimensions, attention types, dropout rates, activation functions, and normalization layers.

The construct method processes input hidden states through the self-attention mechanism, followed by encoder attention if provided. It also handles dropout, residual connections, and feedforward network transformations. The method allows for caching of key-value states and optionally returns attention weights and cached states.

Please refer to the method docstrings for detailed information on the input and output parameters, as well as their respective shapes and purposes.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperDecoderLayer(nn.Cell):

    """
    The WhisperDecoderLayer class represents a single layer of the Whisper decoder model, which includes self-attention
    and cross-attention mechanisms. This class is designed to be used within the WhisperTransformer model for
    sequence-to-sequence tasks.

    This class inherits from nn.Cell and contains methods for initializing the layer and performing computations on
    input tensors. The layer consists of self-attention, encoder attention, feedforward neural network, and layer
    normalization modules.

    The __init__ method sets up the layer with parameters such as embedding dimensions, attention types, dropout rates,
    activation functions, and normalization layers.

    The construct method processes input hidden states through the self-attention mechanism, followed by encoder
    attention if provided. It also handles dropout, residual connections, and feedforward network transformations.
    The method allows for caching of key-value states and optionally returns attention weights and cached states.

    Please refer to the method docstrings for detailed information on the input and output parameters, as well as
    their respective shapes and purposes.
    """
    def __init__(self, config: WhisperConfig):
        """
        Initializes a WhisperDecoderLayer object.

        Args:
            self (WhisperDecoderLayer): The current instance of the WhisperDecoderLayer class.
            config (WhisperConfig): An instance of the WhisperConfig class containing configuration settings.

        Returns:
            None.

        Raises:
            ValueError: If the attention type specified in the config is not supported.
            TypeError: If the input parameters are not of the expected types.
            RuntimeError: If an error occurs during the initialization process.
        """
        super().__init__()
        self.embed_dim = config.d_model
        attn_type = "default"

        self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            is_causal=True,
            config=config,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout

        self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
        self.encoder_attn = WHISPER_ATTENTION_CLASSES[attn_type](
            self.embed_dim,
            config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            config=config,
        )
        self.encoder_attn_layer_norm = nn.LayerNorm([self.embed_dim])
        self.fc1 = nn.Dense(self.embed_dim, config.decoder_ffn_dim)
        self.fc2 = nn.Dense(config.decoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm([self.embed_dim])

    def construct(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        layer_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_layer_head_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            encoder_hidden_states (`mindspore.Tensor`):
                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
            encoder_attention_mask (`mindspore.Tensor`): encoder attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            cross_attn_layer_head_mask (`mindspore.Tensor`): mask for cross-attention heads in a given layer of
                size `(decoder_attention_heads,)`.
            past_key_value (`Tuple(mindspore.Tensor)`): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        # Self Attention
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        # add present self-attn cache to positions 1,2 of present_key_value tuple
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        # Cross-Attention Block
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.encoder_attn_layer_norm(hidden_states)

            # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
                output_attentions=output_attentions,
            )

            hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
            hidden_states = residual + hidden_states

            # add cross-attn to positions 3,4 of present_key_value tuple
            present_key_value = present_key_value + cross_attn_present_key_value

        # Fully Connected
        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        if use_cache:
            outputs += (present_key_value,)

        return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderLayer.__init__(config)

Initializes a WhisperDecoderLayer object.

PARAMETER	DESCRIPTION
self	The current instance of the WhisperDecoderLayer class. TYPE: WhisperDecoderLayer
config	An instance of the WhisperConfig class containing configuration settings. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the attention type specified in the config is not supported.
TypeError	If the input parameters are not of the expected types.
RuntimeError	If an error occurs during the initialization process.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes a WhisperDecoderLayer object.

    Args:
        self (WhisperDecoderLayer): The current instance of the WhisperDecoderLayer class.
        config (WhisperConfig): An instance of the WhisperConfig class containing configuration settings.

    Returns:
        None.

    Raises:
        ValueError: If the attention type specified in the config is not supported.
        TypeError: If the input parameters are not of the expected types.
        RuntimeError: If an error occurs during the initialization process.
    """
    super().__init__()
    self.embed_dim = config.d_model
    attn_type = "default"

    self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
        embed_dim=self.embed_dim,
        num_heads=config.decoder_attention_heads,
        dropout=config.attention_dropout,
        is_decoder=True,
        is_causal=True,
        config=config,
    )
    self.dropout = config.dropout
    self.activation_fn = ACT2FN[config.activation_function]
    self.activation_dropout = config.activation_dropout

    self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
    self.encoder_attn = WHISPER_ATTENTION_CLASSES[attn_type](
        self.embed_dim,
        config.decoder_attention_heads,
        dropout=config.attention_dropout,
        is_decoder=True,
        config=config,
    )
    self.encoder_attn_layer_norm = nn.LayerNorm([self.embed_dim])
    self.fc1 = nn.Dense(self.embed_dim, config.decoder_ffn_dim)
    self.fc2 = nn.Dense(config.decoder_ffn_dim, self.embed_dim)
    self.final_layer_norm = nn.LayerNorm([self.embed_dim])

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderLayer.construct(hidden_states, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, layer_head_mask=None, cross_attn_layer_head_mask=None, past_key_value=None, output_attentions=False, use_cache=True)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
encoder_hidden_states	cross attention input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor` DEFAULT: None
encoder_attention_mask	encoder attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
layer_head_mask	mask for attention heads in a given layer of size (encoder_attention_heads,). TYPE: `mindspore.Tensor` DEFAULT: None
cross_attn_layer_head_mask	mask for cross-attention heads in a given layer of size (decoder_attention_heads,). TYPE: `mindspore.Tensor` DEFAULT: None
past_key_value	cached past key and value projection states TYPE: `Tuple(mindspore.Tensor)` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    layer_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_layer_head_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    output_attentions: Optional[bool] = False,
    use_cache: Optional[bool] = True,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`): attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        encoder_hidden_states (`mindspore.Tensor`):
            cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
        encoder_attention_mask (`mindspore.Tensor`): encoder attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
            `(encoder_attention_heads,)`.
        cross_attn_layer_head_mask (`mindspore.Tensor`): mask for cross-attention heads in a given layer of
            size `(decoder_attention_heads,)`.
        past_key_value (`Tuple(mindspore.Tensor)`): cached past key and value projection states
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)

    # Self Attention
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    # add present self-attn cache to positions 1,2 of present_key_value tuple
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
        hidden_states=hidden_states,
        past_key_value=self_attn_past_key_value,
        attention_mask=attention_mask,
        layer_head_mask=layer_head_mask,
        output_attentions=output_attentions,
    )
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    # Cross-Attention Block
    cross_attn_present_key_value = None
    cross_attn_weights = None
    if encoder_hidden_states is not None:
        residual = hidden_states
        hidden_states = self.encoder_attn_layer_norm(hidden_states)

        # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
            hidden_states=hidden_states,
            key_value_states=encoder_hidden_states,
            attention_mask=encoder_attention_mask,
            layer_head_mask=cross_attn_layer_head_mask,
            past_key_value=cross_attn_past_key_value,
            output_attentions=output_attentions,
        )

        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        # add cross-attn to positions 3,4 of present_key_value tuple
        present_key_value = present_key_value + cross_attn_present_key_value

    # Fully Connected
    residual = hidden_states
    hidden_states = self.final_layer_norm(hidden_states)
    hidden_states = self.activation_fn(self.fc1(hidden_states))
    hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
    hidden_states = self.fc2(hidden_states)
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (self_attn_weights, cross_attn_weights)

    if use_cache:
        outputs += (present_key_value,)

    return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper

Bases: WhisperPreTrainedModel

This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the [EncoderDecoderModel] framework.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperDecoderWrapper(WhisperPreTrainedModel):
    """
    This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is
    used in combination with the [`EncoderDecoderModel`] framework.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the WhisperDecoderWrapper class.

        Args:
            self: The instance of the class.
            config (object): The configuration object containing the settings for the decoder.
                The config object should have the following attributes:

                - is_encoder_decoder (bool): Indicates if the WhisperDecoderWrapper is used as an encoder-decoder.
                This should be set to False for the WhisperDecoderWrapper class.
                - Other attributes specific to the WhisperDecoder class.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        config.is_encoder_decoder = False
        self.decoder = WhisperDecoder(config)

    def get_input_embeddings(self):
        """
        Get input embeddings for the WhisperDecoderWrapper.

        Args:
            self (WhisperDecoderWrapper):
                The instance of WhisperDecoderWrapper for which input embeddings are to be retrieved.

        Returns:
            None.

        Raises:
            None.
        """
        return self.decoder.embed_tokens

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperDecoderWrapper.

        Args:
            self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.
            value (object): The input embeddings to be set for the decoder. It should be an object of the
                desired input embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.decoder.embed_tokens = value

    def construct(self, *args, **kwargs):
        """
        Method to construct a WhisperDecoderWrapper object by invoking the decoder with the provided arguments.

        Args:
            self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.

        Returns:
            None: This method does not return any value explicitly. It delegates the construction to the decoder method.

        Raises:
            None.
        """
        return self.decoder(*args, **kwargs)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.__init__(config)

Initializes a new instance of the WhisperDecoderWrapper class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object containing the settings for the decoder. The config object should have the following attributes: is_encoder_decoder (bool): Indicates if the WhisperDecoderWrapper is used as an encoder-decoder. This should be set to False for the WhisperDecoderWrapper class. Other attributes specific to the WhisperDecoder class. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config):
    """
    Initializes a new instance of the WhisperDecoderWrapper class.

    Args:
        self: The instance of the class.
        config (object): The configuration object containing the settings for the decoder.
            The config object should have the following attributes:

            - is_encoder_decoder (bool): Indicates if the WhisperDecoderWrapper is used as an encoder-decoder.
            This should be set to False for the WhisperDecoderWrapper class.
            - Other attributes specific to the WhisperDecoder class.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    config.is_encoder_decoder = False
    self.decoder = WhisperDecoder(config)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.construct(*args, **kwargs)

Method to construct a WhisperDecoderWrapper object by invoking the decoder with the provided arguments.

PARAMETER	DESCRIPTION
self	The instance of the WhisperDecoderWrapper class. TYPE: WhisperDecoderWrapper

RETURNS	DESCRIPTION
None	This method does not return any value explicitly. It delegates the construction to the decoder method.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(self, *args, **kwargs):
    """
    Method to construct a WhisperDecoderWrapper object by invoking the decoder with the provided arguments.

    Args:
        self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.

    Returns:
        None: This method does not return any value explicitly. It delegates the construction to the decoder method.

    Raises:
        None.
    """
    return self.decoder(*args, **kwargs)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.get_input_embeddings()

Get input embeddings for the WhisperDecoderWrapper.

PARAMETER	DESCRIPTION
self	The instance of WhisperDecoderWrapper for which input embeddings are to be retrieved. TYPE: WhisperDecoderWrapper

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self):
    """
    Get input embeddings for the WhisperDecoderWrapper.

    Args:
        self (WhisperDecoderWrapper):
            The instance of WhisperDecoderWrapper for which input embeddings are to be retrieved.

    Returns:
        None.

    Raises:
        None.
    """
    return self.decoder.embed_tokens

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.set_input_embeddings(value)

Sets the input embeddings for the WhisperDecoderWrapper.

PARAMETER	DESCRIPTION
self	The instance of the WhisperDecoderWrapper class. TYPE: WhisperDecoderWrapper
value	The input embeddings to be set for the decoder. It should be an object of the desired input embeddings. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperDecoderWrapper.

    Args:
        self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.
        value (object): The input embeddings to be set for the decoder. It should be an object of the
            desired input embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.decoder.embed_tokens = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder

Bases: WhisperPreTrainedModel

Transformer encoder consisting of config.encoder_layers self attention layers. Each layer is a [WhisperEncoderLayer].

PARAMETER	DESCRIPTION
config	WhisperConfig TYPE: WhisperConfig

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperEncoder(WhisperPreTrainedModel):
    """
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
    [`WhisperEncoderLayer`].

    Args:
        config: WhisperConfig
    """
    def __init__(self, config: WhisperConfig):
        """Initialize a WhisperEncoder object.

        Args:
            config (WhisperConfig):
                The configuration object containing the parameters for the encoder.

                - dropout (float): The dropout probability for the encoder.
                - encoder_layerdrop (float): The probability of dropping an entire encoder layer.
                - d_model (int): The embedding dimension size.
                - num_mel_bins (int): The number of mel bins for the input audio.
                - pad_token_id (int): The padding token ID.
                - max_source_positions (int): The maximum number of source positions.
                - scale_embedding (bool): Whether to scale the embeddings by math.sqrt(embed_dim).

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop

        embed_dim = config.d_model
        self.num_mel_bins = config.num_mel_bins
        self.padding_idx = config.pad_token_id
        self.max_source_positions = config.max_source_positions
        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

        self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1, pad_mode='pad', has_bias=True)
        self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1, pad_mode='pad', has_bias=True)

        self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
        self.embed_positions.weight.requires_grad = False

        self.layers = nn.CellList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)])
        self.layer_norm = nn.LayerNorm([config.d_model])

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def _freeze_parameters(self):
        """
        Freeze the parameters of the WhisperEncoder.

        Args:
            self (WhisperEncoder): The instance of WhisperEncoder.

        Returns:
            None.

        Raises:
            None.
        """
        for param in self.get_parameters():
            param.requires_grad = False

    def get_input_embeddings(self) -> nn.Cell:
        """
        Get the input embeddings for the WhisperEncoder.

        Args:
            self (WhisperEncoder): The instance of the WhisperEncoder class.

        Returns:
            nn.Cell: The input embeddings.

        Raises:
            None.
        """
        return self.conv1

    def set_input_embeddings(self, value: nn.Cell):
        """
        Method to set input embeddings for the WhisperEncoder class.

        Args:
            self (WhisperEncoder): The instance of the WhisperEncoder class.
                It is used to access the attributes and methods of the class.
            value (nn.Cell): The input embeddings to be set for the WhisperEncoder.
                It should be an instance of the nn.Cell class.

        Returns:
            None: This method sets the input embeddings for the WhisperEncoder instance.

        Raises:
            None.
        """
        self.conv1 = value

    def construct(
        self,
        input_features,
        attention_mask=None,
        head_mask=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r"""
        Args:
            input_features (`mindspore.Tensor` of shape `(batch_size, feature_size, sequence_length)`):
                Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
                `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into
                `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding
                and conversion into a tensor of type `mindspore.Tensor`. See [`~WhisperFeatureExtractor.__call__`]
            attention_mask (`mindspore.Tensor`)`, *optional*):
                Whisper does not support masking of the `input_features`, this argument is preserved for compatibility,
                but it is not used. By default the silence in the input log mel spectrogram are ignored.
            head_mask (`mindspore.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        inputs_embeds = ops.gelu(self.conv1(input_features))
        inputs_embeds = ops.gelu(self.conv2(inputs_embeds))

        inputs_embeds = inputs_embeds.permute(0, 2, 1)
        embed_pos = self.embed_positions.weight

        hidden_states = inputs_embeds + embed_pos
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        # check if head_mask has a correct number of layers specified if desired
        if head_mask is not None:
            assert head_mask.shape[0] == (
                len(self.layers)
            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.shape[0]}."

        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            to_drop = False
            if self.training:
                dropout_probability = ops.rand((1,))
                if dropout_probability < self.layerdrop:  # skip the layer
                    to_drop = True

            if to_drop:
                layer_outputs = (None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.__call__,
                        hidden_states,
                        None,
                        (head_mask[idx] if head_mask is not None else None),
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        None,
                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                        output_attentions=output_attentions,
                    )

                hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        hidden_states = self.layer_norm(hidden_states)
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.__init__(config)

Initialize a WhisperEncoder object.

PARAMETER

DESCRIPTION

config

The configuration object containing the parameters for the encoder. dropout (float): The dropout probability for the encoder. encoder_layerdrop (float): The probability of dropping an entire encoder layer. d_model (int): The embedding dimension size. num_mel_bins (int): The number of mel bins for the input audio. pad_token_id (int): The padding token ID. max_source_positions (int): The maximum number of source positions. scale_embedding (bool): Whether to scale the embeddings by math.sqrt(embed_dim). TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """Initialize a WhisperEncoder object.

    Args:
        config (WhisperConfig):
            The configuration object containing the parameters for the encoder.

            - dropout (float): The dropout probability for the encoder.
            - encoder_layerdrop (float): The probability of dropping an entire encoder layer.
            - d_model (int): The embedding dimension size.
            - num_mel_bins (int): The number of mel bins for the input audio.
            - pad_token_id (int): The padding token ID.
            - max_source_positions (int): The maximum number of source positions.
            - scale_embedding (bool): Whether to scale the embeddings by math.sqrt(embed_dim).

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.dropout = config.dropout
    self.layerdrop = config.encoder_layerdrop

    embed_dim = config.d_model
    self.num_mel_bins = config.num_mel_bins
    self.padding_idx = config.pad_token_id
    self.max_source_positions = config.max_source_positions
    self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

    self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1, pad_mode='pad', has_bias=True)
    self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1, pad_mode='pad', has_bias=True)

    self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
    self.embed_positions.weight.requires_grad = False

    self.layers = nn.CellList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)])
    self.layer_norm = nn.LayerNorm([config.d_model])

    self.gradient_checkpointing = False
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.construct(input_features, attention_mask=None, head_mask=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_features	Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a .flac or .wav audio file into an array of type List[float] or a numpy.ndarray, e.g. via the soundfile library (pip install soundfile). To prepare the array into input_features, the [AutoFeatureExtractor] should be used for extracting the mel features, padding and conversion into a tensor of type mindspore.Tensor. See [~WhisperFeatureExtractor.__call__] TYPE: `mindspore.Tensor` of shape `(batch_size, feature_size, sequence_length)`
attention_mask	Whisper does not support masking of the input_features, this argument is preserved for compatibility, but it is not used. By default the silence in the input log mel spectrogram are ignored. TYPE: `mindspore.Tensor`)`, *optional* DEFAULT: None
head_mask	Mask to nullify selected heads of the attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    input_features,
    attention_mask=None,
    head_mask=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
):
    r"""
    Args:
        input_features (`mindspore.Tensor` of shape `(batch_size, feature_size, sequence_length)`):
            Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
            `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into
            `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding
            and conversion into a tensor of type `mindspore.Tensor`. See [`~WhisperFeatureExtractor.__call__`]
        attention_mask (`mindspore.Tensor`)`, *optional*):
            Whisper does not support masking of the `input_features`, this argument is preserved for compatibility,
            but it is not used. By default the silence in the input log mel spectrogram are ignored.
        head_mask (`mindspore.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    inputs_embeds = ops.gelu(self.conv1(input_features))
    inputs_embeds = ops.gelu(self.conv2(inputs_embeds))

    inputs_embeds = inputs_embeds.permute(0, 2, 1)
    embed_pos = self.embed_positions.weight

    hidden_states = inputs_embeds + embed_pos
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

    encoder_states = () if output_hidden_states else None
    all_attentions = () if output_attentions else None

    # check if head_mask has a correct number of layers specified if desired
    if head_mask is not None:
        assert head_mask.shape[0] == (
            len(self.layers)
        ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.shape[0]}."

    for idx, encoder_layer in enumerate(self.layers):
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        to_drop = False
        if self.training:
            dropout_probability = ops.rand((1,))
            if dropout_probability < self.layerdrop:  # skip the layer
                to_drop = True

        if to_drop:
            layer_outputs = (None, None)
        else:
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    encoder_layer.__call__,
                    hidden_states,
                    None,
                    (head_mask[idx] if head_mask is not None else None),
                    output_attentions,
                )
            else:
                layer_outputs = encoder_layer(
                    hidden_states,
                    None,
                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                    output_attentions=output_attentions,
                )

            hidden_states = layer_outputs[0]

        if output_attentions:
            all_attentions = all_attentions + (layer_outputs[1],)

    hidden_states = self.layer_norm(hidden_states)
    if output_hidden_states:
        encoder_states = encoder_states + (hidden_states,)

    if not return_dict:
        return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
    return BaseModelOutput(
        last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.get_input_embeddings()

Get the input embeddings for the WhisperEncoder.

PARAMETER	DESCRIPTION
self	The instance of the WhisperEncoder class. TYPE: WhisperEncoder

RETURNS	DESCRIPTION
Cell	nn.Cell: The input embeddings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Cell:
    """
    Get the input embeddings for the WhisperEncoder.

    Args:
        self (WhisperEncoder): The instance of the WhisperEncoder class.

    Returns:
        nn.Cell: The input embeddings.

    Raises:
        None.
    """
    return self.conv1

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.set_input_embeddings(value)

Method to set input embeddings for the WhisperEncoder class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperEncoder class. It is used to access the attributes and methods of the class. TYPE: WhisperEncoder
value	The input embeddings to be set for the WhisperEncoder. It should be an instance of the nn.Cell class. TYPE: Cell

RETURNS	DESCRIPTION
None	This method sets the input embeddings for the WhisperEncoder instance.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value: nn.Cell):
    """
    Method to set input embeddings for the WhisperEncoder class.

    Args:
        self (WhisperEncoder): The instance of the WhisperEncoder class.
            It is used to access the attributes and methods of the class.
        value (nn.Cell): The input embeddings to be set for the WhisperEncoder.
            It should be an instance of the nn.Cell class.

    Returns:
        None: This method sets the input embeddings for the WhisperEncoder instance.

    Raises:
        None.
    """
    self.conv1 = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoderLayer

Bases: Cell

The WhisperEncoderLayer class represents a single layer of the Whisper Encoder, which is used in the training and inference process of the Whisper model. This class inherits from the nn.Cell class.

ATTRIBUTE	DESCRIPTION
`embed_dim`	The dimension size of the input embedding. TYPE: int
`self_attn`	The self-attention module used in the encoder layer. TYPE: Cell
`self_attn_layer_norm`	Layer normalization module applied to the output of the self-attention module. TYPE: LayerNorm
`dropout`	Dropout probability applied to the output of the self-attention module. TYPE: float
`activation_fn`	Activation function applied to the output of the feed-forward network. TYPE: function
`activation_dropout`	Dropout probability applied to the output of the activation function. TYPE: float
`fc1`	First fully connected layer of the feed-forward network. TYPE: Dense
`fc2`	Second fully connected layer of the feed-forward network. TYPE: Dense
`final_layer_norm`	Layer normalization module applied to the output of the feed-forward network. TYPE: LayerNorm

METHOD	DESCRIPTION
`construct`	Constructs the encoder layer by applying the self-attention, feed-forward network, and residual connections to the input hidden states.

PARAMETER	DESCRIPTION
`hidden_states`	The input to the layer of shape (batch, seq_len, embed_dim). TYPE: Tensor
`attention_mask`	The attention mask of size (batch, 1, tgt_len, src_len), where padding elements are indicated by very large negative values. TYPE: Tensor
`layer_head_mask`	The mask for attention heads in a given layer of size (encoder_attention_heads,). TYPE: Tensor
`output_attentions`	Whether or not to return the attentions tensors of all attention layers. TYPE: bool

RETURNS	DESCRIPTION
	(mindspore.Tensor): The output hidden states of the encoder layer.

Note

The construct method does not include the signatures or any other code.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperEncoderLayer(nn.Cell):

    """
    The `WhisperEncoderLayer` class represents a single layer of the Whisper Encoder, which is used in the training
    and inference process of the Whisper model. This class inherits from the `nn.Cell` class.

    Attributes:
        `embed_dim` (int): The dimension size of the input embedding.
        `self_attn` (nn.Cell): The self-attention module used in the encoder layer.
        `self_attn_layer_norm` (nn.LayerNorm): Layer normalization module applied to the output of the
            self-attention module.
        `dropout` (float): Dropout probability applied to the output of the self-attention module.
        `activation_fn` (function): Activation function applied to the output of the feed-forward network.
        `activation_dropout` (float): Dropout probability applied to the output of the activation function.
        `fc1` (nn.Dense): First fully connected layer of the feed-forward network.
        `fc2` (nn.Dense): Second fully connected layer of the feed-forward network.
        `final_layer_norm` (nn.LayerNorm): Layer normalization module applied to the output of the feed-forward network.

    Methods:
        `construct`: Constructs the encoder layer by applying the self-attention, feed-forward network, and residual
            connections to the input hidden states.

    Args:
        `hidden_states` (mindspore.Tensor): The input to the layer of shape `(batch, seq_len, embed_dim)`.
        `attention_mask` (mindspore.Tensor): The attention mask of size `(batch, 1, tgt_len, src_len)`,
            where padding elements are indicated by very large negative values.
        `layer_head_mask` (mindspore.Tensor): The mask for attention heads in a given layer of size
            `(encoder_attention_heads,)`.
        `output_attentions` (bool, optional): Whether or not to return the attentions tensors of all attention layers.

    Returns:
        `(mindspore.Tensor)`: The output hidden states of the encoder layer.

    Raises:
        None

    Note:
        The construct method does not include the signatures or any other code.
    """
    def __init__(self, config: WhisperConfig):
        """
        Initializes a new instance of the WhisperEncoderLayer class.

        Args:
            self: The instance of the class.
            config (WhisperConfig): The configuration object for the WhisperEncoderLayer.
                It contains various settings and parameters for the WhisperEncoderLayer.

                - config.d_model (int): The embedding dimension.
                - config._flash_attn_2_enabled (bool, optional): Whether to enable the flash_attention_2.
                Defaults to False.
                - config.encoder_attention_heads (int): The number of attention heads in the self-attention layer.
                - config.attention_dropout (float): The dropout probability for the attention layer.
                - config.dropout (float): The dropout probability for the layer.
                - config.activation_function (str): The activation function to be used.
                - config.activation_dropout (float): The dropout probability for the activation function.
                - config.encoder_ffn_dim (int): The dimension of the feed-forward network.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.embed_dim = config.d_model
        attn_type = "flash_attention_2" if getattr(config, "_flash_attn_2_enabled", False) else "default"

        self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
            embed_dim=self.embed_dim,
            num_heads=config.encoder_attention_heads,
            dropout=config.attention_dropout,
            config=config,
        )
        self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout
        self.fc1 = nn.Dense(self.embed_dim, config.encoder_ffn_dim)
        self.fc2 = nn.Dense(config.encoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm([self.embed_dim])

    def construct(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: mindspore.Tensor,
        layer_head_mask: mindspore.Tensor,
        output_attentions: bool = False,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        if hidden_states.dtype == mindspore.float16 and (
            ops.isinf(hidden_states).any() or ops.isnan(hidden_states).any()
        ):
            clamp_value = np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max - 1000
            hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoderLayer.__init__(config)

Initializes a new instance of the WhisperEncoderLayer class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object for the WhisperEncoderLayer. It contains various settings and parameters for the WhisperEncoderLayer. config.d_model (int): The embedding dimension. config._flash_attn_2_enabled (bool, optional): Whether to enable the flash_attention_2. Defaults to False. config.encoder_attention_heads (int): The number of attention heads in the self-attention layer. config.attention_dropout (float): The dropout probability for the attention layer. config.dropout (float): The dropout probability for the layer. config.activation_function (str): The activation function to be used. config.activation_dropout (float): The dropout probability for the activation function. config.encoder_ffn_dim (int): The dimension of the feed-forward network. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes a new instance of the WhisperEncoderLayer class.

    Args:
        self: The instance of the class.
        config (WhisperConfig): The configuration object for the WhisperEncoderLayer.
            It contains various settings and parameters for the WhisperEncoderLayer.

            - config.d_model (int): The embedding dimension.
            - config._flash_attn_2_enabled (bool, optional): Whether to enable the flash_attention_2.
            Defaults to False.
            - config.encoder_attention_heads (int): The number of attention heads in the self-attention layer.
            - config.attention_dropout (float): The dropout probability for the attention layer.
            - config.dropout (float): The dropout probability for the layer.
            - config.activation_function (str): The activation function to be used.
            - config.activation_dropout (float): The dropout probability for the activation function.
            - config.encoder_ffn_dim (int): The dimension of the feed-forward network.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.embed_dim = config.d_model
    attn_type = "flash_attention_2" if getattr(config, "_flash_attn_2_enabled", False) else "default"

    self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
        embed_dim=self.embed_dim,
        num_heads=config.encoder_attention_heads,
        dropout=config.attention_dropout,
        config=config,
    )
    self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
    self.dropout = config.dropout
    self.activation_fn = ACT2FN[config.activation_function]
    self.activation_dropout = config.activation_dropout
    self.fc1 = nn.Dense(self.embed_dim, config.encoder_ffn_dim)
    self.fc2 = nn.Dense(config.encoder_ffn_dim, self.embed_dim)
    self.final_layer_norm = nn.LayerNorm([self.embed_dim])

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoderLayer.construct(hidden_states, attention_mask, layer_head_mask, output_attentions=False)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor`
layer_head_mask	mask for attention heads in a given layer of size (encoder_attention_heads,). TYPE: `mindspore.Tensor`
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: mindspore.Tensor,
    layer_head_mask: mindspore.Tensor,
    output_attentions: bool = False,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`): attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
            `(encoder_attention_heads,)`.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)
    hidden_states, attn_weights, _ = self.self_attn(
        hidden_states=hidden_states,
        attention_mask=attention_mask,
        layer_head_mask=layer_head_mask,
        output_attentions=output_attentions,
    )
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    residual = hidden_states
    hidden_states = self.final_layer_norm(hidden_states)
    hidden_states = self.activation_fn(self.fc1(hidden_states))
    hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
    hidden_states = self.fc2(hidden_states)
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    if hidden_states.dtype == mindspore.float16 and (
        ops.isinf(hidden_states).any() or ops.isnan(hidden_states).any()
    ):
        clamp_value = np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max - 1000
        hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (attn_weights,)

    return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification

Bases: WhisperPreTrainedModel

This class represents a Whisper model for audio classification tasks. It is a subclass of the WhisperPreTrainedModel class.

The WhisperForAudioClassification class consists of various methods and attributes that are used for audio classification tasks.

METHOD	DESCRIPTION
`__init__`	Initializes the WhisperForAudioClassification instance.
`freeze_encoder`	Disables gradient computation for the Whisper encoder, preventing its parameters from being updated during training.
`get_input_embeddings`	Retrieves the input embeddings from the encoder.
`set_input_embeddings`	Sets the input embeddings for the encoder.
`construct`	Constructs the forward pass of the model for audio classification.

ATTRIBUTE	DESCRIPTION
`encoder`	Instance of the WhisperEncoder class used for encoding audio input.
`layer_weights`	Parameter representing weights for weighted layer sum, if enabled.
`projector`	Instance of the nn.Dense class used for projecting hidden states.
`classifier`	Instance of the nn.Dense class used for classification.
`config`	Configuration object containing model settings.

Example

>>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
>>> from datasets import load_dataset
...
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
>>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
...
>>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
>>> sample = next(iter(ds))
...
>>> inputs = feature_extractor(
...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
... )
>>> input_features = inputs.input_features
...
>>> with torch.no_grad():
>>>     logits = model(input_features).logits
...
>>> predicted_class_ids = torch.argmax(logits).item()
>>> predicted_label = model.config.id2label[predicted_class_ids]
>>> predicted_label
'Afrikaans'

For more details on the class methods and attributes, refer to the individual method docstrings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperForAudioClassification(WhisperPreTrainedModel):

    """
    This class represents a Whisper model for audio classification tasks.
    It is a subclass of the `WhisperPreTrainedModel` class.

    The `WhisperForAudioClassification` class consists of various methods and attributes that are used for audio
    classification tasks.

    Methods:
        `__init__`: Initializes the `WhisperForAudioClassification` instance.
        `freeze_encoder`: Disables gradient computation for the Whisper encoder, preventing its parameters
            from being updated during training.
        `get_input_embeddings`: Retrieves the input embeddings from the encoder.
        `set_input_embeddings`: Sets the input embeddings for the encoder.
        `construct`: Constructs the forward pass of the model for audio classification.

    Attributes:
        `encoder`: Instance of the `WhisperEncoder` class used for encoding audio input.
        `layer_weights`: Parameter representing weights for weighted layer sum, if enabled.
        `projector`: Instance of the `nn.Dense` class used for projecting hidden states.
        `classifier`: Instance of the `nn.Dense` class used for classification.
        `config`: Configuration object containing model settings.

    Example:
        ```python
        >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
        >>> from datasets import load_dataset
        ...
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        ...
        >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
        >>> sample = next(iter(ds))
        ...
        >>> inputs = feature_extractor(
        ...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
        ... )
        >>> input_features = inputs.input_features
        ...
        >>> with torch.no_grad():
        >>>     logits = model(input_features).logits
        ...
        >>> predicted_class_ids = torch.argmax(logits).item()
        >>> predicted_label = model.config.id2label[predicted_class_ids]
        >>> predicted_label
        'Afrikaans'
        ```

    For more details on the class methods and attributes, refer to the individual method docstrings.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the WhisperForAudioClassification class.

        Args:
            self: The instance of the WhisperForAudioClassification class.
            config: A configuration object containing settings for the model.
                It should be an instance of the configuration class specific to WhisperForAudioClassification.

        Returns:
            None.

        Raises:
            TypeError: If the 'config' parameter is not provided or is not of the expected type.
            ValueError: If the 'num_hidden_layers' attribute in the 'config' parameter is not defined.
            RuntimeError: If an error occurs during initialization.
        """
        super().__init__(config)

        self.encoder = WhisperEncoder(config)
        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
        if config.use_weighted_layer_sum:
            self.layer_weights = Parameter(ops.ones(num_layers) / num_layers)
        self.projector = nn.Dense(config.hidden_size, config.classifier_proj_size)
        self.classifier = nn.Dense(config.classifier_proj_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def freeze_encoder(self):
        """
        Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
        not be updated during training. Only the projection layers and classification head will be updated.
        """
        self.encoder._freeze_parameters()

    def get_input_embeddings(self) -> nn.Cell:
        """
        This method returns the input embeddings from the encoder for audio classification.

        Args:
            self (WhisperForAudioClassification): The instance of the WhisperForAudioClassification class.

        Returns:
            nn.Cell: The input embeddings from the encoder for audio classification.

        Raises:
            None
        """
        return self.encoder.get_input_embeddings()

    def set_input_embeddings(self, value: nn.Cell):
        """
        Method to set the input embeddings for the WhisperForAudioClassification class.

        Args:
            self:
                The instance of the WhisperForAudioClassification class.

                - Type: WhisperForAudioClassification
                - Purpose: Represents the current instance of the class.
                - Restrictions: None

            value:
                The input embeddings to be set for the encoder.

                - Type: nn.Cell
                - Purpose: Represents the input embeddings used for encoding.
                - Restrictions: Should be an instance of nn.Cell.

        Returns:
            None.

        Raises:
            None.
        """
        self.encoder.set_input_embeddings(value)

    def construct(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
                config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
                `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:
            Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]

        Example:
            ```python
            >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
            >>> from datasets import load_dataset
            ...
            >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
            >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
            ...
            >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
            >>> sample = next(iter(ds))
            ...
            >>> inputs = feature_extractor(
            ...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
            ... )
            >>> input_features = inputs.input_features
            ...
            >>> with torch.no_grad():
            ...     logits = model(input_features).logits
            ...
            >>> predicted_class_ids = torch.argmax(logits).item()
            >>> predicted_label = model.config.id2label[predicted_class_ids]
            >>> predicted_label
            'Afrikaans'
            ```
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_features,
                head_mask=head_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        if self.config.use_weighted_layer_sum:
            hidden_states = ops.stack(encoder_outputs, axis=1)
            norm_weights = ops.softmax(self.layer_weights, axis=-1)
            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(axis=1)
        else:
            hidden_states = encoder_outputs[0]

        hidden_states = self.projector(hidden_states)
        pooled_output = hidden_states.mean(axis=1)

        logits = self.classifier(pooled_output)

        loss = None

        if labels is not None:
            loss = ops.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1))

        if not return_dict:
            output = (logits,) + encoder_outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.__init__(config)

Initializes a new instance of the WhisperForAudioClassification class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForAudioClassification class.
config	A configuration object containing settings for the model. It should be an instance of the configuration class specific to WhisperForAudioClassification.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the 'config' parameter is not provided or is not of the expected type.
ValueError	If the 'num_hidden_layers' attribute in the 'config' parameter is not defined.
RuntimeError	If an error occurs during initialization.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config):
    """
    Initializes a new instance of the WhisperForAudioClassification class.

    Args:
        self: The instance of the WhisperForAudioClassification class.
        config: A configuration object containing settings for the model.
            It should be an instance of the configuration class specific to WhisperForAudioClassification.

    Returns:
        None.

    Raises:
        TypeError: If the 'config' parameter is not provided or is not of the expected type.
        ValueError: If the 'num_hidden_layers' attribute in the 'config' parameter is not defined.
        RuntimeError: If an error occurs during initialization.
    """
    super().__init__(config)

    self.encoder = WhisperEncoder(config)
    num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
    if config.use_weighted_layer_sum:
        self.layer_weights = Parameter(ops.ones(num_layers) / num_layers)
    self.projector = nn.Dense(config.hidden_size, config.classifier_proj_size)
    self.classifier = nn.Dense(config.classifier_proj_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.construct(input_features=None, head_mask=None, encoder_outputs=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., config.num_labels - 1]. If config.num_labels == 1 a regression loss is computed (Mean-Square loss), If config.num_labels > 1 a classification loss is computed (Cross-Entropy). TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple[Tensor], SequenceClassifierOutput]	Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]

Example

>>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
>>> from datasets import load_dataset
...
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
>>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
...
>>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
>>> sample = next(iter(ds))
...
>>> inputs = feature_extractor(
...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
... )
>>> input_features = inputs.input_features
...
>>> with torch.no_grad():
...     logits = model(input_features).logits
...
>>> predicted_class_ids = torch.argmax(logits).item()
>>> predicted_label = model.config.id2label[predicted_class_ids]
>>> predicted_label
'Afrikaans'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

    Returns:
        Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]

    Example:
        ```python
        >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
        >>> from datasets import load_dataset
        ...
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        ...
        >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
        >>> sample = next(iter(ds))
        ...
        >>> inputs = feature_extractor(
        ...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
        ... )
        >>> input_features = inputs.input_features
        ...
        >>> with torch.no_grad():
        ...     logits = model(input_features).logits
        ...
        >>> predicted_class_ids = torch.argmax(logits).item()
        >>> predicted_label = model.config.id2label[predicted_class_ids]
        >>> predicted_label
        'Afrikaans'
        ```
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if encoder_outputs is None:
        encoder_outputs = self.encoder(
            input_features,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

    if self.config.use_weighted_layer_sum:
        hidden_states = ops.stack(encoder_outputs, axis=1)
        norm_weights = ops.softmax(self.layer_weights, axis=-1)
        hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(axis=1)
    else:
        hidden_states = encoder_outputs[0]

    hidden_states = self.projector(hidden_states)
    pooled_output = hidden_states.mean(axis=1)

    logits = self.classifier(pooled_output)

    loss = None

    if labels is not None:
        loss = ops.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1))

    if not return_dict:
        output = (logits,) + encoder_outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return SequenceClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=encoder_outputs.hidden_states,
        attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.freeze_encoder()

Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will not be updated during training. Only the projection layers and classification head will be updated.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def freeze_encoder(self):
    """
    Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
    not be updated during training. Only the projection layers and classification head will be updated.
    """
    self.encoder._freeze_parameters()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.get_input_embeddings()

This method returns the input embeddings from the encoder for audio classification.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForAudioClassification class. TYPE: WhisperForAudioClassification

RETURNS	DESCRIPTION
Cell	nn.Cell: The input embeddings from the encoder for audio classification.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Cell:
    """
    This method returns the input embeddings from the encoder for audio classification.

    Args:
        self (WhisperForAudioClassification): The instance of the WhisperForAudioClassification class.

    Returns:
        nn.Cell: The input embeddings from the encoder for audio classification.

    Raises:
        None
    """
    return self.encoder.get_input_embeddings()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.set_input_embeddings(value)

Method to set the input embeddings for the WhisperForAudioClassification class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForAudioClassification class. Type: WhisperForAudioClassification Purpose: Represents the current instance of the class. Restrictions: None
value	The input embeddings to be set for the encoder. Type: nn.Cell Purpose: Represents the input embeddings used for encoding. Restrictions: Should be an instance of nn.Cell. TYPE: Cell

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value: nn.Cell):
    """
    Method to set the input embeddings for the WhisperForAudioClassification class.

    Args:
        self:
            The instance of the WhisperForAudioClassification class.

            - Type: WhisperForAudioClassification
            - Purpose: Represents the current instance of the class.
            - Restrictions: None

        value:
            The input embeddings to be set for the encoder.

            - Type: nn.Cell
            - Purpose: Represents the input embeddings used for encoding.
            - Restrictions: Should be an instance of nn.Cell.

    Returns:
        None.

    Raises:
        None.
    """
    self.encoder.set_input_embeddings(value)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM

Bases: WhisperPreTrainedModel

WhisperForCausalLM is a class representing a Whisper model for causal language modeling tasks. It inherits from WhisperPreTrainedModel and provides methods for generating text based on input sequences.

METHOD	DESCRIPTION
__init__	Initializes the WhisperForCausalLM model with the given configuration.
get_output_embeddings	Returns the output embeddings of the model.
set_output_embeddings	Sets new output embeddings for the model.
get_input_embeddings	Returns the input embeddings of the model.
set_input_embeddings	Sets new input embeddings for the model.
set_decoder	Sets the decoder for the model.
get_decoder	Returns the decoder of the model.
construct	Constructs the model architecture and processes input data for generation.
prepare_inputs_for_generation	Prepares inputs for text generation based on the provided parameters.
_reorder_cache	Reorders cache items based on a given beam index for generation.

Example

>>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
>>> from datasets import load_dataset
...
>>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
>>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
...
>>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> sample = ds[0]["audio"]
>>> input_features = processor(
...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
... ).input_features
...
>>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
...
>>> # Decode token ids to text
>>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
>>> transcription
' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperForCausalLM(WhisperPreTrainedModel):

    """
    WhisperForCausalLM is a class representing a Whisper model for causal language modeling tasks.
    It inherits from WhisperPreTrainedModel and provides methods for generating text based on input sequences.

    Methods:
        __init__: Initializes the WhisperForCausalLM model with the given configuration.
        get_output_embeddings: Returns the output embeddings of the model.
        set_output_embeddings: Sets new output embeddings for the model.
        get_input_embeddings: Returns the input embeddings of the model.
        set_input_embeddings: Sets new input embeddings for the model.
        set_decoder: Sets the decoder for the model.
        get_decoder: Returns the decoder of the model.
        construct: Constructs the model architecture and processes input data for generation.
        prepare_inputs_for_generation: Prepares inputs for text generation based on the provided parameters.
        _reorder_cache: Reorders cache items based on a given beam index for generation.

    Example:
        ```python
        >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
        >>> from datasets import load_dataset
        ...
        >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
        ...
        >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> sample = ds[0]["audio"]
        >>> input_features = processor(
        ...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
        ... ).input_features
        ...
        >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
        ...
        >>> # Decode token ids to text
        >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
        >>> transcription
        ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'
        ```
    """
    _tied_weights_keys = ["proj_out.weight"]
    main_input_name = "input_ids"

    def __init__(self, config):
        """
        Initializes an instance of the WhisperForCausalLM class.

        Args:
            self (WhisperForCausalLM): The instance of the class.
            config: A configuration object containing various settings for the model.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        config.is_encoder_decoder = False
        self.model = WhisperDecoderWrapper(config)

        self.proj_out = nn.Dense(config.hidden_size, config.vocab_size, has_bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        This method returns the output embeddings for WhisperForCausalLM model.

        Args:
            self: The instance of WhisperForCausalLM class.

        Returns:
            proj_out: This method returns the output embeddings.

        Raises:
            None.
        """
        return self.proj_out

    def set_output_embeddings(self, new_embeddings):
        """
        Set the output embeddings for the WhisperForCausalLM model.

        Args:
            self (WhisperForCausalLM): The instance of WhisperForCausalLM class.
            new_embeddings (Any): The new embeddings to be set as the output embeddings for the model.

        Returns:
            None.

        Raises:
            TypeError: If the new_embeddings parameter is not of the correct type.
            ValueError: If any restrictions or validations fail during the setting of new embeddings.
        """
        self.proj_out = new_embeddings

    def get_input_embeddings(self) -> nn.Cell:
        """
        Retrieves the input embeddings from the underlying model.

        Args:
            self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.

        Returns:
            nn.Cell: The input embeddings obtained from the underlying model.

        Raises:
            None.

        Description:
            This method returns the input embeddings of the WhisperForCausalLM model.
            The input embeddings are responsible for mapping the input tokens to their corresponding embedding vectors.
            The underlying model's 'get_input_embeddings' function is called to retrieve these embeddings.

        Note:
            - The returned input embeddings can be used for various downstream tasks such as fine-tuning or feature
            extraction.
            - It is assumed that the underlying model has a 'get_input_embeddings' method implemented.

        Example:
            ```python
            >>> model = WhisperForCausalLM()
            >>> embeddings = model.get_input_embeddings()
            ```
        """
        return self.model.get_input_embeddings()

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperForCausalLM model.

        Args:
            self: The object instance.
            value: A tensor of shape (vocab_size, hidden_size) representing the new input embeddings for the model.
                The vocab_size is the size of the vocabulary used by the model, and the hidden_size is the size of
                the hidden states in the model. The input embeddings are used to encode the input tokens in the model's
                forward pass. This parameter is required.

        Returns:
            None.

        Raises:
            None.
        """
        self.model.set_input_embeddings(value)

    def set_decoder(self, decoder):
        """
        Method to set the decoder for the WhisperForCausalLM model.

        Args:
            self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.
            decoder: The decoder to be set for the model. It should be compatible with the model's decoder architecture.

        Returns:
            None.

        Raises:
            None.
        """
        self.model.decoder = decoder

    def get_decoder(self):
        """
        Returns the decoder of the WhisperForCausalLM model.

        Args:
            self: The instance of the WhisperForCausalLM class.

        Returns:
            decoder: This method returns the decoder of the WhisperForCausalLM model.

        Raises:
            None.
        """
        return self.model.decoder

    def construct(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[mindspore.Tensor]] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            encoder_outputs  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                if the model is configured as a decoder.
            head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed
                or when `config.use_cache=True`):
                Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
                tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains
                pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
                blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If
                `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
                don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
                `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

        Returns:
            Union[Tuple, CausalLMOutputWithCrossAttentions]

        Example:
            ```python
            >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
            >>> from datasets import load_dataset
            ...
            >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
            >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
            ...
            >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
            ...
            >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
            >>> sample = ds[0]["audio"]
            >>> input_features = processor(
            ...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
            ... ).input_features
            ...
            >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
            ...
            >>> # decode token ids to text
            >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
            >>> transcription
            ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'
            ```
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # If the user passed a tuple or `BaseModelOutput` for encoder_outputs, we extract only the hidden states
        if isinstance(encoder_outputs, (BaseModelOutput, tuple, list)):
            encoder_outputs = encoder_outputs[0]

        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        outputs = self.model.decoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_outputs,
            head_mask=head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        logits = self.proj_out(outputs[0])

        loss = None
        if labels is not None:
            loss = ops.cross_entropy(logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[1:]
            return (loss,) + output if loss is not None else output

        return CausalLMOutputWithCrossAttentions(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            cross_attentions=outputs.cross_attentions,
        )

    def prepare_inputs_for_generation(
        self,
        input_ids,
        past_key_values=None,
        use_cache=None,
        encoder_outputs=None,
        attention_mask=None,
        **kwargs,
    ):
        """
        Prepare inputs for generation.

        Args:
            self (object): The instance of the class.
            input_ids (Tensor): The input tensor containing the token ids.
            past_key_values (tuple, optional): The past key values for efficient generation. Defaults to None.
            use_cache (bool, optional): Whether to use caching for the generation process. Defaults to None.
            encoder_outputs (Tensor, optional): The outputs of the encoder. Defaults to None.
            attention_mask (Tensor, optional): The attention mask for the input_ids. Defaults to None.

        Returns:
            dict: A dictionary containing the prepared inputs for generation including encoder_outputs, past_key_values,
                input_ids, use_cache, and attention_mask.

        Raises:
            ValueError: If the input_ids and past_key_values are not of compatible shape.
            IndexError: If the input_ids shape is not as expected.
        """
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        return {
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "input_ids": input_ids,
            "use_cache": use_cache,
            "attention_mask": attention_mask,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        """
        Reorders the cache of past key values for the beam search in the WhisperForCausalLM class.

        Args:
            past_key_values (tuple): A tuple containing the past key values for each layer.
                Each element in the tuple is expected to be a tensor.
            beam_idx (tensor): The indices of the beams for reordering the past key values.

        Returns:
            None: This method modifies the past_key_values in place.

        Raises:
            None.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.__init__(config)

Initializes an instance of the WhisperForCausalLM class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: WhisperForCausalLM
config	A configuration object containing various settings for the model.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config):
    """
    Initializes an instance of the WhisperForCausalLM class.

    Args:
        self (WhisperForCausalLM): The instance of the class.
        config: A configuration object containing various settings for the model.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    config.is_encoder_decoder = False
    self.model = WhisperDecoderWrapper(config)

    self.proj_out = nn.Dense(config.hidden_size, config.vocab_size, has_bias=False)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.construct(input_ids=None, attention_mask=None, encoder_outputs=None, head_mask=None, cross_attn_head_mask=None, past_key_values=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [AutoTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
encoder_outputs	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. TYPE: (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
head_mask	Mask to nullify selected heads of the attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
cross_attn_head_mask	Mask to nullify selected heads of the cross-attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
inputs_embeds	Optionally, instead of passing input_ids you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert input_ids indices into associated vectors than the model's internal embedding lookup matrix. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
labels	Labels for computing the masked language modeling loss. Indices should either be in [0, ..., config.vocab_size] or -100 (see input_ids docstring). Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
use_cache	If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values). 1 for tokens that are not masked, 0 for tokens that are masked. TYPE: `bool`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple, CausalLMOutputWithCrossAttentions]	Union[Tuple, CausalLMOutputWithCrossAttentions]

Example

>>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
>>> from datasets import load_dataset
...
>>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
>>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
...
>>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> sample = ds[0]["audio"]
>>> input_features = processor(
...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
... ).input_features
...
>>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
...
>>> # decode token ids to text
>>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
>>> transcription
' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[mindspore.Tensor]] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        encoder_outputs  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            if the model is configured as a decoder.
        head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed
            or when `config.use_cache=True`):
            Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
            shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
            tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains
            pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If
            `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
            (see `past_key_values`).

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

    Returns:
        Union[Tuple, CausalLMOutputWithCrossAttentions]

    Example:
        ```python
        >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
        >>> from datasets import load_dataset
        ...
        >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
        ...
        >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> sample = ds[0]["audio"]
        >>> input_features = processor(
        ...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
        ... ).input_features
        ...
        >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
        ...
        >>> # decode token ids to text
        >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
        >>> transcription
        ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'
        ```
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # If the user passed a tuple or `BaseModelOutput` for encoder_outputs, we extract only the hidden states
    if isinstance(encoder_outputs, (BaseModelOutput, tuple, list)):
        encoder_outputs = encoder_outputs[0]

    # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
    outputs = self.model.decoder(
        input_ids=input_ids,
        attention_mask=attention_mask,
        encoder_hidden_states=encoder_outputs,
        head_mask=head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    logits = self.proj_out(outputs[0])

    loss = None
    if labels is not None:
        loss = ops.cross_entropy(logits.view(-1, self.config.vocab_size), labels.view(-1))

    if not return_dict:
        output = (logits,) + outputs[1:]
        return (loss,) + output if loss is not None else output

    return CausalLMOutputWithCrossAttentions(
        loss=loss,
        logits=logits,
        past_key_values=outputs.past_key_values,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
        cross_attentions=outputs.cross_attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.get_decoder()

Returns the decoder of the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForCausalLM class.

RETURNS	DESCRIPTION
decoder	This method returns the decoder of the WhisperForCausalLM model.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_decoder(self):
    """
    Returns the decoder of the WhisperForCausalLM model.

    Args:
        self: The instance of the WhisperForCausalLM class.

    Returns:
        decoder: This method returns the decoder of the WhisperForCausalLM model.

    Raises:
        None.
    """
    return self.model.decoder

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.get_input_embeddings()

Retrieves the input embeddings from the underlying model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForCausalLM class. TYPE: WhisperForCausalLM

RETURNS	DESCRIPTION
Cell	nn.Cell: The input embeddings obtained from the underlying model.

Description

This method returns the input embeddings of the WhisperForCausalLM model. The input embeddings are responsible for mapping the input tokens to their corresponding embedding vectors. The underlying model's 'get_input_embeddings' function is called to retrieve these embeddings.

Note

• The returned input embeddings can be used for various downstream tasks such as fine-tuning or feature extraction.
• It is assumed that the underlying model has a 'get_input_embeddings' method implemented.

Example

>>> model = WhisperForCausalLM()
>>> embeddings = model.get_input_embeddings()

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Cell:
    """
    Retrieves the input embeddings from the underlying model.

    Args:
        self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.

    Returns:
        nn.Cell: The input embeddings obtained from the underlying model.

    Raises:
        None.

    Description:
        This method returns the input embeddings of the WhisperForCausalLM model.
        The input embeddings are responsible for mapping the input tokens to their corresponding embedding vectors.
        The underlying model's 'get_input_embeddings' function is called to retrieve these embeddings.

    Note:
        - The returned input embeddings can be used for various downstream tasks such as fine-tuning or feature
        extraction.
        - It is assumed that the underlying model has a 'get_input_embeddings' method implemented.

    Example:
        ```python
        >>> model = WhisperForCausalLM()
        >>> embeddings = model.get_input_embeddings()
        ```
    """
    return self.model.get_input_embeddings()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.get_output_embeddings()

This method returns the output embeddings for WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of WhisperForCausalLM class.

RETURNS	DESCRIPTION
proj_out	This method returns the output embeddings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_output_embeddings(self):
    """
    This method returns the output embeddings for WhisperForCausalLM model.

    Args:
        self: The instance of WhisperForCausalLM class.

    Returns:
        proj_out: This method returns the output embeddings.

    Raises:
        None.
    """
    return self.proj_out

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.prepare_inputs_for_generation(input_ids, past_key_values=None, use_cache=None, encoder_outputs=None, attention_mask=None, **kwargs)

Prepare inputs for generation.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
input_ids	The input tensor containing the token ids. TYPE: Tensor
past_key_values	The past key values for efficient generation. Defaults to None. TYPE: tuple DEFAULT: None
use_cache	Whether to use caching for the generation process. Defaults to None. TYPE: bool DEFAULT: None
encoder_outputs	The outputs of the encoder. Defaults to None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask for the input_ids. Defaults to None. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
dict	A dictionary containing the prepared inputs for generation including encoder_outputs, past_key_values, input_ids, use_cache, and attention_mask.

RAISES	DESCRIPTION
ValueError	If the input_ids and past_key_values are not of compatible shape.
IndexError	If the input_ids shape is not as expected.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def prepare_inputs_for_generation(
    self,
    input_ids,
    past_key_values=None,
    use_cache=None,
    encoder_outputs=None,
    attention_mask=None,
    **kwargs,
):
    """
    Prepare inputs for generation.

    Args:
        self (object): The instance of the class.
        input_ids (Tensor): The input tensor containing the token ids.
        past_key_values (tuple, optional): The past key values for efficient generation. Defaults to None.
        use_cache (bool, optional): Whether to use caching for the generation process. Defaults to None.
        encoder_outputs (Tensor, optional): The outputs of the encoder. Defaults to None.
        attention_mask (Tensor, optional): The attention mask for the input_ids. Defaults to None.

    Returns:
        dict: A dictionary containing the prepared inputs for generation including encoder_outputs, past_key_values,
            input_ids, use_cache, and attention_mask.

    Raises:
        ValueError: If the input_ids and past_key_values are not of compatible shape.
        IndexError: If the input_ids shape is not as expected.
    """
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    return {
        "encoder_outputs": encoder_outputs,
        "past_key_values": past_key_values,
        "input_ids": input_ids,
        "use_cache": use_cache,
        "attention_mask": attention_mask,
    }

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.set_decoder(decoder)

Method to set the decoder for the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForCausalLM class. TYPE: WhisperForCausalLM
decoder	The decoder to be set for the model. It should be compatible with the model's decoder architecture.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_decoder(self, decoder):
    """
    Method to set the decoder for the WhisperForCausalLM model.

    Args:
        self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.
        decoder: The decoder to be set for the model. It should be compatible with the model's decoder architecture.

    Returns:
        None.

    Raises:
        None.
    """
    self.model.decoder = decoder

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.set_input_embeddings(value)

Sets the input embeddings for the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The object instance.
value	A tensor of shape (vocab_size, hidden_size) representing the new input embeddings for the model. The vocab_size is the size of the vocabulary used by the model, and the hidden_size is the size of the hidden states in the model. The input embeddings are used to encode the input tokens in the model's forward pass. This parameter is required.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperForCausalLM model.

    Args:
        self: The object instance.
        value: A tensor of shape (vocab_size, hidden_size) representing the new input embeddings for the model.
            The vocab_size is the size of the vocabulary used by the model, and the hidden_size is the size of
            the hidden states in the model. The input embeddings are used to encode the input tokens in the model's
            forward pass. This parameter is required.

    Returns:
        None.

    Raises:
        None.
    """
    self.model.set_input_embeddings(value)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.set_output_embeddings(new_embeddings)

Set the output embeddings for the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of WhisperForCausalLM class. TYPE: WhisperForCausalLM
new_embeddings	The new embeddings to be set as the output embeddings for the model. TYPE: Any

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the new_embeddings parameter is not of the correct type.
ValueError	If any restrictions or validations fail during the setting of new embeddings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_output_embeddings(self, new_embeddings):
    """
    Set the output embeddings for the WhisperForCausalLM model.

    Args:
        self (WhisperForCausalLM): The instance of WhisperForCausalLM class.
        new_embeddings (Any): The new embeddings to be set as the output embeddings for the model.

    Returns:
        None.

    Raises:
        TypeError: If the new_embeddings parameter is not of the correct type.
        ValueError: If any restrictions or validations fail during the setting of new embeddings.
    """
    self.proj_out = new_embeddings

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration

Bases: WhisperPreTrainedModel

The WhisperForConditionalGeneration class is a model class for conditional text generation, inheriting from WhisperPreTrainedModel. It provides methods for initializing the model, generating sequences of token ids, preparing inputs for generation, and extracting token-level timestamps for predicted tokens.

The class contains methods such as construct, generate, prepare_inputs_for_generation, and _reorder_cache for handling conditional generation tasks. It also includes methods for freezing the encoder, getting the encoder and decoder, and managing the input and output embeddings.

The class's main methods include:

• construct: Prepares inputs and generates sequences of token ids for conditional text generation, allowing for the configuration of various generation parameters.
• generate: Generates sequences of token ids for models with a language modeling head, allowing for custom logits processors, stopping criteria, and other advanced generation parameters.
• prepare_inputs_for_generation: Prepares input data for generation, including decoder input ids, past key values, cache usage, encoder outputs, and attention masks.
• _reorder_cache: Reorders the past key values based on beam indices during generation.
• _extract_token_timestamps: Calculates token-level timestamps using encoder-decoder cross-attentions and dynamic time-warping (DTW) to map each output token to a position in the input audio.

This class provides a comprehensive set of tools for conditional text generation tasks, including multilingual and multitask generation support, as well as token-level timestamps extraction for predicted tokens.

For more details on how to use the class and its methods, including code examples, refer to the official documentation and the following guide.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperForConditionalGeneration(WhisperPreTrainedModel):

    """
    The `WhisperForConditionalGeneration` class is a model class for conditional text generation, inheriting from 
    `WhisperPreTrainedModel`. It provides methods for initializing the model, generating sequences of token ids, 
    preparing inputs for generation, and extracting token-level timestamps for predicted tokens.

    The class contains methods such as `construct`, `generate`, `prepare_inputs_for_generation`, and `_reorder_cache` 
    for handling conditional generation tasks. It also includes methods for freezing the encoder, getting the encoder 
    and decoder, and managing the input and output embeddings.

    The class's main methods include:

    - `construct`: Prepares inputs and generates sequences of token ids for conditional text generation, allowing for 
    the configuration of various generation parameters.
    - `generate`: Generates sequences of token ids for models with a language modeling head, allowing for custom logits 
    processors, stopping criteria, and other advanced generation parameters.
    - `prepare_inputs_for_generation`: Prepares input data for generation, including decoder input ids, past key values, 
    cache usage, encoder outputs, and attention masks.
    - `_reorder_cache`: Reorders the past key values based on beam indices during generation.
    - `_extract_token_timestamps`: Calculates token-level timestamps using encoder-decoder cross-attentions and dynamic 
    time-warping (DTW) to map each output token to a position in the input audio.

    This class provides a comprehensive set of tools for conditional text generation tasks, including multilingual 
    and multitask generation support, as well as token-level timestamps extraction for predicted tokens.

    For more details on how to use the class and its methods, including code examples, refer to the official 
    documentation and the [following guide](./generation_strategies).
    """
    base_model_prefix = "model"
    _tied_weights_keys = ["proj_out.weight"]

    def __init__(self, config: WhisperConfig):
        """
        Initializes an instance of the WhisperForConditionalGeneration class.

        Args:
            self (WhisperForConditionalGeneration): The instance of the class itself.
            config (WhisperConfig): An instance of WhisperConfig containing configuration parameters for the model.

        Returns:
            None.

        Raises:
            AssertionError: If the config parameter is not of type WhisperConfig.
            ValueError: If an unexpected error occurs during initialization.
        """
        super().__init__(config)
        self.model = WhisperModel(config)
        self.proj_out = nn.Dense(config.d_model, config.vocab_size, has_bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_encoder(self):
        """
        Retrieves the encoder from the model instance.

        Args:
            self (WhisperForConditionalGeneration): The object instance.

        Returns:
            None.

        Raises:
            None.

        """
        return self.model.get_encoder()

    def get_decoder(self):
        """
        This method 'get_decoder' is part of the class 'WhisperForConditionalGeneration' and retrieves 
        the decoder from the model.

        Args:
            self:
                Instance of the 'WhisperForConditionalGeneration' class.

                - Type: object
                - Purpose: Represents the current instance of the class.
                - Restrictions: This parameter is required for accessing the decoder.

        Returns:
            None:

                - Type: None
                - Purpose: The method returns None as it retrieves the decoder from the model.

        Raises:
            None.
        """
        return self.model.get_decoder()

    def get_output_embeddings(self):
        """
        Method to retrieve the output embeddings from the WhisperForConditionalGeneration class.

        Args:
            self:
                An instance of the WhisperForConditionalGeneration class.

                - Type: WhisperForConditionalGeneration
                - Purpose: Represents the current object of the class.
                - Restrictions: Must be an instance of WhisperForConditionalGeneration class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.proj_out

    def set_output_embeddings(self, new_embeddings):
        """
        This method sets the output embeddings for the WhisperForConditionalGeneration class.

        Args:
            self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.
            new_embeddings (any): The new embeddings to be set as the output embeddings for the 
                WhisperForConditionalGeneration class. It can be of any type.

        Returns:
            None.

        Raises:
            None.
        """
        self.proj_out = new_embeddings

    def get_input_embeddings(self) -> nn.Cell:
        """
        Returns the input embeddings for the WhisperForConditionalGeneration model.

        Args:
            self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.

        Returns:
            nn.Cell: The input embeddings for the model.

        Raises:
            None.
        """
        return self.model.get_input_embeddings()

    def freeze_encoder(self):
        """
        Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
        not be updated during training.
        """
        self.model.encoder._freeze_parameters()

    def construct(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        decoder_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        decoder_inputs_embeds: Optional[Tuple[mindspore.Tensor]] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]`
                or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is
                only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:
            Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

        Example:
            ```python
            >>> from transformers import AutoProcessor, WhisperForConditionalGeneration
            >>> from datasets import load_dataset
            ...
            >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
            >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
            ...
            >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
            ...
            >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
            >>> input_features = inputs.input_features
            ...
            >>> generated_ids = model.generate(inputs=input_features)
            ...
            >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
            >>> transcription
            ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
            ```
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None:
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        outputs = self.model(
            input_features,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        lm_logits = self.proj_out(outputs[0])

        loss = None
        if labels is not None:
            loss = ops.cross(lm_logits.view(-1, self.config.vocab_size), labels.reshape(-1))

        if not return_dict:
            output = (lm_logits,) + outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return Seq2SeqLMOutput(
            loss=loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,
            decoder_hidden_states=outputs.decoder_hidden_states,
            decoder_attentions=outputs.decoder_attentions,
            cross_attentions=outputs.cross_attentions,
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
            encoder_hidden_states=outputs.encoder_hidden_states,
            encoder_attentions=outputs.encoder_attentions,
        )

    def generate(
        self,
        inputs: Optional[mindspore.Tensor] = None,
        generation_config=None,
        logits_processor=None,
        stopping_criteria=None,
        prefix_allowed_tokens_fn=None,
        synced_gpus=False,
        return_timestamps=None,
        task=None,
        language=None,
        is_multilingual=None,
        prompt_ids: Optional[mindspore.Tensor] = None,
        return_token_timestamps=None,
        **kwargs,
    ):
        """

        Generates sequences of token ids for models with a language modeling head.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Parameters:
            inputs (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
                The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
                method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
                should of in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of
                `input_ids`, `input_values`, `input_features`, or `pixel_values`.
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            logits_processor (`LogitsProcessorList`, *optional*):
                Custom logits processors that complement the default logits processors built from arguments and
                generation config. If a logit processor is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            stopping_criteria (`StoppingCriteriaList`, *optional*):
                Custom stopping criteria that complement the default stopping criteria built from arguments and a
                generation config. If a stopping criteria is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
                If provided, this function constraints the beam search to allowed tokens only at each step. If not
                provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
                `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
                on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
                for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
                Retrieval](https://arxiv.org/abs/2010.00904).
            synced_gpus (`bool`, *optional*, defaults to `False`):
                Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
            return_timestamps (`bool`, *optional*):
                Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`.
            task (`str`, *optional*):
                Task to use for generation, either "translate" or "transcribe". The `model.config.forced_decoder_ids`
                will be updated accordingly.
            language (`str`, *optional*):
                Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. You can
                find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary.
            is_multilingual (`bool`, *optional*):
                Whether or not the model is multilingual.
            prompt_ids (`mindspore.Tensor`, *optional*):
                Rank-1 tensor of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is
                provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for
                transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words
                correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value.
            return_token_timestamps (`bool`, *optional*):
                Whether to return token-level timestamps with the text. This can be used with or without the
                `return_timestamps` option. To get word-level timestamps, use the tokenizer to group the tokens into
                words.
            kwargs (`Dict[str, Any]`, *optional*):
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
                specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.

        Returns:
            [`~utils.ModelOutput`] or `mindspore.Tensor`:
                A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` or when
                `config.return_dict_in_generate=True`) or a `mindspore.Tensor`.
                If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
                [`~utils.ModelOutput`] types are:

                - [`~generation.GreedySearchDecoderOnlyOutput`],
                - [`~generation.SampleDecoderOnlyOutput`],
                - [`~generation.BeamSearchDecoderOnlyOutput`],
                - [`~generation.BeamSampleDecoderOnlyOutput`]

                If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
                [`~utils.ModelOutput`] types are:

                - [`~generation.GreedySearchEncoderDecoderOutput`],
                - [`~generation.SampleEncoderDecoderOutput`],
                - [`~generation.BeamSearchEncoderDecoderOutput`],
                - [`~generation.BeamSampleEncoderDecoderOutput`]
        """
        if generation_config is None:
            generation_config = self.generation_config

        if return_timestamps is not None:
            if not hasattr(generation_config, "no_timestamps_token_id"):
                raise ValueError(
                    "You are trying to return timestamps, but the generation config is not properly set. "
                    "Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`. "
                    "For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363"
                )

            generation_config.return_timestamps = return_timestamps
        else:
            generation_config.return_timestamps = False

        if is_multilingual is not None:
            if not hasattr(generation_config, "is_multilingual"):
                raise ValueError(
                    "The generation config is outdated and is thus not compatible with the `is_multilingual` argument "
                    "to `generate`. Please update the generation config as per the instructions "
                    "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
                )
            generation_config.is_multilingual = is_multilingual

        if hasattr(generation_config, "is_multilingual") and not generation_config.is_multilingual:
            if task is not None or language is not None:
                raise ValueError(
                    "Cannot specify `task` or `language` for an English-only model. If the model is intended to be "
                    "multilingual, pass `is_multilingual=True` to generate, or update the generation config."
                )

        if language is not None:
            if not hasattr(generation_config, "lang_to_id"):
                raise ValueError(
                    "The generation config is outdated and is thus not compatible with the `language` argument "
                    "to `generate`. Either set the language using the `forced_decoder_ids` in the model config, "
                    "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
                )
            language = language.lower()
            generation_config.language = language
        if task is not None:
            if not hasattr(generation_config, "task_to_id"):
                raise ValueError(
                    "The generation config is outdated and is thus not compatible with the `task` argument "
                    "to `generate`. Either set the task using the `forced_decoder_ids` in the model config, "
                    "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
                )
            generation_config.task = task

        forced_decoder_ids = None

        # Legacy code for backward compatibility
        if hasattr(self.config, "forced_decoder_ids") and self.config.forced_decoder_ids is not None:
            forced_decoder_ids = self.config.forced_decoder_ids
        elif (
            hasattr(self.generation_config, "forced_decoder_ids")
            and self.generation_config.forced_decoder_ids is not None
        ):
            forced_decoder_ids = self.generation_config.forced_decoder_ids
        else:
            forced_decoder_ids = kwargs.get("forced_decoder_ids", None)

        if task is not None or language is not None or (forced_decoder_ids is None and prompt_ids is not None):
            forced_decoder_ids = []
            if hasattr(generation_config, "language"):
                if generation_config.language in generation_config.lang_to_id.keys():
                    language_token = generation_config.language
                elif generation_config.language in TO_LANGUAGE_CODE.keys():
                    language_token = f"<|{TO_LANGUAGE_CODE[generation_config.language]}|>"
                elif generation_config.language in TO_LANGUAGE_CODE.values():
                    language_token = f"<|{generation_config.language}|>"
                else:
                    is_language_code = len(generation_config.language) == 2
                    raise ValueError(
                        f"Unsupported language: {generation_config.language}. Language should be one of:"
                        f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}."
                    )
                forced_decoder_ids.append((1, generation_config.lang_to_id[language_token]))
            else:
                forced_decoder_ids.append((1, None))  # automatically detect the language

            if hasattr(generation_config, "task"):
                if generation_config.task in TASK_IDS:
                    forced_decoder_ids.append((2, generation_config.task_to_id[generation_config.task]))
                else:
                    raise ValueError(
                        f"The `{generation_config.task}`task is not supported. The task should be one of `{TASK_IDS}`"
                    )
            elif hasattr(generation_config, "task_to_id"):
                forced_decoder_ids.append((2, generation_config.task_to_id["transcribe"]))  # defaults to transcribe
            if hasattr(generation_config, "no_timestamps_token_id") and not generation_config.return_timestamps:
                idx = forced_decoder_ids[-1][0] + 1 if forced_decoder_ids else 1
                forced_decoder_ids.append((idx, generation_config.no_timestamps_token_id))

        if forced_decoder_ids is not None:
            generation_config.forced_decoder_ids = forced_decoder_ids

        if prompt_ids is not None:
            if kwargs.get("decoder_start_token_id") is not None:
                raise ValueError(
                    "When specifying `prompt_ids`, you cannot also specify `decoder_start_token_id` as it gets overwritten."
                )
            prompt_ids = prompt_ids.tolist()
            decoder_start_token_id, *text_prompt_ids = prompt_ids
            # Slicing the text prompt ids in a manner consistent with the OpenAI implementation
            # to accomodate context space for the prefix (see https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/decoding.py#L599)
            text_prompt_ids = text_prompt_ids[-self.config.max_target_positions // 2 - 1 :]
            # Set the decoder_start_token_id to <|startofprev|>
            kwargs.update({"decoder_start_token_id": decoder_start_token_id})

            # If the user passes `max_new_tokens`, increase its number to account for the prompt
            if kwargs.get("max_new_tokens", None) is not None:
                kwargs["max_new_tokens"] += len(text_prompt_ids)
                if kwargs["max_new_tokens"] >= self.config.max_target_positions:
                    raise ValueError(
                        f"The length of the sliced `prompt_ids` is {len(text_prompt_ids)}, and the `max_new_tokens` "
                        f"{kwargs['max_new_tokens'] - len(text_prompt_ids)}. Thus, the combined length of the sliced "
                        f"`prompt_ids` and `max_new_tokens` is: {kwargs['max_new_tokens']}. This exceeds the "
                        f"`max_target_positions` of the Whisper model: {self.config.max_target_positions}. "
                        "You should either reduce the length of your prompt, or reduce the value of `max_new_tokens`, "
                        f"so that their combined length is less that {self.config.max_target_positions}."
                    )

            # Reformat the forced_decoder_ids to incorporate the prompt
            non_prompt_forced_decoder_ids = (
                kwargs.pop("forced_decoder_ids", None) or generation_config.forced_decoder_ids
            )
            forced_decoder_ids = [
                *text_prompt_ids,
                generation_config.decoder_start_token_id,
                *[token for _rank, token in non_prompt_forced_decoder_ids],
            ]
            forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_decoder_ids)]
            generation_config.forced_decoder_ids = forced_decoder_ids

        if generation_config.return_timestamps:
            logits_processor = [WhisperTimeStampLogitsProcessor(generation_config)]

        if return_token_timestamps:
            kwargs["output_attentions"] = True
            kwargs["return_dict_in_generate"] = True

            if getattr(generation_config, "task", None) == "translate":
                logger.warning("Token-level timestamps may not be reliable for task 'translate'.")
            if not hasattr(generation_config, "alignment_heads"):
                raise ValueError(
                    "Model generation config has no `alignment_heads`, token-level timestamps not available. "
                    "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config."
                )

            if kwargs.get("num_frames") is not None:
                generation_config.num_frames = kwargs.pop("num_frames")

        outputs = super().generate(
            inputs,
            generation_config,
            logits_processor,
            stopping_criteria,
            prefix_allowed_tokens_fn,
            synced_gpus,
            **kwargs,
        )

        if return_token_timestamps and hasattr(generation_config, "alignment_heads"):
            num_frames = getattr(generation_config, "num_frames", None)
            outputs["token_timestamps"] = self._extract_token_timestamps(
                outputs, generation_config.alignment_heads, num_frames=num_frames
            )

        return outputs

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        use_cache=None,
        encoder_outputs=None,
        attention_mask=None,
        **kwargs,
    ):
        """
        Prepare inputs for generation.

        Args:
            self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.
            decoder_input_ids (torch.Tensor): The input tensor for the decoder.
                Shape: (batch_size, sequence_length).
            past_key_values (tuple, optional): The past key values for caching computations in auto-regressive decoding.
                Default: None.
            use_cache (bool, optional): Whether to use caching for fast decoding.
                Default: None.
            encoder_outputs (torch.Tensor, optional): The output of the encoder.
                Shape: (batch_size, sequence_length, hidden_size).
                Default: None.
            attention_mask (torch.Tensor, optional): The attention mask for the decoder input.
                Shape: (batch_size, sequence_length).
                Default: None.
            **kwargs: Additional keyword arguments.

        Returns:
            dict: A dictionary containing the prepared inputs for generation.
                It includes the following keys:

                - 'encoder_outputs' (torch.Tensor): The output of the encoder.
                - 'past_key_values' (tuple): The past key values for caching computations in auto-regressive decoding.
                - 'decoder_input_ids' (torch.Tensor): The input tensor for the decoder.
                - 'use_cache' (bool): Whether to use caching for fast decoding.
                - 'decoder_attention_mask' (None): The attention mask for the decoder input.

        Raises:
            None.
        """
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if decoder_input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = decoder_input_ids.shape[1] - 1

            decoder_input_ids = decoder_input_ids[:, remove_prefix_length:]

        return {
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "use_cache": use_cache,
            "decoder_attention_mask": None,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        """
        Reorders the cache according to the provided beam index.

        Args:
            past_key_values (tuple): A tuple containing the past key-value states for each layer.
                Each element in the tuple is a tensor representing the past state of a layer.
            beam_idx (Tensor): A tensor containing the indices of the beams to be reordered.

        Returns:
            tuple: A tuple containing the reordered past key-value states for each layer. Each element in the tuple
                is a tensor representing the reordered past state of a layer.

        Raises:
            None.

        This static method takes the past key-value states and a beam index tensor, and reorders the past key-value
            states according to the beam index. It returns the reordered past key-value states as a
            tuple, where each element in the tuple represents the reordered past state of a layer.

        Note:
            The returned reordered_past tuple has the same length as the number of layers in the model, and each
                element in the tuple has the same shape as the corresponding element in the past_key_values tuple.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

    def _extract_token_timestamps(self, generate_outputs, alignment_heads, time_precision=0.02, num_frames=None):
        """
        Calculates token-level timestamps using the encoder-decoder cross-attentions and dynamic time-warping (DTW) to
        map each output token to a position in the input audio. If `num_frames` is specified, the encoder-decoder
        cross-attentions will be cropped before applying DTW.

        Returns:
            tensor containing the timestamps in seconds for each predicted token
        """
        # Create a list with `decoder_layers` elements, each a tensor of shape
        # (batch size, attention_heads, output length, input length).
        cross_attentions = []
        for i in range(self.config.decoder_layers):
            cross_attentions.append(ops.cat([x[i] for x in generate_outputs.cross_attentions], axis=2))

        # Select specific cross-attention layers and heads. This is a tensor
        # of shape (batch size, num selected, output length, input length).
        weights = ops.stack([cross_attentions[l][:, h] for l, h in alignment_heads])
        weights = weights.permute([1, 0, 2, 3])
        if num_frames is not None:
            weights = weights[..., : num_frames // 2]

        # Normalize and smoothen the weights.
        std, mean = ops.std_mean(weights, axis=-2, keepdims=True)
        weights = (weights - mean) / std
        weights = _median_filter(weights, self.config.median_filter_width)

        # Average the different cross-attention heads.
        matrix = weights.mean(axis=1)

        timestamps = ops.zeros_like(generate_outputs.sequences, dtype=mindspore.float32)

        # Perform dynamic time warping on each element of the batch.
        for batch_idx in range(timestamps.shape[0]):
            text_indices, time_indices = _dynamic_time_warping(-matrix[batch_idx].asnumpy())
            jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)
            jump_times = time_indices[jumps] * time_precision
            timestamps[batch_idx, 1:] = mindspore.tensor(jump_times)

        return timestamps

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.__init__(config)

Initializes an instance of the WhisperForConditionalGeneration class.

PARAMETER	DESCRIPTION
self	The instance of the class itself. TYPE: WhisperForConditionalGeneration
config	An instance of WhisperConfig containing configuration parameters for the model. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
AssertionError	If the config parameter is not of type WhisperConfig.
ValueError	If an unexpected error occurs during initialization.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes an instance of the WhisperForConditionalGeneration class.

    Args:
        self (WhisperForConditionalGeneration): The instance of the class itself.
        config (WhisperConfig): An instance of WhisperConfig containing configuration parameters for the model.

    Returns:
        None.

    Raises:
        AssertionError: If the config parameter is not of type WhisperConfig.
        ValueError: If an unexpected error occurs during initialization.
    """
    super().__init__(config)
    self.model = WhisperModel(config)
    self.proj_out = nn.Dense(config.d_model, config.vocab_size, has_bias=False)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.construct(input_features=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, past_key_values=None, decoder_inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the language modeling loss. Indices should either be in [0, ..., config.vocab_size] or -100 (see input_ids docstring). Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple[Tensor], Seq2SeqLMOutput]	Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

Example

>>> from transformers import AutoProcessor, WhisperForConditionalGeneration
>>> from datasets import load_dataset
...
>>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
>>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
...
>>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_features = inputs.input_features
...
>>> generated_ids = model.generate(inputs=input_features)
...
>>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
>>> transcription
' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    decoder_input_ids: Optional[mindspore.Tensor] = None,
    decoder_attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    decoder_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    decoder_inputs_embeds: Optional[Tuple[mindspore.Tensor]] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]`
            or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is
            only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

    Returns:
        Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

    Example:
        ```python
        >>> from transformers import AutoProcessor, WhisperForConditionalGeneration
        >>> from datasets import load_dataset
        ...
        >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        ...
        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_features = inputs.input_features
        ...
        >>> generated_ids = model.generate(inputs=input_features)
        ...
        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if labels is not None:
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            decoder_input_ids = shift_tokens_right(
                labels, self.config.pad_token_id, self.config.decoder_start_token_id
            )

    outputs = self.model(
        input_features,
        attention_mask=attention_mask,
        decoder_input_ids=decoder_input_ids,
        encoder_outputs=encoder_outputs,
        decoder_attention_mask=decoder_attention_mask,
        head_mask=head_mask,
        decoder_head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        past_key_values=past_key_values,
        decoder_inputs_embeds=decoder_inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )
    lm_logits = self.proj_out(outputs[0])

    loss = None
    if labels is not None:
        loss = ops.cross(lm_logits.view(-1, self.config.vocab_size), labels.reshape(-1))

    if not return_dict:
        output = (lm_logits,) + outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return Seq2SeqLMOutput(
        loss=loss,
        logits=lm_logits,
        past_key_values=outputs.past_key_values,
        decoder_hidden_states=outputs.decoder_hidden_states,
        decoder_attentions=outputs.decoder_attentions,
        cross_attentions=outputs.cross_attentions,
        encoder_last_hidden_state=outputs.encoder_last_hidden_state,
        encoder_hidden_states=outputs.encoder_hidden_states,
        encoder_attentions=outputs.encoder_attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.freeze_encoder()

Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will not be updated during training.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def freeze_encoder(self):
    """
    Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
    not be updated during training.
    """
    self.model.encoder._freeze_parameters()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.generate(inputs=None, generation_config=None, logits_processor=None, stopping_criteria=None, prefix_allowed_tokens_fn=None, synced_gpus=False, return_timestamps=None, task=None, language=None, is_multilingual=None, prompt_ids=None, return_token_timestamps=None, **kwargs)

Generates sequences of token ids for models with a language modeling head.

Most generation-controlling parameters are set in generation_config which, if not passed, will be set to the model's default generation configuration. You can override any generation_config by passing the corresponding parameters to generate(), e.g. .generate(inputs, num_beams=4, do_sample=True).

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
inputs	The sequence used as a prompt for the generation or as model inputs to the encoder. If None the method initializes it with bos_token_id and a batch size of 1. For decoder-only models inputs should of in the format of input_ids. For encoder-decoder models inputs can represent any of input_ids, input_values, input_features, or pixel_values. TYPE: `mindspore.Tensor` of varying shape depending on the modality, *optional* DEFAULT: None
generation_config	The generation configuration to be used as base parametrization for the generation call. **kwargs passed to generate matching the attributes of generation_config will override them. If generation_config is not provided, the default will be used, which had the following loading priority: 1) from the generation_config.json model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [~generation.GenerationConfig]'s default values, whose documentation should be checked to parameterize generation. TYPE: `~generation.GenerationConfig`, *optional* DEFAULT: None
logits_processor	Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `LogitsProcessorList`, *optional* DEFAULT: None
stopping_criteria	Custom stopping criteria that complement the default stopping criteria built from arguments and a generation config. If a stopping criteria is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `StoppingCriteriaList`, *optional* DEFAULT: None
prefix_allowed_tokens_fn	If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID batch_id and input_ids. It has to return a list with the allowed tokens for the next generation step conditioned on the batch ID batch_id and the previously generated tokens inputs_ids. This argument is useful for constrained generation conditioned on the prefix, as described in Autoregressive Entity Retrieval. TYPE: `Callable[[int, mindspore.Tensor], List[int]]`, *optional* DEFAULT: None
synced_gpus	Whether to continue running the while loop until max_length (needed for ZeRO stage 3) TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
return_timestamps	Whether to return the timestamps with the text. This enables the WhisperTimestampsLogitsProcessor. TYPE: `bool`, *optional* DEFAULT: None
task	Task to use for generation, either "translate" or "transcribe". The model.config.forced_decoder_ids will be updated accordingly. TYPE: `str`, *optional* DEFAULT: None
language	Language token to use for generation, can be either in the form of <|en|>, en or english. You can find all the possible language tokens in the model.generation_config.lang_to_id dictionary. TYPE: `str`, *optional* DEFAULT: None
is_multilingual	Whether or not the model is multilingual. TYPE: `bool`, *optional* DEFAULT: None
prompt_ids	Rank-1 tensor of token IDs created by passing text to [~WhisperProcessor.get_prompt_ids] that is provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words correctly. It cannot be used in conjunction with decoder_start_token_id as it overwrites this value. TYPE: `mindspore.Tensor`, *optional* DEFAULT: None
return_token_timestamps	Whether to return token-level timestamps with the text. This can be used with or without the return_timestamps option. To get word-level timestamps, use the tokenizer to group the tokens into words. TYPE: `bool`, *optional* DEFAULT: None
kwargs	Ad hoc parametrization of generate_config and/or additional model-specific kwargs that will be forwarded to the forward function of the model. If the model is an encoder-decoder model, encoder specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with decoder_. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

RETURNS

DESCRIPTION

[~utils.ModelOutput] or mindspore.Tensor: A [~utils.ModelOutput] (if return_dict_in_generate=True or when config.return_dict_in_generate=True) or a mindspore.Tensor. If the model is not an encoder-decoder model (model.config.is_encoder_decoder=False), the possible [~utils.ModelOutput] types are: [~generation.GreedySearchDecoderOnlyOutput], [~generation.SampleDecoderOnlyOutput], [~generation.BeamSearchDecoderOnlyOutput], [~generation.BeamSampleDecoderOnlyOutput] If the model is an encoder-decoder model (model.config.is_encoder_decoder=True), the possible [~utils.ModelOutput] types are: [~generation.GreedySearchEncoderDecoderOutput], [~generation.SampleEncoderDecoderOutput], [~generation.BeamSearchEncoderDecoderOutput], [~generation.BeamSampleEncoderDecoderOutput]

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def generate(
    self,
    inputs: Optional[mindspore.Tensor] = None,
    generation_config=None,
    logits_processor=None,
    stopping_criteria=None,
    prefix_allowed_tokens_fn=None,
    synced_gpus=False,
    return_timestamps=None,
    task=None,
    language=None,
    is_multilingual=None,
    prompt_ids: Optional[mindspore.Tensor] = None,
    return_token_timestamps=None,
    **kwargs,
):
    """

    Generates sequences of token ids for models with a language modeling head.

    <Tip warning={true}>

    Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
    model's default generation configuration. You can override any `generation_config` by passing the corresponding
    parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Parameters:
        inputs (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
            The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
            method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
            should of in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of
            `input_ids`, `input_values`, `input_features`, or `pixel_values`.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call. `**kwargs`
            passed to generate matching the attributes of `generation_config` will override them. If
            `generation_config` is not provided, the default will be used, which had the following loading
            priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
            configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
            default values, whose documentation should be checked to parameterize generation.
        logits_processor (`LogitsProcessorList`, *optional*):
            Custom logits processors that complement the default logits processors built from arguments and
            generation config. If a logit processor is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        stopping_criteria (`StoppingCriteriaList`, *optional*):
            Custom stopping criteria that complement the default stopping criteria built from arguments and a
            generation config. If a stopping criteria is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
            If provided, this function constraints the beam search to allowed tokens only at each step. If not
            provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
            `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
            on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
            for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
            Retrieval](https://arxiv.org/abs/2010.00904).
        synced_gpus (`bool`, *optional*, defaults to `False`):
            Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
        return_timestamps (`bool`, *optional*):
            Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`.
        task (`str`, *optional*):
            Task to use for generation, either "translate" or "transcribe". The `model.config.forced_decoder_ids`
            will be updated accordingly.
        language (`str`, *optional*):
            Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. You can
            find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary.
        is_multilingual (`bool`, *optional*):
            Whether or not the model is multilingual.
        prompt_ids (`mindspore.Tensor`, *optional*):
            Rank-1 tensor of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is
            provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for
            transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words
            correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value.
        return_token_timestamps (`bool`, *optional*):
            Whether to return token-level timestamps with the text. This can be used with or without the
            `return_timestamps` option. To get word-level timestamps, use the tokenizer to group the tokens into
            words.
        kwargs (`Dict[str, Any]`, *optional*):
            Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
            forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
            specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.

    Returns:
        [`~utils.ModelOutput`] or `mindspore.Tensor`:
            A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` or when
            `config.return_dict_in_generate=True`) or a `mindspore.Tensor`.
            If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
            [`~utils.ModelOutput`] types are:

            - [`~generation.GreedySearchDecoderOnlyOutput`],
            - [`~generation.SampleDecoderOnlyOutput`],
            - [`~generation.BeamSearchDecoderOnlyOutput`],
            - [`~generation.BeamSampleDecoderOnlyOutput`]

            If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
            [`~utils.ModelOutput`] types are:

            - [`~generation.GreedySearchEncoderDecoderOutput`],
            - [`~generation.SampleEncoderDecoderOutput`],
            - [`~generation.BeamSearchEncoderDecoderOutput`],
            - [`~generation.BeamSampleEncoderDecoderOutput`]
    """
    if generation_config is None:
        generation_config = self.generation_config

    if return_timestamps is not None:
        if not hasattr(generation_config, "no_timestamps_token_id"):
            raise ValueError(
                "You are trying to return timestamps, but the generation config is not properly set. "
                "Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`. "
                "For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363"
            )

        generation_config.return_timestamps = return_timestamps
    else:
        generation_config.return_timestamps = False

    if is_multilingual is not None:
        if not hasattr(generation_config, "is_multilingual"):
            raise ValueError(
                "The generation config is outdated and is thus not compatible with the `is_multilingual` argument "
                "to `generate`. Please update the generation config as per the instructions "
                "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
            )
        generation_config.is_multilingual = is_multilingual

    if hasattr(generation_config, "is_multilingual") and not generation_config.is_multilingual:
        if task is not None or language is not None:
            raise ValueError(
                "Cannot specify `task` or `language` for an English-only model. If the model is intended to be "
                "multilingual, pass `is_multilingual=True` to generate, or update the generation config."
            )

    if language is not None:
        if not hasattr(generation_config, "lang_to_id"):
            raise ValueError(
                "The generation config is outdated and is thus not compatible with the `language` argument "
                "to `generate`. Either set the language using the `forced_decoder_ids` in the model config, "
                "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
            )
        language = language.lower()
        generation_config.language = language
    if task is not None:
        if not hasattr(generation_config, "task_to_id"):
            raise ValueError(
                "The generation config is outdated and is thus not compatible with the `task` argument "
                "to `generate`. Either set the task using the `forced_decoder_ids` in the model config, "
                "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
            )
        generation_config.task = task

    forced_decoder_ids = None

    # Legacy code for backward compatibility
    if hasattr(self.config, "forced_decoder_ids") and self.config.forced_decoder_ids is not None:
        forced_decoder_ids = self.config.forced_decoder_ids
    elif (
        hasattr(self.generation_config, "forced_decoder_ids")
        and self.generation_config.forced_decoder_ids is not None
    ):
        forced_decoder_ids = self.generation_config.forced_decoder_ids
    else:
        forced_decoder_ids = kwargs.get("forced_decoder_ids", None)

    if task is not None or language is not None or (forced_decoder_ids is None and prompt_ids is not None):
        forced_decoder_ids = []
        if hasattr(generation_config, "language"):
            if generation_config.language in generation_config.lang_to_id.keys():
                language_token = generation_config.language
            elif generation_config.language in TO_LANGUAGE_CODE.keys():
                language_token = f"<|{TO_LANGUAGE_CODE[generation_config.language]}|>"
            elif generation_config.language in TO_LANGUAGE_CODE.values():
                language_token = f"<|{generation_config.language}|>"
            else:
                is_language_code = len(generation_config.language) == 2
                raise ValueError(
                    f"Unsupported language: {generation_config.language}. Language should be one of:"
                    f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}."
                )
            forced_decoder_ids.append((1, generation_config.lang_to_id[language_token]))
        else:
            forced_decoder_ids.append((1, None))  # automatically detect the language

        if hasattr(generation_config, "task"):
            if generation_config.task in TASK_IDS:
                forced_decoder_ids.append((2, generation_config.task_to_id[generation_config.task]))
            else:
                raise ValueError(
                    f"The `{generation_config.task}`task is not supported. The task should be one of `{TASK_IDS}`"
                )
        elif hasattr(generation_config, "task_to_id"):
            forced_decoder_ids.append((2, generation_config.task_to_id["transcribe"]))  # defaults to transcribe
        if hasattr(generation_config, "no_timestamps_token_id") and not generation_config.return_timestamps:
            idx = forced_decoder_ids[-1][0] + 1 if forced_decoder_ids else 1
            forced_decoder_ids.append((idx, generation_config.no_timestamps_token_id))

    if forced_decoder_ids is not None:
        generation_config.forced_decoder_ids = forced_decoder_ids

    if prompt_ids is not None:
        if kwargs.get("decoder_start_token_id") is not None:
            raise ValueError(
                "When specifying `prompt_ids`, you cannot also specify `decoder_start_token_id` as it gets overwritten."
            )
        prompt_ids = prompt_ids.tolist()
        decoder_start_token_id, *text_prompt_ids = prompt_ids
        # Slicing the text prompt ids in a manner consistent with the OpenAI implementation
        # to accomodate context space for the prefix (see https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/decoding.py#L599)
        text_prompt_ids = text_prompt_ids[-self.config.max_target_positions // 2 - 1 :]
        # Set the decoder_start_token_id to <|startofprev|>
        kwargs.update({"decoder_start_token_id": decoder_start_token_id})

        # If the user passes `max_new_tokens`, increase its number to account for the prompt
        if kwargs.get("max_new_tokens", None) is not None:
            kwargs["max_new_tokens"] += len(text_prompt_ids)
            if kwargs["max_new_tokens"] >= self.config.max_target_positions:
                raise ValueError(
                    f"The length of the sliced `prompt_ids` is {len(text_prompt_ids)}, and the `max_new_tokens` "
                    f"{kwargs['max_new_tokens'] - len(text_prompt_ids)}. Thus, the combined length of the sliced "
                    f"`prompt_ids` and `max_new_tokens` is: {kwargs['max_new_tokens']}. This exceeds the "
                    f"`max_target_positions` of the Whisper model: {self.config.max_target_positions}. "
                    "You should either reduce the length of your prompt, or reduce the value of `max_new_tokens`, "
                    f"so that their combined length is less that {self.config.max_target_positions}."
                )

        # Reformat the forced_decoder_ids to incorporate the prompt
        non_prompt_forced_decoder_ids = (
            kwargs.pop("forced_decoder_ids", None) or generation_config.forced_decoder_ids
        )
        forced_decoder_ids = [
            *text_prompt_ids,
            generation_config.decoder_start_token_id,
            *[token for _rank, token in non_prompt_forced_decoder_ids],
        ]
        forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_decoder_ids)]
        generation_config.forced_decoder_ids = forced_decoder_ids

    if generation_config.return_timestamps:
        logits_processor = [WhisperTimeStampLogitsProcessor(generation_config)]

    if return_token_timestamps:
        kwargs["output_attentions"] = True
        kwargs["return_dict_in_generate"] = True

        if getattr(generation_config, "task", None) == "translate":
            logger.warning("Token-level timestamps may not be reliable for task 'translate'.")
        if not hasattr(generation_config, "alignment_heads"):
            raise ValueError(
                "Model generation config has no `alignment_heads`, token-level timestamps not available. "
                "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config."
            )

        if kwargs.get("num_frames") is not None:
            generation_config.num_frames = kwargs.pop("num_frames")

    outputs = super().generate(
        inputs,
        generation_config,
        logits_processor,
        stopping_criteria,
        prefix_allowed_tokens_fn,
        synced_gpus,
        **kwargs,
    )

    if return_token_timestamps and hasattr(generation_config, "alignment_heads"):
        num_frames = getattr(generation_config, "num_frames", None)
        outputs["token_timestamps"] = self._extract_token_timestamps(
            outputs, generation_config.alignment_heads, num_frames=num_frames
        )

    return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.get_decoder()

This method 'get_decoder' is part of the class 'WhisperForConditionalGeneration' and retrieves the decoder from the model.

PARAMETER	DESCRIPTION
self	Instance of the 'WhisperForConditionalGeneration' class. Type: object Purpose: Represents the current instance of the class. Restrictions: This parameter is required for accessing the decoder.

RETURNS	DESCRIPTION
None	Type: None Purpose: The method returns None as it retrieves the decoder from the model.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_decoder(self):
    """
    This method 'get_decoder' is part of the class 'WhisperForConditionalGeneration' and retrieves 
    the decoder from the model.

    Args:
        self:
            Instance of the 'WhisperForConditionalGeneration' class.

            - Type: object
            - Purpose: Represents the current instance of the class.
            - Restrictions: This parameter is required for accessing the decoder.

    Returns:
        None:

            - Type: None
            - Purpose: The method returns None as it retrieves the decoder from the model.

    Raises:
        None.
    """
    return self.model.get_decoder()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.get_encoder()

Retrieves the encoder from the model instance.

PARAMETER	DESCRIPTION
self	The object instance. TYPE: WhisperForConditionalGeneration

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_encoder(self):
    """
    Retrieves the encoder from the model instance.

    Args:
        self (WhisperForConditionalGeneration): The object instance.

    Returns:
        None.

    Raises:
        None.

    """
    return self.model.get_encoder()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.get_input_embeddings()

Returns the input embeddings for the WhisperForConditionalGeneration model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForConditionalGeneration class. TYPE: WhisperForConditionalGeneration

RETURNS	DESCRIPTION
Cell	nn.Cell: The input embeddings for the model.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Cell:
    """
    Returns the input embeddings for the WhisperForConditionalGeneration model.

    Args:
        self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.

    Returns:
        nn.Cell: The input embeddings for the model.

    Raises:
        None.
    """
    return self.model.get_input_embeddings()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.get_output_embeddings()

Method to retrieve the output embeddings from the WhisperForConditionalGeneration class.

PARAMETER	DESCRIPTION
self	An instance of the WhisperForConditionalGeneration class. Type: WhisperForConditionalGeneration Purpose: Represents the current object of the class. Restrictions: Must be an instance of WhisperForConditionalGeneration class.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_output_embeddings(self):
    """
    Method to retrieve the output embeddings from the WhisperForConditionalGeneration class.

    Args:
        self:
            An instance of the WhisperForConditionalGeneration class.

            - Type: WhisperForConditionalGeneration
            - Purpose: Represents the current object of the class.
            - Restrictions: Must be an instance of WhisperForConditionalGeneration class.

    Returns:
        None.

    Raises:
        None.
    """
    return self.proj_out

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.prepare_inputs_for_generation(decoder_input_ids, past_key_values=None, use_cache=None, encoder_outputs=None, attention_mask=None, **kwargs)

Prepare inputs for generation.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForConditionalGeneration class. TYPE: WhisperForConditionalGeneration
decoder_input_ids	The input tensor for the decoder. Shape: (batch_size, sequence_length). TYPE: Tensor
past_key_values	The past key values for caching computations in auto-regressive decoding. Default: None. TYPE: tuple DEFAULT: None
use_cache	Whether to use caching for fast decoding. Default: None. TYPE: bool DEFAULT: None
encoder_outputs	The output of the encoder. Shape: (batch_size, sequence_length, hidden_size). Default: None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask for the decoder input. Shape: (batch_size, sequence_length). Default: None. TYPE: Tensor DEFAULT: None
**kwargs	Additional keyword arguments. DEFAULT: {}

RETURNS

DESCRIPTION

dict

A dictionary containing the prepared inputs for generation. It includes the following keys: 'encoder_outputs' (torch.Tensor): The output of the encoder. 'past_key_values' (tuple): The past key values for caching computations in auto-regressive decoding. 'decoder_input_ids' (torch.Tensor): The input tensor for the decoder. 'use_cache' (bool): Whether to use caching for fast decoding. 'decoder_attention_mask' (None): The attention mask for the decoder input.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def prepare_inputs_for_generation(
    self,
    decoder_input_ids,
    past_key_values=None,
    use_cache=None,
    encoder_outputs=None,
    attention_mask=None,
    **kwargs,
):
    """
    Prepare inputs for generation.

    Args:
        self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.
        decoder_input_ids (torch.Tensor): The input tensor for the decoder.
            Shape: (batch_size, sequence_length).
        past_key_values (tuple, optional): The past key values for caching computations in auto-regressive decoding.
            Default: None.
        use_cache (bool, optional): Whether to use caching for fast decoding.
            Default: None.
        encoder_outputs (torch.Tensor, optional): The output of the encoder.
            Shape: (batch_size, sequence_length, hidden_size).
            Default: None.
        attention_mask (torch.Tensor, optional): The attention mask for the decoder input.
            Shape: (batch_size, sequence_length).
            Default: None.
        **kwargs: Additional keyword arguments.

    Returns:
        dict: A dictionary containing the prepared inputs for generation.
            It includes the following keys:

            - 'encoder_outputs' (torch.Tensor): The output of the encoder.
            - 'past_key_values' (tuple): The past key values for caching computations in auto-regressive decoding.
            - 'decoder_input_ids' (torch.Tensor): The input tensor for the decoder.
            - 'use_cache' (bool): Whether to use caching for fast decoding.
            - 'decoder_attention_mask' (None): The attention mask for the decoder input.

    Raises:
        None.
    """
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if decoder_input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = decoder_input_ids.shape[1] - 1

        decoder_input_ids = decoder_input_ids[:, remove_prefix_length:]

    return {
        "encoder_outputs": encoder_outputs,
        "past_key_values": past_key_values,
        "decoder_input_ids": decoder_input_ids,
        "use_cache": use_cache,
        "decoder_attention_mask": None,
    }

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.set_output_embeddings(new_embeddings)

This method sets the output embeddings for the WhisperForConditionalGeneration class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForConditionalGeneration class. TYPE: WhisperForConditionalGeneration
new_embeddings	The new embeddings to be set as the output embeddings for the WhisperForConditionalGeneration class. It can be of any type. TYPE: any

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_output_embeddings(self, new_embeddings):
    """
    This method sets the output embeddings for the WhisperForConditionalGeneration class.

    Args:
        self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.
        new_embeddings (any): The new embeddings to be set as the output embeddings for the 
            WhisperForConditionalGeneration class. It can be of any type.

    Returns:
        None.

    Raises:
        None.
    """
    self.proj_out = new_embeddings

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel

Bases: WhisperPreTrainedModel

WhisperModel Represents a Whisper model for sequence-to-sequence tasks.

This class inherits from WhisperPreTrainedModel and provides methods for initializing the model, accessing input embeddings, accessing the encoder and decoder, freezing the encoder, masking input features, and constructing the model with various input parameters.

Example

>>> from transformers import AutoFeatureExtractor, WhisperModel
>>> from datasets import load_dataset
...
>>> model = WhisperModel.from_pretrained("openai/whisper-base")
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base")
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_features = inputs.input_features
>>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
>>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
>>> list(last_hidden_state.shape)
[1, 2, 512]

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperModel(WhisperPreTrainedModel):

    """
    WhisperModel
    Represents a Whisper model for sequence-to-sequence tasks.

    This class inherits from WhisperPreTrainedModel and provides methods for initializing the model,
    accessing input embeddings, accessing the encoder and decoder, freezing the encoder, masking input features,
    and constructing the model with various input parameters.

    Example:
        ```python
        >>> from transformers import AutoFeatureExtractor, WhisperModel
        >>> from datasets import load_dataset
        ...
        >>> model = WhisperModel.from_pretrained("openai/whisper-base")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_features = inputs.input_features
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 512]
        ```
    """
    def __init__(self, config: WhisperConfig):
        """
        Initializes an instance of the WhisperModel class.

        Args:
            self: The instance of the class.
            config (WhisperConfig): The configuration object used for initialization.
                This object contains various settings and parameters required for the model.
                It should be an instance of the WhisperConfig class.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.encoder = WhisperEncoder(config)
        self.decoder = WhisperDecoder(config)
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        This method returns the input embeddings for the WhisperModel.

        Args:
            self (WhisperModel): The instance of the WhisperModel class.

        Returns:
            None: This method returns the input embeddings for the WhisperModel.

        Raises:
            None
        """
        return self.decoder.embed_tokens

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperModel.

        Args:
            self (WhisperModel): The instance of the WhisperModel class.
            value (object): The input embeddings to be set for the decoder embed_tokens.

        Returns:
            None.

        Raises:
            None.
        """
        self.decoder.embed_tokens = value

    def get_encoder(self):
        """
        This method returns the encoder associated with the WhisperModel.

        Args:
            self (WhisperModel): The instance of the WhisperModel class.

        Returns:
            encoder: This method returns the encoder associated with the WhisperModel.

        Raises:
            None
        """
        return self.encoder

    def get_decoder(self):
        """
        Retrieve the decoder used in the WhisperModel.

        Args:
            self (WhisperModel): The instance of the class.

        Returns:
            decoder: This method does not return any value.

        Raises:
            None.
        """
        return self.decoder

    def freeze_encoder(self):
        """
        Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
        not be updated during training.
        """
        self.encoder._freeze_parameters()

    def _mask_input_features(
        self,
        input_features: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
    ):
        """
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://arxiv.org/abs/1904.08779).
        """
        # `config.apply_spec_augment` can set masking to False
        if not getattr(self.config, "apply_spec_augment", True):
            return input_features

        # generate indices & apply SpecAugment along time axis
        batch_size, hidden_size, sequence_length = input_features.shape

        if self.config.mask_time_prob > 0 and self.training:
            # generate indices & apply SpecAugment along time axis
            mask_time_indices = _compute_mask_indices(
                (batch_size, sequence_length),
                mask_prob=self.config.mask_time_prob,
                mask_length=self.config.mask_time_length,
                attention_mask=attention_mask,
                min_masks=self.config.mask_time_min_masks,
            )
            mask_time_indices = mindspore.tensor(mask_time_indices, dtype=mindspore.bool_)
            mask_time_indices = mask_time_indices[:, None].broadcast_to((-1, hidden_size, -1))
            input_features[mask_time_indices] = 0

        if self.config.mask_feature_prob > 0 and self.training:
            # generate indices & apply SpecAugment along feature axis
            mask_feature_indices = _compute_mask_indices(
                (batch_size, hidden_size),
                mask_prob=self.config.mask_feature_prob,
                mask_length=self.config.mask_feature_length,
                min_masks=self.config.mask_feature_min_masks,
            )
            mask_feature_indices = mindspore.tensor(mask_feature_indices, dtype=mindspore.bool_)
            input_features[mask_feature_indices] = 0

        return input_features

    def construct(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        decoder_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        decoder_inputs_embeds: Optional[Tuple[mindspore.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], Seq2SeqModelOutput]:
        r"""

        Returns:
            `Union[Tuple[mindspore.Tensor], Seq2SeqModelOutput]`

        Example:
            ```python
            >>> from transformers import AutoFeatureExtractor, WhisperModel
            >>> from datasets import load_dataset
            ...
            >>> model = WhisperModel.from_pretrained("openai/whisper-base")
            >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base")
            >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
            >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
            >>> input_features = inputs.input_features
            >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
            >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
            >>> list(last_hidden_state.shape)
            [1, 2, 512]
            ```
         """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            input_features = self._mask_input_features(input_features, attention_mask=attention_mask)

            encoder_outputs = self.encoder(
                input_features,
                head_mask=head_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )
        # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return Seq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.__init__(config)

Initializes an instance of the WhisperModel class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object used for initialization. This object contains various settings and parameters required for the model. It should be an instance of the WhisperConfig class. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes an instance of the WhisperModel class.

    Args:
        self: The instance of the class.
        config (WhisperConfig): The configuration object used for initialization.
            This object contains various settings and parameters required for the model.
            It should be an instance of the WhisperConfig class.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.encoder = WhisperEncoder(config)
    self.decoder = WhisperDecoder(config)
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.construct(input_features=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, past_key_values=None, decoder_inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

RETURNS	DESCRIPTION
Union[Tuple[Tensor], Seq2SeqModelOutput]	Union[Tuple[mindspore.Tensor], Seq2SeqModelOutput]

Example

>>> from transformers import AutoFeatureExtractor, WhisperModel
>>> from datasets import load_dataset
...
>>> model = WhisperModel.from_pretrained("openai/whisper-base")
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base")
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_features = inputs.input_features
>>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
>>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
>>> list(last_hidden_state.shape)
[1, 2, 512]

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    decoder_input_ids: Optional[mindspore.Tensor] = None,
    decoder_attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    decoder_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    decoder_inputs_embeds: Optional[Tuple[mindspore.Tensor]] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], Seq2SeqModelOutput]:
    r"""

    Returns:
        `Union[Tuple[mindspore.Tensor], Seq2SeqModelOutput]`

    Example:
        ```python
        >>> from transformers import AutoFeatureExtractor, WhisperModel
        >>> from datasets import load_dataset
        ...
        >>> model = WhisperModel.from_pretrained("openai/whisper-base")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_features = inputs.input_features
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 512]
        ```
     """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if encoder_outputs is None:
        input_features = self._mask_input_features(input_features, attention_mask=attention_mask)

        encoder_outputs = self.encoder(
            input_features,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
    # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True
    elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
        encoder_outputs = BaseModelOutput(
            last_hidden_state=encoder_outputs[0],
            hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
            attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
        )
    # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)
    decoder_outputs = self.decoder(
        input_ids=decoder_input_ids,
        attention_mask=decoder_attention_mask,
        encoder_hidden_states=encoder_outputs[0],
        head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        past_key_values=past_key_values,
        inputs_embeds=decoder_inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    if not return_dict:
        return decoder_outputs + encoder_outputs

    return Seq2SeqModelOutput(
        last_hidden_state=decoder_outputs.last_hidden_state,
        past_key_values=decoder_outputs.past_key_values,
        decoder_hidden_states=decoder_outputs.hidden_states,
        decoder_attentions=decoder_outputs.attentions,
        cross_attentions=decoder_outputs.cross_attentions,
        encoder_last_hidden_state=encoder_outputs.last_hidden_state,
        encoder_hidden_states=encoder_outputs.hidden_states,
        encoder_attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.freeze_encoder()

Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will not be updated during training.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def freeze_encoder(self):
    """
    Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
    not be updated during training.
    """
    self.encoder._freeze_parameters()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.get_decoder()

Retrieve the decoder used in the WhisperModel.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: WhisperModel

RETURNS	DESCRIPTION
decoder	This method does not return any value.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_decoder(self):
    """
    Retrieve the decoder used in the WhisperModel.

    Args:
        self (WhisperModel): The instance of the class.

    Returns:
        decoder: This method does not return any value.

    Raises:
        None.
    """
    return self.decoder

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.get_encoder()

This method returns the encoder associated with the WhisperModel.

PARAMETER	DESCRIPTION
self	The instance of the WhisperModel class. TYPE: WhisperModel

RETURNS	DESCRIPTION
encoder	This method returns the encoder associated with the WhisperModel.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_encoder(self):
    """
    This method returns the encoder associated with the WhisperModel.

    Args:
        self (WhisperModel): The instance of the WhisperModel class.

    Returns:
        encoder: This method returns the encoder associated with the WhisperModel.

    Raises:
        None
    """
    return self.encoder

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.get_input_embeddings()

This method returns the input embeddings for the WhisperModel.

PARAMETER	DESCRIPTION
self	The instance of the WhisperModel class. TYPE: WhisperModel

RETURNS	DESCRIPTION
None	This method returns the input embeddings for the WhisperModel.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self):
    """
    This method returns the input embeddings for the WhisperModel.

    Args:
        self (WhisperModel): The instance of the WhisperModel class.

    Returns:
        None: This method returns the input embeddings for the WhisperModel.

    Raises:
        None
    """
    return self.decoder.embed_tokens

mindnlp.transformers.models.whisper.modeling_whisper.WhisperModel.set_input_embeddings(value)

Sets the input embeddings for the WhisperModel.

PARAMETER	DESCRIPTION
self	The instance of the WhisperModel class. TYPE: WhisperModel
value	The input embeddings to be set for the decoder embed_tokens. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperModel.

    Args:
        self (WhisperModel): The instance of the WhisperModel class.
        value (object): The input embeddings to be set for the decoder embed_tokens.

    Returns:
        None.

    Raises:
        None.
    """
    self.decoder.embed_tokens = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperPositionalEmbedding

Bases: Embedding

Represents a Positional Embedding layer tailored for Whisper models.

This class provides a custom implementation of positional embedding for Whisper models, inheriting from nn.Embedding. It allows for flexible initialization with the specified number of positions and embedding dimensions, with optional padding index support.

ATTRIBUTE	DESCRIPTION
num_positions	The total number of positions to be embedded. TYPE: int
embedding_dim	The dimensionality of the embedding vectors. TYPE: int
padding_idx	The index used for padding, if specified. TYPE: Optional[int]

METHOD	DESCRIPTION
__init__	Initializes the WhisperPositionalEmbedding instance with the given parameters.
construct	Constructs the positional embeddings for the input_ids, considering past key values length when applicable.

RETURNS	DESCRIPTION
	The positional embeddings corresponding to the input_ids with respect to the past key values length.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperPositionalEmbedding(nn.Embedding):

    """
    Represents a Positional Embedding layer tailored for Whisper models.

    This class provides a custom implementation of positional embedding for Whisper models, inheriting from nn.Embedding.
    It allows for flexible initialization with the specified number of positions and embedding dimensions, with optional
    padding index support.

    Attributes:
        num_positions (int): The total number of positions to be embedded.
        embedding_dim (int): The dimensionality of the embedding vectors.
        padding_idx (Optional[int]): The index used for padding, if specified.

    Methods:
        __init__:
            Initializes the WhisperPositionalEmbedding instance with the given parameters.

        construct:
            Constructs the positional embeddings for the input_ids, considering past key values length when applicable.

    Returns:
        The positional embeddings corresponding to the input_ids with respect to the past key values length.

    """
    def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None):
        """
        Initializes a WhisperPositionalEmbedding instance.

        Args:
            self: The instance of the class.
            num_positions (int): The number of positions to be embedded.
            embedding_dim (int): The dimension of the embedding.
            padding_idx (Optional[int]): The index used for padding sequences. Defaults to None.

        Returns:
            None.

        Raises:
            TypeError: If num_positions or embedding_dim are not integers.
            ValueError: If num_positions or embedding_dim are less than or equal to 0.
        """
        super().__init__(num_positions, embedding_dim)

    def construct(self, input_ids, past_key_values_length=0):
        """
        Constructs the positional embeddings for the input_ids.

        Args:
            self (WhisperPositionalEmbedding): The instance of the WhisperPositionalEmbedding class.

            input_ids (torch.Tensor): The input tensor containing the token ids. It is expected to have a shape of
                (batch_size, sequence_length).

            past_key_values_length (int, optional): The length of the past key values. Defaults to 0. This parameter is
                used to slice the positional embeddings based on the past key values length.

        Returns:
            None.

        Raises:
            None.
        """
        return self.weight[past_key_values_length : past_key_values_length + input_ids.shape[1]]

mindnlp.transformers.models.whisper.modeling_whisper.WhisperPositionalEmbedding.__init__(num_positions, embedding_dim, padding_idx=None)

Initializes a WhisperPositionalEmbedding instance.

PARAMETER	DESCRIPTION
self	The instance of the class.
num_positions	The number of positions to be embedded. TYPE: int
embedding_dim	The dimension of the embedding. TYPE: int
padding_idx	The index used for padding sequences. Defaults to None. TYPE: Optional[int] DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If num_positions or embedding_dim are not integers.
ValueError	If num_positions or embedding_dim are less than or equal to 0.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None):
    """
    Initializes a WhisperPositionalEmbedding instance.

    Args:
        self: The instance of the class.
        num_positions (int): The number of positions to be embedded.
        embedding_dim (int): The dimension of the embedding.
        padding_idx (Optional[int]): The index used for padding sequences. Defaults to None.

    Returns:
        None.

    Raises:
        TypeError: If num_positions or embedding_dim are not integers.
        ValueError: If num_positions or embedding_dim are less than or equal to 0.
    """
    super().__init__(num_positions, embedding_dim)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperPositionalEmbedding.construct(input_ids, past_key_values_length=0)

Constructs the positional embeddings for the input_ids.

PARAMETER	DESCRIPTION
self	The instance of the WhisperPositionalEmbedding class. TYPE: WhisperPositionalEmbedding
input_ids	The input tensor containing the token ids. It is expected to have a shape of (batch_size, sequence_length). TYPE: Tensor
past_key_values_length	The length of the past key values. Defaults to 0. This parameter is used to slice the positional embeddings based on the past key values length. TYPE: int DEFAULT: 0

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def construct(self, input_ids, past_key_values_length=0):
    """
    Constructs the positional embeddings for the input_ids.

    Args:
        self (WhisperPositionalEmbedding): The instance of the WhisperPositionalEmbedding class.

        input_ids (torch.Tensor): The input tensor containing the token ids. It is expected to have a shape of
            (batch_size, sequence_length).

        past_key_values_length (int, optional): The length of the past key values. Defaults to 0. This parameter is
            used to slice the positional embeddings based on the past key values length.

    Returns:
        None.

    Raises:
        None.
    """
    return self.weight[past_key_values_length : past_key_values_length + input_ids.shape[1]]

mindnlp.transformers.models.whisper.modeling_whisper.WhisperPreTrainedModel

Bases: PreTrainedModel

This class represents a pre-trained model for the Whisper framework. It inherits from the PreTrainedModel class, providing additional functionality and customization specific to Whisper models.

The class contains methods for initializing weights and for computing the output length of convolutional layers. The _init_weights method initializes the weights of various types of neural network cells, including dense, convolutional, and embedding layers, as well as custom WhisperEncoder cells. The _get_feat_extract_output_lengths method computes the output length of convolutional layers based on the input lengths provided.

Overall, the WhisperPreTrainedModel class serves as a foundational framework for creating and customizing pre-trained models within the Whisper environment, offering flexibility in weight initialization and feature extraction output length computations.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperPreTrainedModel(PreTrainedModel):

    """
    This class represents a pre-trained model for the Whisper framework. It inherits from the PreTrainedModel class,
    providing additional functionality and customization specific to Whisper models.

    The class contains methods for initializing weights and for computing the output length of convolutional layers.
    The _init_weights method initializes the weights of various types of neural network cells, including dense,
    convolutional, and embedding layers, as well as custom WhisperEncoder cells. The _get_feat_extract_output_lengths
    method computes the output length of convolutional layers based on the input lengths provided.

    Overall, the WhisperPreTrainedModel class serves as a foundational framework for creating and customizing
    pre-trained models within the Whisper environment, offering flexibility in weight initialization and feature
    extraction output length computations.
    """
    config_class = WhisperConfig
    base_model_prefix = "model"
    main_input_name = "input_features"
    supports_gradient_checkpointing = True
    _no_split_modules = ["WhisperEncoderLayer", "WhisperDecoderLayer"]
    _supports_flash_attn_2 = True

    def _init_weights(self, cell):
        """
        Initializes weights for the specified cell based on the configuration settings of the WhisperPreTrainedModel.

        Args:
            self (WhisperPreTrainedModel): The instance of the WhisperPreTrainedModel class.
            cell (nn.Module): The neural network cell for which weights are to be initialized.
                It can be an instance of nn.Dense, nn.Conv1d, nn.Embedding, or WhisperEncoder.

        Returns:
            None.

        Raises:
            TypeError: If the cell parameter is not an instance of nn.Module.
            ValueError: If the cell parameter is not one of the supported types
                (nn.Dense, nn.Conv1d, nn.Embedding, or WhisperEncoder).
            ValueError: If the cell type is nn.Embedding and the padding index is not provided.
            ValueError: If the cell type is WhisperEncoder and the embed_positions weight shape is not
                compatible with the sinusoids function output.
        """
        std = self.config.init_std
        if isinstance(cell, (nn.Dense, nn.Conv1d)):
            cell.weight.set_data(initializer(Normal(std),
                                                    cell.weight.shape, cell.weight.dtype))
            if cell.has_bias:
                cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))
        elif isinstance(cell, nn.Embedding):
            weight = np.random.normal(0.0, std, cell.weight.shape)
            if cell.padding_idx:
                weight[cell.padding_idx] = 0

            cell.weight.set_data(Tensor(weight, cell.weight.dtype))
        elif isinstance(cell, WhisperEncoder):
            embed_positions = cell.embed_positions.weight
            embed_positions.set_data(sinusoids(*embed_positions.shape))

    def _get_feat_extract_output_lengths(self, input_lengths: mindspore.Tensor):
        """
        Computes the output length of the convolutional layers
        """
        input_lengths = (input_lengths - 1) // 2 + 1

        return input_lengths

mindnlp.transformers.models.whisper.modeling_whisper.shift_tokens_right(input_ids, pad_token_id, decoder_start_token_id)

Shift input ids one token to the right.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def shift_tokens_right(input_ids: mindspore.Tensor, pad_token_id: int, decoder_start_token_id: int):
    """
    Shift input ids one token to the right.
    """
    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    # replace possible -100 values in labels by `pad_token_id`
    shifted_input_ids = shifted_input_ids.masked_fill(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids

mindnlp.transformers.models.whisper.modeling_whisper.sinusoids(length, channels, max_timescale=10000)

Returns sinusoids for positional embedding

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def sinusoids(length: int, channels: int, max_timescale: float = 10000) -> mindspore.Tensor:
    """Returns sinusoids for positional embedding"""
    if channels % 2 != 0:
        raise ValueError(
            f"Number of channels has to be divisible by 2 for sinusoidal positional embeddings, got {channels} channels."
        )
    log_timescale_increment = math.log(max_timescale) / (channels // 2 - 1)
    inv_timescales = ops.exp(-log_timescale_increment * ops.arange(channels // 2))
    scaled_time = ops.arange(length).view(-1, 1) * inv_timescales.view(1, -1)
    return ops.cat([scaled_time.sin(), scaled_time.cos()], axis=1)

mindnlp.transformers.models.whisper.tokenization_whisper

Tokenization classes for Whisper.

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer

Bases: PreTrainedTokenizer

Construct a Whisper tokenizer.

This tokenizer inherits from [PreTrainedTokenizer] which contains some of the main methods. Users should refer to the superclass for more information regarding such methods.

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`
merges_file	Path to the merges file. TYPE: `str`
normalizer_file	Path to the normalizer_file file. TYPE: `str`, *optional* DEFAULT: None
errors	Paradigm to follow when decoding bytes to UTF-8. See bytes.decode for more information. TYPE: `str`, *optional*, defaults to `"replace"` DEFAULT: 'replace'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
bos_token	The beginning of sequence token. The decoder_start_token_id is used to set the first token as "<|startoftranscript|>" when generating. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
eos_token	The end of sequence token. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional* DEFAULT: None
add_prefix_space	Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
language	The language of the transcription text. The corresponding language id token is appended to the start of the sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token "<|es|>" is appended to the start of sequence. This should be used for multilingual fine-tuning only. TYPE: `str`, *optional* DEFAULT: None
task	Task identifier to append at the start of sequence (if any). This should be used for mulitlingual fine-tuning, with "transcribe" for speech recognition and "translate" for speech translation. TYPE: `str`, *optional* DEFAULT: None
predict_timestamps	Whether to omit the <|notimestamps|> token at the start of the sequence. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

class WhisperTokenizer(PreTrainedTokenizer):
    """
    Construct a Whisper tokenizer.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
    the superclass for more information regarding such methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        normalizer_file (`str`, *optional*):
            Path to the normalizer_file file.
        errors (`str`, *optional*, defaults to `"replace"`):
            Paradigm to follow when decoding bytes to UTF-8. See
            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The beginning of sequence token. The `decoder_start_token_id` is used to set the first token as
            `"<|startoftranscript|>"` when generating.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        pad_token (`str`, *optional*):
            The token used for padding, for example when batching sequences of different lengths.
        add_prefix_space (`bool`, *optional*, defaults to `False`):
            Whether or not to add an initial space to the input. This allows to treat the leading word just as any
            other word.
        language (`str`, *optional*):
            The language of the transcription text. The corresponding language id token is appended to the start of the
            sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token
            `"<|es|>"` is appended to the start of sequence. This should be used for multilingual fine-tuning only.
        task (`str`, *optional*):
            Task identifier to append at the start of sequence (if any). This should be used for mulitlingual
            fine-tuning, with `"transcribe"` for speech recognition and `"translate"` for speech translation.
        predict_timestamps (`bool`, *optional*, defaults to `False`):
            Whether to omit the `<|notimestamps|>` token at the start of the sequence.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = MAX_MODEL_INPUT_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        merges_file,
        normalizer_file=None,
        errors="replace",
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        pad_token=None,
        add_prefix_space=False,
        language=None,
        task=None,
        predict_timestamps=False,
        **kwargs,
    ):
        """
        __init__

        This method initializes an instance of the WhisperTokenizer class.

        Args:
            self: The instance of the class.
            vocab_file (str): The path to the vocabulary file containing the token encoding.
            merges_file (str): The path to the file containing BPE merges for tokenization.
            normalizer_file (str, optional): The path to the file containing English spelling normalizer.
                Defaults to None.
            errors (str): The error handling scheme to use for encoding/decoding errors. Defaults to 'replace'.
            unk_token (str): The unknown token to be used during tokenization. Defaults to 'endoftext'.
            bos_token (str): The beginning of sentence token. Defaults to 'endoftext'.
            eos_token (str): The end of sentence token. Defaults to 'endoftext'.
            pad_token (str, optional): The padding token. Defaults to None.
            add_prefix_space (bool): Whether to add a prefix space during tokenization. Defaults to False.
            language (str, optional): The language of the text. Defaults to None.
            task (str, optional): The task for tokenization. Defaults to None.
            predict_timestamps (bool): Whether to predict timestamps. Defaults to False.

        Returns:
            None.

        Raises:
            FileNotFoundError: If the vocab_file, merges_file, or normalizer_file does not exist.
            ValueError: If the provided unk_token, bos_token, eos_token, or pad_token is not a string.
            TypeError: If the provided unk_token, bos_token, eos_token, or pad_token is not a string or an
                AddedToken instance.
            UnicodeDecodeError: If an error occurs during the decoding of vocab_file or merges_file.
            KeyError: If an error occurs during the creation of the bpe_ranks dictionary.
            re.error: If an error occurs during the compilation of regular expressions.
            json.JSONDecodeError: If an error occurs during the decoding of normalizer_file.
        """
        bos_token = (
            AddedToken(bos_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(bos_token, str)
            else bos_token
        )
        eos_token = (
            AddedToken(eos_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(eos_token, str)
            else eos_token
        )
        unk_token = (
            AddedToken(unk_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(unk_token, str)
            else unk_token
        )
        pad_token = (
            AddedToken(pad_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(pad_token, str)
            else pad_token
        )

        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.errors = errors  # how to handle errors in decoding
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            bpe_merges = merges_handle.read().split("\n")[1:-1]
        bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
        self.cache = {}
        self.add_prefix_space = add_prefix_space

        if normalizer_file is not None:
            with open(normalizer_file, encoding="utf-8") as vocab_handle:
                self.english_spelling_normalizer = json.load(vocab_handle)
        else:
            self.english_spelling_normalizer = None

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
        self.timestamp_pat = re.compile(r"<\|(\d+\.\d+)\|>")

        self.language = language
        super().__init__(
            errors=errors,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            pad_token=pad_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )

        self.task = task
        self.predict_timestamps = predict_timestamps

    @property
    def vocab_size(self) -> int:
        """
        This method returns the size of the vocabulary used by the WhisperTokenizer.

        Args:
            self (WhisperTokenizer): The instance of the WhisperTokenizer class.

        Returns:
            int: The size of the vocabulary used by the WhisperTokenizer.

        Raises:
            This method does not raise any exceptions.
        """
        return len(self.encoder)

    def get_vocab(self):
        """
        Returns the vocabulary of the WhisperTokenizer instance.

        Args:
            self (WhisperTokenizer): The instance of the WhisperTokenizer class.

        Returns:
            dict: A dictionary representing the vocabulary of the WhisperTokenizer instance.
                The keys of the dictionary are the tokens in the vocabulary, and the values are their respective
                token IDs.

        Raises:
            None.

        Note:
            This method builds the vocabulary by converting the token IDs to tokens using the `convert_ids_to_tokens`
            method of the WhisperTokenizer instance. The tokens and their corresponding IDs are stored in a dictionary.
            Additionally, any added tokens are also included in the vocabulary.

        Example:
            ```python
            >>> tokenizer = WhisperTokenizer()
            >>> vocab = tokenizer.get_vocab()
            >>> print(vocab)
            {'[PAD]': 0, '[UNK]': 1, '[CLS]': 2, '[SEP]': 3, '[MASK]': 4, 'hello': 5, 'world': 6}
            ```

        In the example above, the `get_vocab` method is called on a WhisperTokenizer instance, which returns a
        dictionary representing the vocabulary of the tokenizer. The vocabulary includes the default special tokens
        as well as any added tokens.
        """
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe with GPT2 -> Whisper
    def bpe(self, token):
        """
        This method, named 'bpe', is part of the class 'WhisperTokenizer' and performs a Byte Pair Encoding (BPE)
        algorithm on the given 'token' parameter.

        Args:
            self: An instance of the 'WhisperTokenizer' class.
            token (str): The token to be encoded using the BPE algorithm.

        Returns:
            str: The encoded token after applying the BPE algorithm.

        Raises:
            None.

        The BPE algorithm is applied to the 'token' by iteratively replacing the most frequent pairs of characters with
        a new character, until no more pairs can be found. The resulting encoded token is then returned.

        The method first checks if the 'token' is already present in the cache. If so, it returns the cached value.
        Otherwise, it proceeds with the BPE encoding process.

        The 'token' is converted into a tuple of characters named 'word'. The method then obtains all possible pairs
        of characters from 'word' using the 'get_pairs' function.

        If no pairs are found, the 'token' is already fully encoded and is returned as is.

        Otherwise, the method enters a loop, where it finds the most frequent pair of characters ('bigram') from the
        'pairs' list based on the 'bpe_ranks' dictionary. If the 'bigram' is not present in the 'bpe_ranks' dictionary,
        the loop is terminated.

        The method then replaces all occurrences of the 'bigram' in the 'word' with a new character. The 'word' is
        iterated through, and whenever the current character matches the first character of the 'bigram' and the next
        character matches the second character of the 'bigram', the new character is appended to the 'new_word' list.
        Otherwise, the current character is appended as is.

        The 'new_word' is converted back to a tuple and assigned to 'word'. If the length of 'word' becomes 1,
        indicating that the token is fully encoded, the loop is terminated.

        The 'pairs' are updated by obtaining all possible pairs from the updated 'word'.

        Finally, the 'word' is joined using spaces to form a string, which is then cached with the 'token' as
        the key in the 'cache' dictionary.

        The encoded 'word' is returned as the result of the method.
        """
        if token in self.cache:
            return self.cache[token]
        word = tuple(token)
        pairs = get_pairs(word)

        if not pairs:
            return token

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            pairs = get_pairs(word)
        word = " ".join(word)
        self.cache[token] = word
        return word

    def set_prefix_tokens(self, language: str = None, task: str = None, predict_timestamps: bool = None):
        """
        Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to
        update the prefix tokens as required when fine-tuning.

        Example:
            ```python
            >>> # instantiate the tokenizer and set the prefix token to Spanish
            >>> tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish")
            >>> # now switch the prefix token from Spanish to French
            >>> tokenizer.set_prefix_tokens(language="french")
            ```

        Args:
            language (`str`, *optional*, defaults to `None`):
                The language of the transcription text.
            task (`str`, *optional*, defaults to `None`):
                Task identifier to append at the start of sequence (if any).
            predict_timestamps (`bool`, *optional*, defaults to `None`):
                Whether to omit the `<|notimestamps|>` token at the start of the sequence.
        """
        self.language = language if language is not None else self.language
        self.task = task if task is not None else self.task
        self.predict_timestamps = predict_timestamps if predict_timestamps is not None else self.predict_timestamps

    @property
    def prefix_tokens(self) -> List[int]:
        """
        This method generates a list of integer tokens representing the prefix sequence for tokenization.

        Args:
            self (WhisperTokenizer): The instance of the WhisperTokenizer class.

        Returns:
            List[int]: A list of integer tokens representing the prefix sequence for tokenization.

        Raises:
            ValueError: If the provided language is unsupported or not found in the language code mapping.
            ValueError: If the provided task is unsupported or not found in the task IDs list.
        """
        bos_token_id = self.convert_tokens_to_ids("<|startoftranscript|>")
        translate_token_id = self.convert_tokens_to_ids("<|translate|>")
        transcribe_token_id = self.convert_tokens_to_ids("<|transcribe|>")
        notimestamps_token_id = self.convert_tokens_to_ids("<|notimestamps|>")
        langs = tuple(LANGUAGES.keys())

        if self.language is not None:
            self.language = self.language.lower()
            if self.language in TO_LANGUAGE_CODE:
                language_id = TO_LANGUAGE_CODE[self.language]
            elif self.language in TO_LANGUAGE_CODE.values():
                language_id = self.language
            else:
                is_language_code = len(self.language) == 2
                raise ValueError(
                    f"Unsupported language: {self.language}. Language should be one of:"
                    f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}."
                )

        if self.task is not None:
            if self.task not in TASK_IDS:
                raise ValueError(f"Unsupported task: {self.task}. Task should be in: {TASK_IDS}")

        bos_sequence = [bos_token_id]
        if self.language is not None:
            bos_sequence.append(bos_token_id + 1 + langs.index(language_id))
        if self.task is not None:
            bos_sequence.append(transcribe_token_id if self.task == "transcribe" else translate_token_id)
        if not self.predict_timestamps:
            bos_sequence.append(notimestamps_token_id)
        return bos_sequence

    # Copied from transformers.models.speech_to_text.tokenization_speech_to_text.Speech2TextTokenizer.build_inputs_with_special_tokens
    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
        """Build model inputs from a sequence by appending eos_token_id."""
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0 + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_0 + token_ids_1 + [self.eos_token_id]

    # Copied from transformers.models.speech_to_text.tokenization_speech_to_text.Speech2TextTokenizer.get_special_tokens_mask
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        prefix_ones = [1] * len(self.prefix_tokens)
        suffix_ones = [1]
        if token_ids_1 is None:
            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize with GPT2 -> Whisper
    def _tokenize(self, text):
        """Tokenize a string."""
        bpe_tokens = []
        for token in re.findall(self.pat, text):
            token = "".join(
                self.byte_encoder[b] for b in token.encode("utf-8")
            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
        return bpe_tokens

    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_token_to_id with GPT2 -> Whisper
    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """
        Converts an index (integer) in a token (str) using the vocab. Whisper's base tokenizer always decodes OOV
        tokens as "", thus we do not use the `unk_token` here.
        """
        return self.decoder.get(index, "")

    def _normalize(self, text):
        """
        Normalize a given string using the `EnglishTextNormalizer` class, which preforms commons transformation on
        english text.
        """
        normalizer = EnglishTextNormalizer(self.english_spelling_normalizer)
        return normalizer(text)

    @staticmethod
    def _basic_normalize(text, remove_diacritics=False):
        """
        Normalize a given string using the `BasicTextNormalizer` class, which preforms commons transformation on
        multilingual text.
        """
        normalizer = BasicTextNormalizer(remove_diacritics=remove_diacritics)
        return normalizer(text)

    def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_precision=0.02) -> str:
        """
        Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. This method decodes
        given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
        """
        timestamp_begin = self.all_special_ids[-1] + 1
        outputs = [[]]
        for token in token_ids:
            if token >= timestamp_begin:
                timestamp = f"<|{(token.asnumpy() - timestamp_begin) * time_precision:.2f}|>"
                outputs.append(timestamp)
                outputs.append([])
            else:
                outputs[-1].append(token)
        outputs = [
            s if isinstance(s, str) else self.decode(s, skip_special_tokens=skip_special_tokens) for s in outputs
        ]
        return "".join(outputs)

    def _compute_offsets(self, token_ids, time_precision=0.02):
        """
        Compute offsets for a given tokenized input

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            time_precision (`float`, `optional`, defaults to 0.02):
                The time ratio to convert from token to time.
        """
        offsets = []
        token_ids = np.array(token_ids)
        if token_ids.shape[0] > 1 and len(token_ids.shape) > 1:
            raise ValueError("Can only process a single input at a time")
        timestamp_begin = self.all_special_ids[-1] + 1
        timestamp_tokens = token_ids >= timestamp_begin

        consecutive = np.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0] + 1
        if consecutive.shape[0] == 0 and timestamp_tokens.sum() <= 1:
            # either there are no timestamps or there are no consecutive ones
            return []
        if np.where(timestamp_tokens)[0][-1] + 1 not in consecutive:
            # we add the final timestamp if it is not already in the list
            consecutive = np.append(consecutive, np.where(timestamp_tokens)[0][-1] + 1)

        last_slice = np.where(timestamp_tokens)[0][0]
        for current_slice in consecutive:
            sliced_tokens = token_ids[last_slice:current_slice]
            if len(sliced_tokens) > 1:
                start_timestamp_position = sliced_tokens[0].item() - timestamp_begin
                end_timestamp_position = sliced_tokens[-1].item() - timestamp_begin
                # strip timestamp tokens from the text output
                sliced_tokens = self._preprocess_token_ids(sliced_tokens)
                text = self._decode(sliced_tokens)
                text = self._filter_timestamp_ids(text)
                offsets.append(
                    {
                        "text": text,
                        "timestamp": (
                            start_timestamp_position * time_precision,
                            end_timestamp_position * time_precision,
                        ),
                    }
                )
            last_slice = current_slice

        return offsets

    @lru_cache()
    def timestamp_ids(self, time_precision=0.02):
        """
        Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

        Args:
            time_precision (`float`, `optional`, defaults to 0.02):
                The time ratio to convert from token to time.
        """
        return self.convert_tokens_to_ids([("<|%.2f|>" % (i * time_precision)) for i in range(1500 + 1)])

    def _preprocess_token_ids(self, token_ids, skip_special_tokens: bool = False):
        """
        Pre-process the token ids for decoding by removing the prompt tokens ids and timestamp token ids.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Typically, obtained using the `__call__` method of the tokenizer.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens from the token ids. If `True`, the prompt token ids will be
                removed.
        """
        if skip_special_tokens:
            prompt_token_id = self.convert_tokens_to_ids("<|startofprev|>")
            decoder_start_token_id = self.convert_tokens_to_ids("<|startoftranscript|>")
            token_ids = self._strip_prompt(token_ids, prompt_token_id, decoder_start_token_id)

        return token_ids

    def _filter_timestamp_ids(self, token_ids):
        """
        This method removes timestamp patterns from the given token IDs.

        Args:
            self (WhisperTokenizer): The instance of the WhisperTokenizer class.
            token_ids (str): A string containing token IDs with timestamp patterns to be filtered.

        Returns:
            None: This method returns None after removing the timestamp patterns from the token IDs.

        Raises:
            None.
        """
        return re.sub(self.timestamp_pat, "", token_ids)

    def decode(
        self,
        token_ids,
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        output_offsets: bool = False,
        time_precision=0.02,
        decode_with_timestamps: bool = False,
        normalize: bool = False,
        basic_normalize: bool = False,
        remove_diacritics: bool = False,
        **kwargs,
    ) -> str:
        """
        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
        tokens and clean up tokenization spaces.

        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding.
            clean_up_tokenization_spaces (`bool`, *optional*):
                Whether or not to clean up the tokenization spaces. If `None`, will default to
                `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
            output_offsets (`bool`, *optional*, defaults to `False`):
                Whether or not to output the offsets of the tokens. This should only be set if the model predicted
                timestamps.
            time_precision (`float`, `optional`, defaults to 0.02):
                The time ratio to convert from token to time.
            decode_with_timestamps (`bool`, *optional*, defaults to `False`):
                Whether or not to decode with timestamps included in the raw text.
            normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to apply the English text normalizer to the decoded text. Only applicable when the
                target text is in English. Otherwise, the basic text normalizer should be applied.
            basic_normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual
                target text.
            remove_diacritics (`bool`, *optional*, defaults to `False`):
                Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may
                destroy information in the decoded text, hence it should be used with caution.
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the underlying model specific decode method.
        Returns:
            `str`: The decoded sentence.
        """
        filtered_ids = self._preprocess_token_ids(
            token_ids,
            skip_special_tokens=skip_special_tokens,
        )

        text = super().decode(
            filtered_ids,
            skip_special_tokens=skip_special_tokens,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            normalize=normalize,
            basic_normalize=basic_normalize,
            remove_diacritics=remove_diacritics,
            **kwargs,
        )
        if decode_with_timestamps:
            # legacy method to decode timestamps when not included in the tokenizer vocabulary
            text = self._decode_with_timestamps(
                filtered_ids, time_precision=time_precision, skip_special_tokens=skip_special_tokens
            )
        else:
            text = self._filter_timestamp_ids(text)

        # retrieve offsets
        if output_offsets:
            offsets = self._compute_offsets(token_ids, time_precision=time_precision)
            return {"text": text, "offsets": offsets}
        return text

    def _decode(
        self,
        token_ids: Union[int, List[int]],
        skip_special_tokens: bool = False,
        normalize: bool = False,
        basic_normalize: bool = False,
        remove_diacritics: bool = False,
        **kwargs,
    ) -> str:
        """
        Decodes a sequence of token IDs into a string representation.

        Args:
            self (WhisperTokenizer): An instance of the WhisperTokenizer class.
            token_ids (Union[int, List[int]]): The token IDs to be decoded. It can be either a single integer
                or a list of integers.
            skip_special_tokens (bool, optional): Whether to skip special tokens while decoding. Defaults to False.
            normalize (bool, optional): Whether to apply normalization to the decoded text. Defaults to False.
            basic_normalize (bool, optional): Whether to apply basic normalization to the decoded text. Defaults to False.
            remove_diacritics (bool, optional): Whether to remove diacritics from the decoded text. Defaults to False.
            **kwargs: Additional keyword arguments.

        Returns:
            str: The decoded text as a string.

        Raises:
            None.

        Note:
            - If `skip_special_tokens` is True, special tokens will be excluded from the decoded text.
            - The `normalize` parameter takes precedence over the `basic_normalize` parameter.
            - If `normalize` is True, the decoded text will be cleaned using the `_normalize` method of the
            WhisperTokenizer class.
            - If `basic_normalize` is True, the decoded text will be cleaned using the `_basic_normalize` method of
            the WhisperTokenizer class, with an option to remove diacritics.
            - If both `normalize` and `basic_normalize` are False, the decoded text will be the concatenation of
            the tokens without any additional cleaning.

        Example:
            ```python
            >>> tokenizer = WhisperTokenizer()
            >>> tokens = [101, 2023, 2003, 1037, 2307, 1055, 1006, 102]
            >>> decoded_text = tokenizer._decode(tokens, skip_special_tokens=True, normalize=True)
            >>> print(decoded_text)
            'the quick brown fox'
            >>> decoded_text = tokenizer._decode(tokens, skip_special_tokens=True, basic_normalize=True, remove_diacritics=True)
            >>> print(decoded_text)
            'the quick brown fox'
            >>> decoded_text = tokenizer._decode(tokens)
            >>> print(decoded_text)
            '[CLS]thequickbrownfox[SEP]'
            ```
        """
        self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)
        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)

        # To avoid mixing byte-level and unicode for byte-level BPT
        # we need to build string separately for added tokens and byte-level tokens
        # cf. https://github.com/huggingface/transformers/issues/1133
        sub_texts = []
        current_sub_text = []
        for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                continue
            if token in self.added_tokens_encoder:
                if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(current_sub_text))
                    current_sub_text = []
                sub_texts.append(token)
            else:
                current_sub_text.append(token)
        if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(current_sub_text))

        text = "".join(sub_texts)

        if normalize:
            clean_text = self._normalize(text)
            return clean_text
        if basic_normalize:
            clean_text = self._basic_normalize(text, remove_diacritics=remove_diacritics)
            return clean_text
        return text

    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.convert_tokens_to_string with GPT2 -> Whisper
    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        text = "".join(tokens)
        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
        return text

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary to specified directory.

        Args:
            self: Instance of the WhisperTokenizer class.
            save_directory (str): The directory path where the vocabulary files will be saved.
            filename_prefix (Optional[str]): A string prefix to be added to the filenames. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the paths to the saved vocabulary files - vocab_file, merge_file,
                and normalizer_file.

        Raises:
            FileNotFoundError: If the save_directory does not exist.
            TypeError: If the save_directory is not a valid directory path.
            IOError: If there is an issue with writing the vocabulary files to the specified directory.
        """
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )
        normalizer_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["normalizer_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
            writer.write("#version: 0.2\n")
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!"
                    )
                    index = token_index
                writer.write(" ".join(bpe_tokens) + "\n")
                index += 1

        if self.english_spelling_normalizer is not None:
            with open(normalizer_file, "w", encoding="utf-8") as f:
                f.write(
                    json.dumps(self.english_spelling_normalizer, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
                )

        return vocab_file, merge_file, normalizer_file

    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.prepare_for_tokenization with GPT2 -> Whisper
    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
        """
        This method prepares the input text for tokenization by potentially adding a prefix space.

        Args:
            self: Reference to the current instance of the class.
            text (str): The input text to be tokenized.
            is_split_into_words (bool): A flag indicating whether the text is already split into words or not.
                Default is False.

        Returns:
            tuple: A tuple containing the modified text and any remaining keyword arguments.

        Raises:
            None
        """
        add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)
        if is_split_into_words or add_prefix_space:
            text = " " + text
        return (text, kwargs)

    @property
    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.default_chat_template
    def default_chat_template(self):
        """
        A simple chat template that ignores role information and just concatenates messages with EOS tokens.
        """
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://hf-mirror.com/docs/transformers/main/chat_templating for more information.\n"
        )
        return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"

    def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
        """
        Retrieve the decoder prompt IDs for the WhisperTokenizer.

        Args:
            self (WhisperTokenizer): An instance of the WhisperTokenizer class.
            task (str, optional): The task associated with the decoder prompt. Defaults to None.
            language (str, optional): The language associated with the decoder prompt. Defaults to None.
            no_timestamps (bool, optional): Flag indicating whether to exclude timestamps from the decoder prompt.
                Defaults to True.

        Returns:
            list: A list of tuples containing the rank and token of each forced decoder token.

        Raises:
            None.

        This method retrieves the decoder prompt IDs to be used during tokenization. The decoder prompt IDs are based on
        the provided task, language, and timestamp preferences. By default, the method excludes timestamps from the
        decoder prompt. The decoder prompt IDs are returned as a list of tuples, where each tuple consists of the rank
        and token of each forced decoder token.

        Note:
            The decoder prompt IDs exclude the initial token which is reserved for special purposes.
        """
        self.set_prefix_tokens(task=task, language=language, predict_timestamps=not no_timestamps)
        # prefix tokens are of the form: <|startoftranscript|> <|lang_id|> <|task|> <|notimestamps|>
        # we don't want to force the bos token at position 1, as this is the starting token
        # when we generate, so we slice the prefix tokens to: <|lang_id|> <|task|> <|notimestamps|>
        # to get the forced tokens
        forced_tokens = self.prefix_tokens[1:]
        forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_tokens)]
        return forced_decoder_ids

    def _decode_asr(self, model_outputs, *, return_timestamps, return_language, time_precision):
        """
        This method decodes the output of an Automatic Speech Recognition (ASR) model.

        Args:
            self (object): The instance of the WhisperTokenizer class.
            model_outputs (object): The output of the ASR model to be decoded.

            return_timestamps (bool): If True, timestamps will be returned along with the decoded output.
                Default is False.
            return_language (bool): If True, the language information will be returned along with the decoded output.
                Default is False.
            time_precision (str): The precision of the timestamps to be returned. Valid values are 'millisecond',
                'second', or 'minute'.

        Returns:
            None: This method does not return a value, but the decoded output, timestamps, and language information
                are accessible through other means.

        Raises:
            ValueError: If the provided model_outputs is invalid or incompatible with the decoding process.
            RuntimeError: If there is an issue with the decoding process that cannot be handled within the method.
        """
        return _decode_asr(
            self,
            model_outputs,
            return_timestamps=return_timestamps,
            return_language=return_language,
            time_precision=time_precision,
        )

    def get_prompt_ids(self, text: str, return_tensors="np"):
        """Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`]."""
        batch_encoding = self("<|startofprev|>", " " + text.strip(), add_special_tokens=False)

        # Check for special tokens
        prompt_text_ids = batch_encoding["input_ids"][1:]
        special_token_id = next((x for x in prompt_text_ids if x >= self.all_special_ids[0]), None)
        if special_token_id is not None:
            token = self.convert_ids_to_tokens(special_token_id)
            raise ValueError(f"Encountered text in the prompt corresponding to disallowed special token: {token}.")

        batch_encoding.convert_to_tensors(tensor_type=return_tensors)
        return batch_encoding["input_ids"]

    @staticmethod
    def _strip_prompt(token_ids: List[int], prompt_token_id: int, decoder_start_token_id: int):
        """
        Removes the prompt from a list of token IDs.

        Args:
            token_ids (List[int]):
                A list of token IDs.

                - The token IDs can be of any type.
                - The list may contain other elements besides token IDs.

            prompt_token_id (int):
                The token ID representing the prompt.

                - The prompt token ID must be an integer.

            decoder_start_token_id (int):
                The token ID representing the start of the decoder.

                - The decoder start token ID must be an integer.

        Returns:
            None

        Raises:
            None

        """
        has_prompt = isinstance(token_ids, list) and token_ids and token_ids[0] == prompt_token_id
        if has_prompt:
            if decoder_start_token_id in token_ids:
                return token_ids[token_ids.index(decoder_start_token_id) :]
            return []

        return token_ids

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.default_chat_template property

A simple chat template that ignores role information and just concatenates messages with EOS tokens.

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.prefix_tokens: List[int] property

This method generates a list of integer tokens representing the prefix sequence for tokenization.

PARAMETER	DESCRIPTION
self	The instance of the WhisperTokenizer class. TYPE: WhisperTokenizer

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integer tokens representing the prefix sequence for tokenization.

RAISES	DESCRIPTION
ValueError	If the provided language is unsupported or not found in the language code mapping.
ValueError	If the provided task is unsupported or not found in the task IDs list.

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.vocab_size: int property

This method returns the size of the vocabulary used by the WhisperTokenizer.

PARAMETER	DESCRIPTION
self	The instance of the WhisperTokenizer class. TYPE: WhisperTokenizer

RETURNS	DESCRIPTION
int	The size of the vocabulary used by the WhisperTokenizer. TYPE: int

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.__init__(vocab_file, merges_file, normalizer_file=None, errors='replace', unk_token='<|endoftext|>', bos_token='<|endoftext|>', eos_token='<|endoftext|>', pad_token=None, add_prefix_space=False, language=None, task=None, predict_timestamps=False, **kwargs)

init

This method initializes an instance of the WhisperTokenizer class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The path to the vocabulary file containing the token encoding. TYPE: str
merges_file	The path to the file containing BPE merges for tokenization. TYPE: str
normalizer_file	The path to the file containing English spelling normalizer. Defaults to None. TYPE: str DEFAULT: None
errors	The error handling scheme to use for encoding/decoding errors. Defaults to 'replace'. TYPE: str DEFAULT: 'replace'
unk_token	The unknown token to be used during tokenization. Defaults to 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
bos_token	The beginning of sentence token. Defaults to 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
eos_token	The end of sentence token. Defaults to 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
pad_token	The padding token. Defaults to None. TYPE: str DEFAULT: None
add_prefix_space	Whether to add a prefix space during tokenization. Defaults to False. TYPE: bool DEFAULT: False
language	The language of the text. Defaults to None. TYPE: str DEFAULT: None
task	The task for tokenization. Defaults to None. TYPE: str DEFAULT: None
predict_timestamps	Whether to predict timestamps. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
FileNotFoundError	If the vocab_file, merges_file, or normalizer_file does not exist.
ValueError	If the provided unk_token, bos_token, eos_token, or pad_token is not a string.
TypeError	If the provided unk_token, bos_token, eos_token, or pad_token is not a string or an AddedToken instance.
UnicodeDecodeError	If an error occurs during the decoding of vocab_file or merges_file.
KeyError	If an error occurs during the creation of the bpe_ranks dictionary.
error	If an error occurs during the compilation of regular expressions.
JSONDecodeError	If an error occurs during the decoding of normalizer_file.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def __init__(
    self,
    vocab_file,
    merges_file,
    normalizer_file=None,
    errors="replace",
    unk_token="<|endoftext|>",
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
    pad_token=None,
    add_prefix_space=False,
    language=None,
    task=None,
    predict_timestamps=False,
    **kwargs,
):
    """
    __init__

    This method initializes an instance of the WhisperTokenizer class.

    Args:
        self: The instance of the class.
        vocab_file (str): The path to the vocabulary file containing the token encoding.
        merges_file (str): The path to the file containing BPE merges for tokenization.
        normalizer_file (str, optional): The path to the file containing English spelling normalizer.
            Defaults to None.
        errors (str): The error handling scheme to use for encoding/decoding errors. Defaults to 'replace'.
        unk_token (str): The unknown token to be used during tokenization. Defaults to 'endoftext'.
        bos_token (str): The beginning of sentence token. Defaults to 'endoftext'.
        eos_token (str): The end of sentence token. Defaults to 'endoftext'.
        pad_token (str, optional): The padding token. Defaults to None.
        add_prefix_space (bool): Whether to add a prefix space during tokenization. Defaults to False.
        language (str, optional): The language of the text. Defaults to None.
        task (str, optional): The task for tokenization. Defaults to None.
        predict_timestamps (bool): Whether to predict timestamps. Defaults to False.

    Returns:
        None.

    Raises:
        FileNotFoundError: If the vocab_file, merges_file, or normalizer_file does not exist.
        ValueError: If the provided unk_token, bos_token, eos_token, or pad_token is not a string.
        TypeError: If the provided unk_token, bos_token, eos_token, or pad_token is not a string or an
            AddedToken instance.
        UnicodeDecodeError: If an error occurs during the decoding of vocab_file or merges_file.
        KeyError: If an error occurs during the creation of the bpe_ranks dictionary.
        re.error: If an error occurs during the compilation of regular expressions.
        json.JSONDecodeError: If an error occurs during the decoding of normalizer_file.
    """
    bos_token = (
        AddedToken(bos_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(bos_token, str)
        else bos_token
    )
    eos_token = (
        AddedToken(eos_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(eos_token, str)
        else eos_token
    )
    unk_token = (
        AddedToken(unk_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(unk_token, str)
        else unk_token
    )
    pad_token = (
        AddedToken(pad_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(pad_token, str)
        else pad_token
    )

    with open(vocab_file, encoding="utf-8") as vocab_handle:
        self.encoder = json.load(vocab_handle)
    self.decoder = {v: k for k, v in self.encoder.items()}
    self.errors = errors  # how to handle errors in decoding
    self.byte_encoder = bytes_to_unicode()
    self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
    with open(merges_file, encoding="utf-8") as merges_handle:
        bpe_merges = merges_handle.read().split("\n")[1:-1]
    bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
    self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
    self.cache = {}
    self.add_prefix_space = add_prefix_space

    if normalizer_file is not None:
        with open(normalizer_file, encoding="utf-8") as vocab_handle:
            self.english_spelling_normalizer = json.load(vocab_handle)
    else:
        self.english_spelling_normalizer = None

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
    self.timestamp_pat = re.compile(r"<\|(\d+\.\d+)\|>")

    self.language = language
    super().__init__(
        errors=errors,
        unk_token=unk_token,
        bos_token=bos_token,
        eos_token=eos_token,
        pad_token=pad_token,
        add_prefix_space=add_prefix_space,
        **kwargs,
    )

    self.task = task
    self.predict_timestamps = predict_timestamps

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.bpe(token)

This method, named 'bpe', is part of the class 'WhisperTokenizer' and performs a Byte Pair Encoding (BPE) algorithm on the given 'token' parameter.

PARAMETER	DESCRIPTION
self	An instance of the 'WhisperTokenizer' class.
token	The token to be encoded using the BPE algorithm. TYPE: str

RETURNS	DESCRIPTION
str	The encoded token after applying the BPE algorithm.

The BPE algorithm is applied to the 'token' by iteratively replacing the most frequent pairs of characters with a new character, until no more pairs can be found. The resulting encoded token is then returned.

The method first checks if the 'token' is already present in the cache. If so, it returns the cached value. Otherwise, it proceeds with the BPE encoding process.

The 'token' is converted into a tuple of characters named 'word'. The method then obtains all possible pairs of characters from 'word' using the 'get_pairs' function.

If no pairs are found, the 'token' is already fully encoded and is returned as is.

Otherwise, the method enters a loop, where it finds the most frequent pair of characters ('bigram') from the 'pairs' list based on the 'bpe_ranks' dictionary. If the 'bigram' is not present in the 'bpe_ranks' dictionary, the loop is terminated.

The method then replaces all occurrences of the 'bigram' in the 'word' with a new character. The 'word' is iterated through, and whenever the current character matches the first character of the 'bigram' and the next character matches the second character of the 'bigram', the new character is appended to the 'new_word' list. Otherwise, the current character is appended as is.

The 'new_word' is converted back to a tuple and assigned to 'word'. If the length of 'word' becomes 1, indicating that the token is fully encoded, the loop is terminated.

The 'pairs' are updated by obtaining all possible pairs from the updated 'word'.

Finally, the 'word' is joined using spaces to form a string, which is then cached with the 'token' as the key in the 'cache' dictionary.

The encoded 'word' is returned as the result of the method.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def bpe(self, token):
    """
    This method, named 'bpe', is part of the class 'WhisperTokenizer' and performs a Byte Pair Encoding (BPE)
    algorithm on the given 'token' parameter.

    Args:
        self: An instance of the 'WhisperTokenizer' class.
        token (str): The token to be encoded using the BPE algorithm.

    Returns:
        str: The encoded token after applying the BPE algorithm.

    Raises:
        None.

    The BPE algorithm is applied to the 'token' by iteratively replacing the most frequent pairs of characters with
    a new character, until no more pairs can be found. The resulting encoded token is then returned.

    The method first checks if the 'token' is already present in the cache. If so, it returns the cached value.
    Otherwise, it proceeds with the BPE encoding process.

    The 'token' is converted into a tuple of characters named 'word'. The method then obtains all possible pairs
    of characters from 'word' using the 'get_pairs' function.

    If no pairs are found, the 'token' is already fully encoded and is returned as is.

    Otherwise, the method enters a loop, where it finds the most frequent pair of characters ('bigram') from the
    'pairs' list based on the 'bpe_ranks' dictionary. If the 'bigram' is not present in the 'bpe_ranks' dictionary,
    the loop is terminated.

    The method then replaces all occurrences of the 'bigram' in the 'word' with a new character. The 'word' is
    iterated through, and whenever the current character matches the first character of the 'bigram' and the next
    character matches the second character of the 'bigram', the new character is appended to the 'new_word' list.
    Otherwise, the current character is appended as is.

    The 'new_word' is converted back to a tuple and assigned to 'word'. If the length of 'word' becomes 1,
    indicating that the token is fully encoded, the loop is terminated.

    The 'pairs' are updated by obtaining all possible pairs from the updated 'word'.

    Finally, the 'word' is joined using spaces to form a string, which is then cached with the 'token' as
    the key in the 'cache' dictionary.

    The encoded 'word' is returned as the result of the method.
    """
    if token in self.cache:
        return self.cache[token]
    word = tuple(token)
    pairs = get_pairs(word)

    if not pairs:
        return token

    while True:
        bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
        if bigram not in self.bpe_ranks:
            break
        first, second = bigram
        new_word = []
        i = 0
        while i < len(word):
            try:
                j = word.index(first, i)
            except ValueError:
                new_word.extend(word[i:])
                break
            else:
                new_word.extend(word[i:j])
                i = j

            if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                new_word.append(first + second)
                i += 2
            else:
                new_word.append(word[i])
                i += 1
        new_word = tuple(new_word)
        word = new_word
        if len(word) == 1:
            break
        pairs = get_pairs(word)
    word = " ".join(word)
    self.cache[token] = word
    return word

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence by appending eos_token_id.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
    """Build model inputs from a sequence by appending eos_token_id."""
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0 + [self.eos_token_id]
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_0 + token_ids_1 + [self.eos_token_id]

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    text = "".join(tokens)
    text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
    return text

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.decode(token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=None, output_offsets=False, time_precision=0.02, decode_with_timestamps=False, normalize=False, basic_normalize=False, remove_diacritics=False, **kwargs)

Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces.

Similar to doing self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids)).

PARAMETER	DESCRIPTION
token_ids	List of tokenized input ids. Can be obtained using the __call__ method. TYPE: `Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`
skip_special_tokens	Whether or not to remove special tokens in the decoding. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
clean_up_tokenization_spaces	Whether or not to clean up the tokenization spaces. If None, will default to self.clean_up_tokenization_spaces (available in the tokenizer_config). TYPE: `bool`, *optional* DEFAULT: None
output_offsets	Whether or not to output the offsets of the tokens. This should only be set if the model predicted timestamps. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
time_precision	The time ratio to convert from token to time. TYPE: `float`, `optional`, defaults to 0.02 DEFAULT: 0.02
decode_with_timestamps	Whether or not to decode with timestamps included in the raw text. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
normalize	Whether or not to apply the English text normalizer to the decoded text. Only applicable when the target text is in English. Otherwise, the basic text normalizer should be applied. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
basic_normalize	Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual target text. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
remove_diacritics	Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may destroy information in the decoded text, hence it should be used with caution. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Will be passed to the underlying model specific decode method. TYPE: additional keyword arguments, *optional* DEFAULT: {}

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def decode(
    self,
    token_ids,
    skip_special_tokens: bool = False,
    clean_up_tokenization_spaces: bool = None,
    output_offsets: bool = False,
    time_precision=0.02,
    decode_with_timestamps: bool = False,
    normalize: bool = False,
    basic_normalize: bool = False,
    remove_diacritics: bool = False,
    **kwargs,
) -> str:
    """
    Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
    tokens and clean up tokenization spaces.

    Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

    Args:
        token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
            List of tokenized input ids. Can be obtained using the `__call__` method.
        skip_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to remove special tokens in the decoding.
        clean_up_tokenization_spaces (`bool`, *optional*):
            Whether or not to clean up the tokenization spaces. If `None`, will default to
            `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
        output_offsets (`bool`, *optional*, defaults to `False`):
            Whether or not to output the offsets of the tokens. This should only be set if the model predicted
            timestamps.
        time_precision (`float`, `optional`, defaults to 0.02):
            The time ratio to convert from token to time.
        decode_with_timestamps (`bool`, *optional*, defaults to `False`):
            Whether or not to decode with timestamps included in the raw text.
        normalize (`bool`, *optional*, defaults to `False`):
            Whether or not to apply the English text normalizer to the decoded text. Only applicable when the
            target text is in English. Otherwise, the basic text normalizer should be applied.
        basic_normalize (`bool`, *optional*, defaults to `False`):
            Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual
            target text.
        remove_diacritics (`bool`, *optional*, defaults to `False`):
            Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may
            destroy information in the decoded text, hence it should be used with caution.
        kwargs (additional keyword arguments, *optional*):
            Will be passed to the underlying model specific decode method.
    Returns:
        `str`: The decoded sentence.
    """
    filtered_ids = self._preprocess_token_ids(
        token_ids,
        skip_special_tokens=skip_special_tokens,
    )

    text = super().decode(
        filtered_ids,
        skip_special_tokens=skip_special_tokens,
        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
        normalize=normalize,
        basic_normalize=basic_normalize,
        remove_diacritics=remove_diacritics,
        **kwargs,
    )
    if decode_with_timestamps:
        # legacy method to decode timestamps when not included in the tokenizer vocabulary
        text = self._decode_with_timestamps(
            filtered_ids, time_precision=time_precision, skip_special_tokens=skip_special_tokens
        )
    else:
        text = self._filter_timestamp_ids(text)

    # retrieve offsets
    if output_offsets:
        offsets = self._compute_offsets(token_ids, time_precision=time_precision)
        return {"text": text, "offsets": offsets}
    return text

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_decoder_prompt_ids(task=None, language=None, no_timestamps=True)

Retrieve the decoder prompt IDs for the WhisperTokenizer.

PARAMETER	DESCRIPTION
self	An instance of the WhisperTokenizer class. TYPE: WhisperTokenizer
task	The task associated with the decoder prompt. Defaults to None. TYPE: str DEFAULT: None
language	The language associated with the decoder prompt. Defaults to None. TYPE: str DEFAULT: None
no_timestamps	Flag indicating whether to exclude timestamps from the decoder prompt. Defaults to True. TYPE: bool DEFAULT: True

RETURNS	DESCRIPTION
list	A list of tuples containing the rank and token of each forced decoder token.

This method retrieves the decoder prompt IDs to be used during tokenization. The decoder prompt IDs are based on the provided task, language, and timestamp preferences. By default, the method excludes timestamps from the decoder prompt. The decoder prompt IDs are returned as a list of tuples, where each tuple consists of the rank and token of each forced decoder token.

Note

The decoder prompt IDs exclude the initial token which is reserved for special purposes.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
    """
    Retrieve the decoder prompt IDs for the WhisperTokenizer.

    Args:
        self (WhisperTokenizer): An instance of the WhisperTokenizer class.
        task (str, optional): The task associated with the decoder prompt. Defaults to None.
        language (str, optional): The language associated with the decoder prompt. Defaults to None.
        no_timestamps (bool, optional): Flag indicating whether to exclude timestamps from the decoder prompt.
            Defaults to True.

    Returns:
        list: A list of tuples containing the rank and token of each forced decoder token.

    Raises:
        None.

    This method retrieves the decoder prompt IDs to be used during tokenization. The decoder prompt IDs are based on
    the provided task, language, and timestamp preferences. By default, the method excludes timestamps from the
    decoder prompt. The decoder prompt IDs are returned as a list of tuples, where each tuple consists of the rank
    and token of each forced decoder token.

    Note:
        The decoder prompt IDs exclude the initial token which is reserved for special purposes.
    """
    self.set_prefix_tokens(task=task, language=language, predict_timestamps=not no_timestamps)
    # prefix tokens are of the form: <|startoftranscript|> <|lang_id|> <|task|> <|notimestamps|>
    # we don't want to force the bos token at position 1, as this is the starting token
    # when we generate, so we slice the prefix tokens to: <|lang_id|> <|task|> <|notimestamps|>
    # to get the forced tokens
    forced_tokens = self.prefix_tokens[1:]
    forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_tokens)]
    return forced_decoder_ids

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_prompt_ids(text, return_tensors='np')

Converts prompt text to IDs that can be passed to [~WhisperForConditionalGeneration.generate].

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def get_prompt_ids(self, text: str, return_tensors="np"):
    """Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`]."""
    batch_encoding = self("<|startofprev|>", " " + text.strip(), add_special_tokens=False)

    # Check for special tokens
    prompt_text_ids = batch_encoding["input_ids"][1:]
    special_token_id = next((x for x in prompt_text_ids if x >= self.all_special_ids[0]), None)
    if special_token_id is not None:
        token = self.convert_ids_to_tokens(special_token_id)
        raise ValueError(f"Encountered text in the prompt corresponding to disallowed special token: {token}.")

    batch_encoding.convert_to_tensors(tensor_type=return_tensors)
    return batch_encoding["input_ids"]

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    prefix_ones = [1] * len(self.prefix_tokens)
    suffix_ones = [1]
    if token_ids_1 is None:
        return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
    return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_vocab()

Returns the vocabulary of the WhisperTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the WhisperTokenizer class. TYPE: WhisperTokenizer

RETURNS	DESCRIPTION
dict	A dictionary representing the vocabulary of the WhisperTokenizer instance. The keys of the dictionary are the tokens in the vocabulary, and the values are their respective token IDs.

Note

This method builds the vocabulary by converting the token IDs to tokens using the convert_ids_to_tokens method of the WhisperTokenizer instance. The tokens and their corresponding IDs are stored in a dictionary. Additionally, any added tokens are also included in the vocabulary.

Example

>>> tokenizer = WhisperTokenizer()
>>> vocab = tokenizer.get_vocab()
>>> print(vocab)
{'[PAD]': 0, '[UNK]': 1, '[CLS]': 2, '[SEP]': 3, '[MASK]': 4, 'hello': 5, 'world': 6}

In the example above, the get_vocab method is called on a WhisperTokenizer instance, which returns a dictionary representing the vocabulary of the tokenizer. The vocabulary includes the default special tokens as well as any added tokens.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def get_vocab(self):
    """
    Returns the vocabulary of the WhisperTokenizer instance.

    Args:
        self (WhisperTokenizer): The instance of the WhisperTokenizer class.

    Returns:
        dict: A dictionary representing the vocabulary of the WhisperTokenizer instance.
            The keys of the dictionary are the tokens in the vocabulary, and the values are their respective
            token IDs.

    Raises:
        None.

    Note:
        This method builds the vocabulary by converting the token IDs to tokens using the `convert_ids_to_tokens`
        method of the WhisperTokenizer instance. The tokens and their corresponding IDs are stored in a dictionary.
        Additionally, any added tokens are also included in the vocabulary.

    Example:
        ```python
        >>> tokenizer = WhisperTokenizer()
        >>> vocab = tokenizer.get_vocab()
        >>> print(vocab)
        {'[PAD]': 0, '[UNK]': 1, '[CLS]': 2, '[SEP]': 3, '[MASK]': 4, 'hello': 5, 'world': 6}
        ```

    In the example above, the `get_vocab` method is called on a WhisperTokenizer instance, which returns a
    dictionary representing the vocabulary of the tokenizer. The vocabulary includes the default special tokens
    as well as any added tokens.
    """
    vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
    vocab.update(self.added_tokens_encoder)
    return vocab

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.prepare_for_tokenization(text, is_split_into_words=False, **kwargs)

This method prepares the input text for tokenization by potentially adding a prefix space.

PARAMETER	DESCRIPTION
self	Reference to the current instance of the class.
text	The input text to be tokenized. TYPE: str
is_split_into_words	A flag indicating whether the text is already split into words or not. Default is False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
tuple	A tuple containing the modified text and any remaining keyword arguments.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
    """
    This method prepares the input text for tokenization by potentially adding a prefix space.

    Args:
        self: Reference to the current instance of the class.
        text (str): The input text to be tokenized.
        is_split_into_words (bool): A flag indicating whether the text is already split into words or not.
            Default is False.

    Returns:
        tuple: A tuple containing the modified text and any remaining keyword arguments.

    Raises:
        None
    """
    add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)
    if is_split_into_words or add_prefix_space:
        text = " " + text
    return (text, kwargs)

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary to specified directory.

PARAMETER	DESCRIPTION
self	Instance of the WhisperTokenizer class.
save_directory	The directory path where the vocabulary files will be saved. TYPE: str
filename_prefix	A string prefix to be added to the filenames. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the paths to the saved vocabulary files - vocab_file, merge_file, and normalizer_file.

RAISES	DESCRIPTION
FileNotFoundError	If the save_directory does not exist.
TypeError	If the save_directory is not a valid directory path.
IOError	If there is an issue with writing the vocabulary files to the specified directory.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary to specified directory.

    Args:
        self: Instance of the WhisperTokenizer class.
        save_directory (str): The directory path where the vocabulary files will be saved.
        filename_prefix (Optional[str]): A string prefix to be added to the filenames. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the paths to the saved vocabulary files - vocab_file, merge_file,
            and normalizer_file.

    Raises:
        FileNotFoundError: If the save_directory does not exist.
        TypeError: If the save_directory is not a valid directory path.
        IOError: If there is an issue with writing the vocabulary files to the specified directory.
    """
    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )
    merge_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
    )
    normalizer_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["normalizer_file"]
    )

    with open(vocab_file, "w", encoding="utf-8") as f:
        f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

    index = 0
    with open(merge_file, "w", encoding="utf-8") as writer:
        writer.write("#version: 0.2\n")
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
            if index != token_index:
                logger.warning(
                    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                    " Please check that the tokenizer is not corrupted!"
                )
                index = token_index
            writer.write(" ".join(bpe_tokens) + "\n")
            index += 1

    if self.english_spelling_normalizer is not None:
        with open(normalizer_file, "w", encoding="utf-8") as f:
            f.write(
                json.dumps(self.english_spelling_normalizer, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
            )

    return vocab_file, merge_file, normalizer_file

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.set_prefix_tokens(language=None, task=None, predict_timestamps=None)

Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to update the prefix tokens as required when fine-tuning.

Example

>>> # instantiate the tokenizer and set the prefix token to Spanish
>>> tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish")
>>> # now switch the prefix token from Spanish to French
>>> tokenizer.set_prefix_tokens(language="french")

PARAMETER	DESCRIPTION
language	The language of the transcription text. TYPE: `str`, *optional*, defaults to `None` DEFAULT: None
task	Task identifier to append at the start of sequence (if any). TYPE: `str`, *optional*, defaults to `None` DEFAULT: None
predict_timestamps	Whether to omit the <|notimestamps|> token at the start of the sequence. TYPE: `bool`, *optional*, defaults to `None` DEFAULT: None

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def set_prefix_tokens(self, language: str = None, task: str = None, predict_timestamps: bool = None):
    """
    Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to
    update the prefix tokens as required when fine-tuning.

    Example:
        ```python
        >>> # instantiate the tokenizer and set the prefix token to Spanish
        >>> tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish")
        >>> # now switch the prefix token from Spanish to French
        >>> tokenizer.set_prefix_tokens(language="french")
        ```

    Args:
        language (`str`, *optional*, defaults to `None`):
            The language of the transcription text.
        task (`str`, *optional*, defaults to `None`):
            Task identifier to append at the start of sequence (if any).
        predict_timestamps (`bool`, *optional*, defaults to `None`):
            Whether to omit the `<|notimestamps|>` token at the start of the sequence.
    """
    self.language = language if language is not None else self.language
    self.task = task if task is not None else self.task
    self.predict_timestamps = predict_timestamps if predict_timestamps is not None else self.predict_timestamps

mindnlp.transformers.models.whisper.tokenization_whisper.WhisperTokenizer.timestamp_ids(time_precision=0.02) cached

Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

PARAMETER	DESCRIPTION
time_precision	The time ratio to convert from token to time. TYPE: `float`, `optional`, defaults to 0.02 DEFAULT: 0.02

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

@lru_cache()
def timestamp_ids(self, time_precision=0.02):
    """
    Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

    Args:
        time_precision (`float`, `optional`, defaults to 0.02):
            The time ratio to convert from token to time.
    """
    return self.convert_tokens_to_ids([("<|%.2f|>" % (i * time_precision)) for i in range(1500 + 1)])

mindnlp.transformers.models.whisper.tokenization_whisper.bytes_to_unicode()

Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control characters the bpe code barfs on.

The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 bytes and unicode strings.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def bytes_to_unicode():
    """
    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
    characters the bpe code barfs on.

    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
    tables between utf-8 bytes and unicode strings.
    """
    bs = (
        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
    )
    cs = bs[:]
    n = 0
    for b in range(2**8):
        if b not in bs:
            bs.append(b)
            cs.append(2**8 + n)
            n += 1
    cs = [chr(n) for n in cs]
    return dict(zip(bs, cs))

mindnlp.transformers.models.whisper.tokenization_whisper.get_pairs(word)

Return set of symbol pairs in a word.

Word is represented as tuple of symbols (symbols being variable-length strings).

Source code in mindnlp/transformers/models/whisper/tokenization_whisper.py

def get_pairs(word):
    """
    Return set of symbol pairs in a word.

    Word is represented as tuple of symbols (symbols being variable-length strings).
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

mindnlp.transformers.models.whisper.processing_whisper

Speech processor class for Whisper

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor

Bases: ProcessorMixin

Constructs a Whisper processor which wraps a Whisper feature extractor and a Whisper tokenizer into a single processor.

[WhisperProcessor] offers all the functionalities of [WhisperFeatureExtractor] and [WhisperTokenizer]. See the [~WhisperProcessor.__call__] and [~WhisperProcessor.decode] for more information.

PARAMETER	DESCRIPTION
feature_extractor	An instance of [WhisperFeatureExtractor]. The feature extractor is a required input. TYPE: `WhisperFeatureExtractor`
tokenizer	An instance of [WhisperTokenizer]. The tokenizer is a required input. TYPE: `WhisperTokenizer`

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

class WhisperProcessor(ProcessorMixin):
    r"""
    Constructs a Whisper processor which wraps a Whisper feature extractor and a Whisper tokenizer into a single
    processor.

    [`WhisperProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`WhisperTokenizer`]. See
    the [`~WhisperProcessor.__call__`] and [`~WhisperProcessor.decode`] for more information.

    Args:
        feature_extractor (`WhisperFeatureExtractor`):
            An instance of [`WhisperFeatureExtractor`]. The feature extractor is a required input.
        tokenizer (`WhisperTokenizer`):
            An instance of [`WhisperTokenizer`]. The tokenizer is a required input.
    """
    feature_extractor_class = "WhisperFeatureExtractor"
    tokenizer_class = "WhisperTokenizer"

    def __init__(self, feature_extractor, tokenizer):
        """
        Initializes a new instance of the WhisperProcessor class.

        Args:
            self (WhisperProcessor): The current instance of the WhisperProcessor class.
            feature_extractor: The feature extractor used for processing.
                This should be an object representing the feature extraction mechanism.
            tokenizer: The tokenizer used for processing.
                This should be an object representing the tokenization mechanism.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(feature_extractor, tokenizer)
        self.current_processor = self.feature_extractor
        self._in_target_context_manager = False

    def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
        """
        Method: get_decoder_prompt_ids

        Description:
            This method retrieves the decoder prompt IDs for a given task and language.
            It utilizes the tokenizer to obtain the prompt IDs.

        Args:
            self: The instance of the WhisperProcessor class.
            task (optional): A string representing the task for which decoder prompt IDs are required. Defaults to None.
            language (optional): A string representing the language for which decoder prompt IDs are required.
                Defaults to None.
            no_timestamps (optional): A boolean indicating whether to include timestamps in the decoder prompt IDs.
                Defaults to True.

        Returns:
            None

        Raises:
            None

        Note:
            The decoder prompt IDs are obtained by calling the tokenizer's get_decoder_prompt_ids method with the
            specified task, language, and no_timestamps parameters. The returned decoder prompt IDs are then
            returned by this method.

        Example:
            ```python
            >>> processor = WhisperProcessor()
            >>> decoder_prompt_ids = processor.get_decoder_prompt_ids(task='translation', language='english', no_timestamps=True)
            ```
        """
        return self.tokenizer.get_decoder_prompt_ids(task=task, language=language, no_timestamps=no_timestamps)

    def __call__(self, *args, **kwargs):
        """
        Forwards the `audio` argument to WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] and the `text`
        argument to [`~WhisperTokenizer.__call__`]. Please refer to the doctsring of the above two methods for more
        information.
        """
        # For backward compatibility
        if self._in_target_context_manager:
            return self.current_processor(*args, **kwargs)

        audio = kwargs.pop("audio", None)
        sampling_rate = kwargs.pop("sampling_rate", None)
        text = kwargs.pop("text", None)
        if len(args) > 0:
            audio = args[0]
            args = args[1:]

        if audio is None and text is None:
            raise ValueError("You need to specify either an `audio` or `text` input to process.")

        if audio is not None:
            inputs = self.feature_extractor(audio, *args, sampling_rate=sampling_rate, **kwargs)
        if text is not None:
            encodings = self.tokenizer(text, **kwargs)

        if text is None:
            return inputs

        if audio is None:
            return encodings

        inputs["labels"] = encodings["input_ids"]
        return inputs

    def batch_decode(self, *args, **kwargs):
        """
        This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
        refer to the docstring of this method for more information.
        """
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        """
        This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
        the docstring of this method for more information.
        """
        return self.tokenizer.decode(*args, **kwargs)

    def get_prompt_ids(self, text: str, return_tensors="np"):
        """
        This method retrieves prompt IDs for the given text using the WhisperProcessor class.

        Args:
            self: The instance of the WhisperProcessor class.
            text (str): The input text for which prompt IDs need to be retrieved.
            return_tensors (str, optional): Specifies the type of tensors to be returned. Defaults to 'np'.
                Possible values: 'np' for numpy arrays, 'pt' for PyTorch tensors, 'tf' for TensorFlow tensors.
                Default value: 'np'.

        Returns:
            None: This method does not return any value directly. Instead, it returns the prompt IDs using the
                WhisperProcessor's tokenizer.

        Raises:
            ValueError: If the specified return_tensors value is not one of the allowed options.
            TypeError: If the input text is not a string.
        """
        return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors)

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor.__call__(*args, **kwargs)

Forwards the audio argument to WhisperFeatureExtractor's [~WhisperFeatureExtractor.__call__] and the text argument to [~WhisperTokenizer.__call__]. Please refer to the doctsring of the above two methods for more information.

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

def __call__(self, *args, **kwargs):
    """
    Forwards the `audio` argument to WhisperFeatureExtractor's [`~WhisperFeatureExtractor.__call__`] and the `text`
    argument to [`~WhisperTokenizer.__call__`]. Please refer to the doctsring of the above two methods for more
    information.
    """
    # For backward compatibility
    if self._in_target_context_manager:
        return self.current_processor(*args, **kwargs)

    audio = kwargs.pop("audio", None)
    sampling_rate = kwargs.pop("sampling_rate", None)
    text = kwargs.pop("text", None)
    if len(args) > 0:
        audio = args[0]
        args = args[1:]

    if audio is None and text is None:
        raise ValueError("You need to specify either an `audio` or `text` input to process.")

    if audio is not None:
        inputs = self.feature_extractor(audio, *args, sampling_rate=sampling_rate, **kwargs)
    if text is not None:
        encodings = self.tokenizer(text, **kwargs)

    if text is None:
        return inputs

    if audio is None:
        return encodings

    inputs["labels"] = encodings["input_ids"]
    return inputs

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor.__init__(feature_extractor, tokenizer)

Initializes a new instance of the WhisperProcessor class.

PARAMETER	DESCRIPTION
self	The current instance of the WhisperProcessor class. TYPE: WhisperProcessor
feature_extractor	The feature extractor used for processing. This should be an object representing the feature extraction mechanism.
tokenizer	The tokenizer used for processing. This should be an object representing the tokenization mechanism.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

def __init__(self, feature_extractor, tokenizer):
    """
    Initializes a new instance of the WhisperProcessor class.

    Args:
        self (WhisperProcessor): The current instance of the WhisperProcessor class.
        feature_extractor: The feature extractor used for processing.
            This should be an object representing the feature extraction mechanism.
        tokenizer: The tokenizer used for processing.
            This should be an object representing the tokenization mechanism.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(feature_extractor, tokenizer)
    self.current_processor = self.feature_extractor
    self._in_target_context_manager = False

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor.batch_decode(*args, **kwargs)

This method forwards all its arguments to WhisperTokenizer's [~PreTrainedTokenizer.batch_decode]. Please refer to the docstring of this method for more information.

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

def batch_decode(self, *args, **kwargs):
    """
    This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
    refer to the docstring of this method for more information.
    """
    return self.tokenizer.batch_decode(*args, **kwargs)

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor.decode(*args, **kwargs)

This method forwards all its arguments to WhisperTokenizer's [~PreTrainedTokenizer.decode]. Please refer to the docstring of this method for more information.

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

def decode(self, *args, **kwargs):
    """
    This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
    the docstring of this method for more information.
    """
    return self.tokenizer.decode(*args, **kwargs)

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor.get_decoder_prompt_ids(task=None, language=None, no_timestamps=True)

Description

This method retrieves the decoder prompt IDs for a given task and language. It utilizes the tokenizer to obtain the prompt IDs.

PARAMETER	DESCRIPTION
self	The instance of the WhisperProcessor class.
task	A string representing the task for which decoder prompt IDs are required. Defaults to None. TYPE: optional DEFAULT: None
language	A string representing the language for which decoder prompt IDs are required. Defaults to None. TYPE: optional DEFAULT: None
no_timestamps	A boolean indicating whether to include timestamps in the decoder prompt IDs. Defaults to True. TYPE: optional DEFAULT: True

RETURNS	DESCRIPTION
	None

Note

The decoder prompt IDs are obtained by calling the tokenizer's get_decoder_prompt_ids method with the specified task, language, and no_timestamps parameters. The returned decoder prompt IDs are then returned by this method.

Example

>>> processor = WhisperProcessor()
>>> decoder_prompt_ids = processor.get_decoder_prompt_ids(task='translation', language='english', no_timestamps=True)

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
    """
    Method: get_decoder_prompt_ids

    Description:
        This method retrieves the decoder prompt IDs for a given task and language.
        It utilizes the tokenizer to obtain the prompt IDs.

    Args:
        self: The instance of the WhisperProcessor class.
        task (optional): A string representing the task for which decoder prompt IDs are required. Defaults to None.
        language (optional): A string representing the language for which decoder prompt IDs are required.
            Defaults to None.
        no_timestamps (optional): A boolean indicating whether to include timestamps in the decoder prompt IDs.
            Defaults to True.

    Returns:
        None

    Raises:
        None

    Note:
        The decoder prompt IDs are obtained by calling the tokenizer's get_decoder_prompt_ids method with the
        specified task, language, and no_timestamps parameters. The returned decoder prompt IDs are then
        returned by this method.

    Example:
        ```python
        >>> processor = WhisperProcessor()
        >>> decoder_prompt_ids = processor.get_decoder_prompt_ids(task='translation', language='english', no_timestamps=True)
        ```
    """
    return self.tokenizer.get_decoder_prompt_ids(task=task, language=language, no_timestamps=no_timestamps)

mindnlp.transformers.models.whisper.processing_whisper.WhisperProcessor.get_prompt_ids(text, return_tensors='np')

This method retrieves prompt IDs for the given text using the WhisperProcessor class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperProcessor class.
text	The input text for which prompt IDs need to be retrieved. TYPE: str
return_tensors	Specifies the type of tensors to be returned. Defaults to 'np'. Possible values: 'np' for numpy arrays, 'pt' for PyTorch tensors, 'tf' for TensorFlow tensors. Default value: 'np'. TYPE: str DEFAULT: 'np'

RETURNS	DESCRIPTION
None	This method does not return any value directly. Instead, it returns the prompt IDs using the WhisperProcessor's tokenizer.

RAISES	DESCRIPTION
ValueError	If the specified return_tensors value is not one of the allowed options.
TypeError	If the input text is not a string.

Source code in mindnlp/transformers/models/whisper/processing_whisper.py

def get_prompt_ids(self, text: str, return_tensors="np"):
    """
    This method retrieves prompt IDs for the given text using the WhisperProcessor class.

    Args:
        self: The instance of the WhisperProcessor class.
        text (str): The input text for which prompt IDs need to be retrieved.
        return_tensors (str, optional): Specifies the type of tensors to be returned. Defaults to 'np'.
            Possible values: 'np' for numpy arrays, 'pt' for PyTorch tensors, 'tf' for TensorFlow tensors.
            Default value: 'np'.

    Returns:
        None: This method does not return any value directly. Instead, it returns the prompt IDs using the
            WhisperProcessor's tokenizer.

    Raises:
        ValueError: If the specified return_tensors value is not one of the allowed options.
        TypeError: If the input text is not a string.
    """
    return self.tokenizer.get_prompt_ids(text, return_tensors=return_tensors)

mindnlp.transformers.models.whisper.configuration_whisper

Whisper model configuration

mindnlp.transformers.models.whisper.configuration_whisper.WhisperConfig

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [WhisperModel]. It is used to instantiate a Whisper model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the Whisper openai/whisper-tiny architecture.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
vocab_size	Vocabulary size of the Whisper model. Defines the number of different tokens that can be represented by the decoder_input_ids passed when calling [WhisperModel] TYPE: `int`, *optional*, defaults to 51865 DEFAULT: 51865
num_mel_bins	Number of mel features used per input features. Should correspond to the value used in the WhisperProcessor class. TYPE: `int`, *optional*, defaults to 80 DEFAULT: 80
encoder_layers	Number of encoder layers. TYPE: `int`, *optional*, defaults to 4 DEFAULT: 4
decoder_layers	Number of decoder layers. TYPE: `int`, *optional*, defaults to 4 DEFAULT: 4
encoder_attention_heads	Number of attention heads for each attention layer in the Transformer encoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
decoder_attention_heads	Number of attention heads for each attention layer in the Transformer decoder. TYPE: `int`, *optional*, defaults to 6 DEFAULT: 6
encoder_ffn_dim	Dimensionality of the "intermediate" (often named feed-forward) layer in encoder. TYPE: `int`, *optional*, defaults to 1536 DEFAULT: 1536
decoder_ffn_dim	Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. TYPE: `int`, *optional*, defaults to 1536 DEFAULT: 1536
encoder_layerdrop	The LayerDrop probability for the encoder. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
decoder_layerdrop	The LayerDrop probability for the decoder. See the LayerDrop paper for more details. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
decoder_start_token_id	Corresponds to the "<|startoftranscript|>" token, which is automatically used when no decoder_input_ids are provided to the generate function. It is used to guide the model`s generation process depending on the task. TYPE: `int`, *optional*, defaults to 50257 DEFAULT: 50257
use_cache	Whether or not the model should return the last key/values attentions (not used by all models). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
is_encoder_decoder	Whether the model is used as an encoder/decoder or not. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
activation_function	The non-linear activation function (function or string) in the encoder and pooler. If string, "gelu", "relu", "silu" and "gelu_new" are supported. TYPE: `str`, *optional*, defaults to `"gelu"` DEFAULT: 'gelu'
d_model	Dimensionality of the layers. TYPE: `int`, *optional*, defaults to 384 DEFAULT: 384
dropout	The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. TYPE: `float`, *optional*, defaults to 0.1 DEFAULT: 0.0
attention_dropout	The dropout ratio for the attention probabilities. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
activation_dropout	The dropout ratio for activations inside the fully connected layer. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
init_std	The standard deviation of the truncated_normal_initializer for initializing all weight matrices. TYPE: `float`, *optional*, defaults to 0.02 DEFAULT: 0.02
scale_embedding	Scale embeddings by diving by sqrt(d_model). TYPE: `bool`, *optional*, defaults to False DEFAULT: False
max_source_positions	The maximum sequence length of log-mel filter-bank features that this model might ever be used with. TYPE: `int`, *optional*, defaults to 1500 DEFAULT: 1500
max_target_positions	The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). TYPE: `int`, *optional*, defaults to 448 DEFAULT: 448
pad_token_id	Padding token id. TYPE: `int`, *optional*, defaults to 50256 DEFAULT: 50256
bos_token_id	Begin of stream token id. TYPE: `int`, *optional*, defaults to 50256 DEFAULT: 50256
eos_token_id	End of stream token id. TYPE: `int`, *optional*, defaults to 50256 DEFAULT: 50256
suppress_tokens	A list containing the non-speech tokens that will be used by the logit processor in the generate function. NON_SPEECH_TOKENS and NON_SPEECH_TOKENS_MULTI each correspond to the english-only and the multilingual model. TYPE: `List[int]`, *optional* DEFAULT: None
begin_suppress_tokens	A list containing tokens that will be supressed at the beginning of the sampling process. Initialized as the token for " " (blank_token_id) and the eos_token_id TYPE: `List[int]`, *optional*, defaults to `[220,50256]` DEFAULT: [220, 50256]
use_weighted_layer_sum	Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an instance of [WhisperForAudioClassification]. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
classifier_proj_size	Dimensionality of the projection before token mean-pooling for classification. Only relevant when using an instance of [WhisperForAudioClassification]. TYPE: `int`, *optional*, defaults to 256 DEFAULT: 256
apply_spec_augment	Whether to apply SpecAugment data augmentation to the outputs of the feature encoder. For reference see SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
mask_time_prob	Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking procecure generates mask_time_prob*len(time_axis)/mask_time_length independent masks over the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked, mask_time_prob should be prob_vector_start*mask_time_length. Note that overlap may decrease the actual percentage of masked vectors. This is only relevant if apply_spec_augment == True. TYPE: `float`, *optional*, defaults to 0.05 DEFAULT: 0.05
mask_time_length	Length of vector span along the time axis. TYPE: `int`, *optional*, defaults to 10 DEFAULT: 10
mask_time_min_masks	The minimum number of masks of length mask_feature_length generated along the time axis, each time step, irrespectively of mask_feature_prob. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks'' TYPE: `int`, *optional*, defaults to 2), DEFAULT: 2
mask_feature_prob	Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The masking procecure generates mask_feature_prob*len(feature_axis)/mask_time_length independent masks over the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector span to be masked, mask_feature_prob should be prob_vector_start*mask_feature_length. Note that overlap may decrease the actual percentage of masked vectors. This is only relevant if apply_spec_augment is True. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
mask_feature_length	Length of vector span along the feature axis. TYPE: `int`, *optional*, defaults to 10 DEFAULT: 10
mask_feature_min_masks	The minimum number of masks of length mask_feature_length generated along the feature axis, each time step, irrespectively of mask_feature_prob. Only relevant if mask_feature_prob*len(feature_axis)/mask_feature_length < mask_feature_min_masks. TYPE: `int`, *optional*, defaults to 0), DEFAULT: 0
median_filter_width	Width of the median filter used to smoothen to cross-attention outputs when computing token timestamps. Should be an odd number. TYPE: `int`, *optional*, defaults to 7 DEFAULT: 7

Example

>>> from transformers import WhisperConfig, WhisperModel
...
>>> # Initializing a Whisper tiny style configuration
>>> configuration = WhisperConfig()
...
>>> # Initializing a model (with random weights) from the tiny style configuration
>>> model = WhisperModel(configuration)
...
>>> # Accessing the model configuration
>>> configuration = model.config

Source code in mindnlp/transformers/models/whisper/configuration_whisper.py

class WhisperConfig(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`WhisperModel`]. It is used to instantiate a
    Whisper model according to the specified arguments, defining the model architecture. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the Whisper
    [openai/whisper-tiny](https://hf-mirror.com/openai/whisper-tiny) architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.


    Args:
        vocab_size (`int`, *optional*, defaults to 51865):
            Vocabulary size of the Whisper model. Defines the number of different tokens that can be represented by the
            `decoder_input_ids` passed when calling [`WhisperModel`]
        num_mel_bins (`int`, *optional*, defaults to 80):
            Number of mel features used per input features. Should correspond to the value used in the
            `WhisperProcessor` class.
        encoder_layers (`int`, *optional*, defaults to 4):
            Number of encoder layers.
        decoder_layers (`int`, *optional*, defaults to 4):
            Number of decoder layers.
        encoder_attention_heads (`int`, *optional*, defaults to 6):
            Number of attention heads for each attention layer in the Transformer encoder.
        decoder_attention_heads (`int`, *optional*, defaults to 6):
            Number of attention heads for each attention layer in the Transformer decoder.
        encoder_ffn_dim (`int`, *optional*, defaults to 1536):
            Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.
        decoder_ffn_dim (`int`, *optional*, defaults to 1536):
            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
        encoder_layerdrop (`float`, *optional*, defaults to 0.0):
            The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
        decoder_layerdrop (`float`, *optional*, defaults to 0.0):
            The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
        decoder_start_token_id (`int`, *optional*, defaults to 50257):
            Corresponds to the "<|startoftranscript|>" token, which is automatically used when no `decoder_input_ids`
            are provided to the `generate` function. It is used to guide the model`s generation process depending on
            the task.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).
        is_encoder_decoder (`bool`, *optional*, defaults to `True`):
            Whether the model is used as an encoder/decoder or not.
        activation_function (`str`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"silu"` and `"gelu_new"` are supported.
        d_model (`int`, *optional*, defaults to 384):
            Dimensionality of the layers.
        dropout (`float`, *optional*, defaults to 0.1):
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        activation_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for activations inside the fully connected layer.
        init_std (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        scale_embedding (`bool`, *optional*, defaults to False):
            Scale embeddings by diving by sqrt(d_model).
        max_source_positions (`int`, *optional*, defaults to 1500):
            The maximum sequence length of log-mel filter-bank features that this model might ever be used with.
        max_target_positions (`int`, *optional*, defaults to 448):
            The maximum sequence length that this model might ever be used with. Typically set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        pad_token_id (`int`, *optional*, defaults to 50256):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 50256):
            Begin of stream token id.
        eos_token_id (`int`, *optional*, defaults to 50256):
            End of stream token id.
        suppress_tokens (`List[int]`, *optional*):
            A list containing the non-speech tokens that will be used by the logit processor in the `generate`
            function. NON_SPEECH_TOKENS and NON_SPEECH_TOKENS_MULTI each correspond to the `english-only` and the
            `multilingual` model.
        begin_suppress_tokens (`List[int]`, *optional*, defaults to `[220,50256]`):
            A list containing tokens that will be supressed at the beginning of the sampling process. Initialized as
            the token for `" "` (`blank_token_id`) and the `eos_token_id`
        use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
            Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
            instance of [`WhisperForAudioClassification`].
        classifier_proj_size (`int`, *optional*, defaults to 256):
            Dimensionality of the projection before token mean-pooling for classification. Only relevant when using an
            instance of [`WhisperForAudioClassification`].
        apply_spec_augment (`bool`, *optional*, defaults to `False`):
            Whether to apply *SpecAugment* data augmentation to the outputs of the feature encoder. For reference see
            [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
            Recognition](https://arxiv.org/abs/1904.08779).
        mask_time_prob (`float`, *optional*, defaults to 0.05):
            Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
            procecure generates `mask_time_prob*len(time_axis)/mask_time_length` independent masks over the axis. If
            reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
            masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
            actual percentage of masked vectors. This is only relevant if `apply_spec_augment == True`.
        mask_time_length (`int`, *optional*, defaults to 10):
            Length of vector span along the time axis.
        mask_time_min_masks (`int`, *optional*, defaults to 2),:
            The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
            irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
            mask_time_min_masks''
        mask_feature_prob (`float`, *optional*, defaults to 0.0):
            Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
            masking procecure generates `mask_feature_prob*len(feature_axis)/mask_time_length` independent masks over
            the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
            span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
            may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
            True`.
        mask_feature_length (`int`, *optional*, defaults to 10):
            Length of vector span along the feature axis.
        mask_feature_min_masks (`int`, *optional*, defaults to 0),:
            The minimum number of masks of length `mask_feature_length` generated along the feature axis, each time
            step, irrespectively of `mask_feature_prob`. Only relevant if
            `mask_feature_prob*len(feature_axis)/mask_feature_length < mask_feature_min_masks`.
        median_filter_width (`int`, *optional*, defaults to 7):
            Width of the median filter used to smoothen to cross-attention outputs when computing token timestamps.
            Should be an odd number.

    Example:
        ```python
        >>> from transformers import WhisperConfig, WhisperModel
        ...
        >>> # Initializing a Whisper tiny style configuration
        >>> configuration = WhisperConfig()
        ...
        >>> # Initializing a model (with random weights) from the tiny style configuration
        >>> model = WhisperModel(configuration)
        ...
        >>> # Accessing the model configuration
        >>> configuration = model.config
        ```
    """
    model_type = "whisper"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

    def __init__(
        self,
        vocab_size=51865,
        num_mel_bins=80,
        encoder_layers=4,
        encoder_attention_heads=6,
        decoder_layers=4,
        decoder_attention_heads=6,
        decoder_ffn_dim=1536,
        encoder_ffn_dim=1536,
        encoder_layerdrop=0.0,
        decoder_layerdrop=0.0,
        decoder_start_token_id=50257,
        use_cache=True,
        is_encoder_decoder=True,
        activation_function="gelu",
        d_model=384,
        dropout=0.0,
        attention_dropout=0.0,
        activation_dropout=0.0,
        init_std=0.02,
        scale_embedding=False,
        max_source_positions=1500,
        max_target_positions=448,
        pad_token_id=50256,
        bos_token_id=50256,
        eos_token_id=50256,
        suppress_tokens=None,
        begin_suppress_tokens=[220, 50256],
        use_weighted_layer_sum=False,
        classifier_proj_size=256,
        apply_spec_augment=False,
        mask_time_prob=0.05,
        mask_time_length=10,
        mask_time_min_masks=2,
        mask_feature_prob=0.0,
        mask_feature_length=10,
        mask_feature_min_masks=0,
        median_filter_width=7,
        **kwargs,
    ):
        """
        Initialize a WhisperConfig object with the provided parameters.

        Args:
            self (object): The instance of the class.
            vocab_size (int, optional): The size of the vocabulary. Default is 51865.
            num_mel_bins (int, optional): The number of Mel bins. Default is 80.
            encoder_layers (int, optional): The number of layers in the encoder. Default is 4.
            encoder_attention_heads (int, optional): The number of attention heads in the encoder. Default is 6.
            decoder_layers (int, optional): The number of layers in the decoder. Default is 4.
            decoder_attention_heads (int, optional): The number of attention heads in the decoder. Default is 6.
            decoder_ffn_dim (int, optional): The dimension of the feed-forward network in the decoder. Default is 1536.
            encoder_ffn_dim (int, optional): The dimension of the feed-forward network in the encoder. Default is 1536.
            encoder_layerdrop (float, optional): The probability of dropping a layer in the encoder. Default is 0.0.
            decoder_layerdrop (float, optional): The probability of dropping a layer in the decoder. Default is 0.0.
            decoder_start_token_id (int, optional): The token id marking the start of decoding. Default is 50257.
            use_cache (bool, optional): Whether to use caching during decoding. Default is True.
            is_encoder_decoder (bool, optional): Whether the model is an encoder-decoder architecture. Default is True.
            activation_function (str, optional): The activation function to use. Default is 'gelu'.
            d_model (int, optional): The model dimension. Default is 384.
            dropout (float, optional): The dropout probability. Default is 0.0.
            attention_dropout (float, optional): The dropout probability for attention layers. Default is 0.0.
            activation_dropout (float, optional): The dropout probability for activation layers. Default is 0.0.
            init_std (float, optional): The standard deviation for weight initialization. Default is 0.02.
            scale_embedding (bool, optional): Whether to scale embeddings. Default is False.
            max_source_positions (int, optional): The maximum number of source positions. Default is 1500.
            max_target_positions (int, optional): The maximum number of target positions. Default is 448.
            pad_token_id (int, optional): The token id for padding. Default is 50256.
            bos_token_id (int, optional): The token id for the beginning of sentence. Default is 50256.
            eos_token_id (int, optional): The token id for the end of sentence. Default is 50256.
            suppress_tokens (list, optional): Tokens to suppress during decoding. Default is None.
            begin_suppress_tokens (list, optional): Tokens to suppress at the beginning of decoding. Default is [220, 50256].
            use_weighted_layer_sum (bool, optional): Whether to use weighted layer sum. Default is False.
            classifier_proj_size (int, optional): The size of the classifier projection. Default is 256.
            apply_spec_augment (bool, optional): Whether to apply spectral augmentation. Default is False.
            mask_time_prob (float, optional): The probability of masking in the time domain. Default is 0.05.
            mask_time_length (int, optional): The length of time masks. Default is 10.
            mask_time_min_masks (int, optional): The minimum number of time masks. Default is 2.
            mask_feature_prob (float, optional): The probability of masking in the feature domain. Default is 0.0.
            mask_feature_length (int, optional): The length of feature masks. Default is 10.
            mask_feature_min_masks (int, optional): The minimum number of feature masks. Default is 0.
            median_filter_width (int, optional): The width of the median filter. Default is 7.
            **kwargs: Additional keyword arguments.

        Returns:
            None.

        Raises:
            None.
        """
        self.vocab_size = vocab_size
        self.num_mel_bins = num_mel_bins
        self.d_model = d_model
        self.encoder_layers = encoder_layers
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_layers = decoder_layers
        self.decoder_attention_heads = decoder_attention_heads
        self.decoder_ffn_dim = decoder_ffn_dim
        self.encoder_ffn_dim = encoder_ffn_dim
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.activation_function = activation_function
        self.init_std = init_std
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop
        self.use_cache = use_cache
        self.num_hidden_layers = encoder_layers
        self.scale_embedding = scale_embedding  # scale factor will be sqrt(d_model) if True
        self.max_source_positions = max_source_positions
        self.max_target_positions = max_target_positions

        # Audio Classification-specific parameters. Feel free to ignore for other classes.
        self.classifier_proj_size = classifier_proj_size
        self.use_weighted_layer_sum = use_weighted_layer_sum

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        self.apply_spec_augment = apply_spec_augment
        self.mask_time_prob = mask_time_prob
        self.mask_time_length = mask_time_length
        self.mask_time_min_masks = mask_time_min_masks
        self.mask_feature_prob = mask_feature_prob
        self.mask_feature_length = mask_feature_length
        self.mask_feature_min_masks = mask_feature_min_masks

        self.median_filter_width = median_filter_width

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            is_encoder_decoder=is_encoder_decoder,
            decoder_start_token_id=decoder_start_token_id,
            suppress_tokens=suppress_tokens,
            begin_suppress_tokens=begin_suppress_tokens,
            **kwargs,
        )

mindnlp.transformers.models.whisper.configuration_whisper.WhisperConfig.__init__(vocab_size=51865, num_mel_bins=80, encoder_layers=4, encoder_attention_heads=6, decoder_layers=4, decoder_attention_heads=6, decoder_ffn_dim=1536, encoder_ffn_dim=1536, encoder_layerdrop=0.0, decoder_layerdrop=0.0, decoder_start_token_id=50257, use_cache=True, is_encoder_decoder=True, activation_function='gelu', d_model=384, dropout=0.0, attention_dropout=0.0, activation_dropout=0.0, init_std=0.02, scale_embedding=False, max_source_positions=1500, max_target_positions=448, pad_token_id=50256, bos_token_id=50256, eos_token_id=50256, suppress_tokens=None, begin_suppress_tokens=[220, 50256], use_weighted_layer_sum=False, classifier_proj_size=256, apply_spec_augment=False, mask_time_prob=0.05, mask_time_length=10, mask_time_min_masks=2, mask_feature_prob=0.0, mask_feature_length=10, mask_feature_min_masks=0, median_filter_width=7, **kwargs)

Initialize a WhisperConfig object with the provided parameters.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
vocab_size	The size of the vocabulary. Default is 51865. TYPE: int DEFAULT: 51865
num_mel_bins	The number of Mel bins. Default is 80. TYPE: int DEFAULT: 80
encoder_layers	The number of layers in the encoder. Default is 4. TYPE: int DEFAULT: 4
encoder_attention_heads	The number of attention heads in the encoder. Default is 6. TYPE: int DEFAULT: 6
decoder_layers	The number of layers in the decoder. Default is 4. TYPE: int DEFAULT: 4
decoder_attention_heads	The number of attention heads in the decoder. Default is 6. TYPE: int DEFAULT: 6
decoder_ffn_dim	The dimension of the feed-forward network in the decoder. Default is 1536. TYPE: int DEFAULT: 1536
encoder_ffn_dim	The dimension of the feed-forward network in the encoder. Default is 1536. TYPE: int DEFAULT: 1536
encoder_layerdrop	The probability of dropping a layer in the encoder. Default is 0.0. TYPE: float DEFAULT: 0.0
decoder_layerdrop	The probability of dropping a layer in the decoder. Default is 0.0. TYPE: float DEFAULT: 0.0
decoder_start_token_id	The token id marking the start of decoding. Default is 50257. TYPE: int DEFAULT: 50257
use_cache	Whether to use caching during decoding. Default is True. TYPE: bool DEFAULT: True
is_encoder_decoder	Whether the model is an encoder-decoder architecture. Default is True. TYPE: bool DEFAULT: True
activation_function	The activation function to use. Default is 'gelu'. TYPE: str DEFAULT: 'gelu'
d_model	The model dimension. Default is 384. TYPE: int DEFAULT: 384
dropout	The dropout probability. Default is 0.0. TYPE: float DEFAULT: 0.0
attention_dropout	The dropout probability for attention layers. Default is 0.0. TYPE: float DEFAULT: 0.0
activation_dropout	The dropout probability for activation layers. Default is 0.0. TYPE: float DEFAULT: 0.0
init_std	The standard deviation for weight initialization. Default is 0.02. TYPE: float DEFAULT: 0.02
scale_embedding	Whether to scale embeddings. Default is False. TYPE: bool DEFAULT: False
max_source_positions	The maximum number of source positions. Default is 1500. TYPE: int DEFAULT: 1500
max_target_positions	The maximum number of target positions. Default is 448. TYPE: int DEFAULT: 448
pad_token_id	The token id for padding. Default is 50256. TYPE: int DEFAULT: 50256
bos_token_id	The token id for the beginning of sentence. Default is 50256. TYPE: int DEFAULT: 50256
eos_token_id	The token id for the end of sentence. Default is 50256. TYPE: int DEFAULT: 50256
suppress_tokens	Tokens to suppress during decoding. Default is None. TYPE: list DEFAULT: None
begin_suppress_tokens	Tokens to suppress at the beginning of decoding. Default is [220, 50256]. TYPE: list DEFAULT: [220, 50256]
use_weighted_layer_sum	Whether to use weighted layer sum. Default is False. TYPE: bool DEFAULT: False
classifier_proj_size	The size of the classifier projection. Default is 256. TYPE: int DEFAULT: 256
apply_spec_augment	Whether to apply spectral augmentation. Default is False. TYPE: bool DEFAULT: False
mask_time_prob	The probability of masking in the time domain. Default is 0.05. TYPE: float DEFAULT: 0.05
mask_time_length	The length of time masks. Default is 10. TYPE: int DEFAULT: 10
mask_time_min_masks	The minimum number of time masks. Default is 2. TYPE: int DEFAULT: 2
mask_feature_prob	The probability of masking in the feature domain. Default is 0.0. TYPE: float DEFAULT: 0.0
mask_feature_length	The length of feature masks. Default is 10. TYPE: int DEFAULT: 10
mask_feature_min_masks	The minimum number of feature masks. Default is 0. TYPE: int DEFAULT: 0
median_filter_width	The width of the median filter. Default is 7. TYPE: int DEFAULT: 7
**kwargs	Additional keyword arguments. DEFAULT: {}

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/configuration_whisper.py

def __init__(
    self,
    vocab_size=51865,
    num_mel_bins=80,
    encoder_layers=4,
    encoder_attention_heads=6,
    decoder_layers=4,
    decoder_attention_heads=6,
    decoder_ffn_dim=1536,
    encoder_ffn_dim=1536,
    encoder_layerdrop=0.0,
    decoder_layerdrop=0.0,
    decoder_start_token_id=50257,
    use_cache=True,
    is_encoder_decoder=True,
    activation_function="gelu",
    d_model=384,
    dropout=0.0,
    attention_dropout=0.0,
    activation_dropout=0.0,
    init_std=0.02,
    scale_embedding=False,
    max_source_positions=1500,
    max_target_positions=448,
    pad_token_id=50256,
    bos_token_id=50256,
    eos_token_id=50256,
    suppress_tokens=None,
    begin_suppress_tokens=[220, 50256],
    use_weighted_layer_sum=False,
    classifier_proj_size=256,
    apply_spec_augment=False,
    mask_time_prob=0.05,
    mask_time_length=10,
    mask_time_min_masks=2,
    mask_feature_prob=0.0,
    mask_feature_length=10,
    mask_feature_min_masks=0,
    median_filter_width=7,
    **kwargs,
):
    """
    Initialize a WhisperConfig object with the provided parameters.

    Args:
        self (object): The instance of the class.
        vocab_size (int, optional): The size of the vocabulary. Default is 51865.
        num_mel_bins (int, optional): The number of Mel bins. Default is 80.
        encoder_layers (int, optional): The number of layers in the encoder. Default is 4.
        encoder_attention_heads (int, optional): The number of attention heads in the encoder. Default is 6.
        decoder_layers (int, optional): The number of layers in the decoder. Default is 4.
        decoder_attention_heads (int, optional): The number of attention heads in the decoder. Default is 6.
        decoder_ffn_dim (int, optional): The dimension of the feed-forward network in the decoder. Default is 1536.
        encoder_ffn_dim (int, optional): The dimension of the feed-forward network in the encoder. Default is 1536.
        encoder_layerdrop (float, optional): The probability of dropping a layer in the encoder. Default is 0.0.
        decoder_layerdrop (float, optional): The probability of dropping a layer in the decoder. Default is 0.0.
        decoder_start_token_id (int, optional): The token id marking the start of decoding. Default is 50257.
        use_cache (bool, optional): Whether to use caching during decoding. Default is True.
        is_encoder_decoder (bool, optional): Whether the model is an encoder-decoder architecture. Default is True.
        activation_function (str, optional): The activation function to use. Default is 'gelu'.
        d_model (int, optional): The model dimension. Default is 384.
        dropout (float, optional): The dropout probability. Default is 0.0.
        attention_dropout (float, optional): The dropout probability for attention layers. Default is 0.0.
        activation_dropout (float, optional): The dropout probability for activation layers. Default is 0.0.
        init_std (float, optional): The standard deviation for weight initialization. Default is 0.02.
        scale_embedding (bool, optional): Whether to scale embeddings. Default is False.
        max_source_positions (int, optional): The maximum number of source positions. Default is 1500.
        max_target_positions (int, optional): The maximum number of target positions. Default is 448.
        pad_token_id (int, optional): The token id for padding. Default is 50256.
        bos_token_id (int, optional): The token id for the beginning of sentence. Default is 50256.
        eos_token_id (int, optional): The token id for the end of sentence. Default is 50256.
        suppress_tokens (list, optional): Tokens to suppress during decoding. Default is None.
        begin_suppress_tokens (list, optional): Tokens to suppress at the beginning of decoding. Default is [220, 50256].
        use_weighted_layer_sum (bool, optional): Whether to use weighted layer sum. Default is False.
        classifier_proj_size (int, optional): The size of the classifier projection. Default is 256.
        apply_spec_augment (bool, optional): Whether to apply spectral augmentation. Default is False.
        mask_time_prob (float, optional): The probability of masking in the time domain. Default is 0.05.
        mask_time_length (int, optional): The length of time masks. Default is 10.
        mask_time_min_masks (int, optional): The minimum number of time masks. Default is 2.
        mask_feature_prob (float, optional): The probability of masking in the feature domain. Default is 0.0.
        mask_feature_length (int, optional): The length of feature masks. Default is 10.
        mask_feature_min_masks (int, optional): The minimum number of feature masks. Default is 0.
        median_filter_width (int, optional): The width of the median filter. Default is 7.
        **kwargs: Additional keyword arguments.

    Returns:
        None.

    Raises:
        None.
    """
    self.vocab_size = vocab_size
    self.num_mel_bins = num_mel_bins
    self.d_model = d_model
    self.encoder_layers = encoder_layers
    self.encoder_attention_heads = encoder_attention_heads
    self.decoder_layers = decoder_layers
    self.decoder_attention_heads = decoder_attention_heads
    self.decoder_ffn_dim = decoder_ffn_dim
    self.encoder_ffn_dim = encoder_ffn_dim
    self.dropout = dropout
    self.attention_dropout = attention_dropout
    self.activation_dropout = activation_dropout
    self.activation_function = activation_function
    self.init_std = init_std
    self.encoder_layerdrop = encoder_layerdrop
    self.decoder_layerdrop = decoder_layerdrop
    self.use_cache = use_cache
    self.num_hidden_layers = encoder_layers
    self.scale_embedding = scale_embedding  # scale factor will be sqrt(d_model) if True
    self.max_source_positions = max_source_positions
    self.max_target_positions = max_target_positions

    # Audio Classification-specific parameters. Feel free to ignore for other classes.
    self.classifier_proj_size = classifier_proj_size
    self.use_weighted_layer_sum = use_weighted_layer_sum

    # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
    self.apply_spec_augment = apply_spec_augment
    self.mask_time_prob = mask_time_prob
    self.mask_time_length = mask_time_length
    self.mask_time_min_masks = mask_time_min_masks
    self.mask_feature_prob = mask_feature_prob
    self.mask_feature_length = mask_feature_length
    self.mask_feature_min_masks = mask_feature_min_masks

    self.median_filter_width = median_filter_width

    super().__init__(
        pad_token_id=pad_token_id,
        bos_token_id=bos_token_id,
        eos_token_id=eos_token_id,
        is_encoder_decoder=is_encoder_decoder,
        decoder_start_token_id=decoder_start_token_id,
        suppress_tokens=suppress_tokens,
        begin_suppress_tokens=begin_suppress_tokens,
        **kwargs,
    )

mindnlp.transformers.models.whisper.feature_extraction_whisper

Feature extractor class for Whisper

mindnlp.transformers.models.whisper.feature_extraction_whisper.WhisperFeatureExtractor

Bases: SequenceFeatureExtractor

Constructs a Whisper feature extractor.

This feature extractor inherits from [~feature_extraction_sequence_utils.SequenceFeatureExtractor] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the Short Time Fourier Transform which should match pytorch's torch.stft equivalent.

PARAMETER	DESCRIPTION
feature_size	The feature dimension of the extracted features. TYPE: `int`, defaults to 80 DEFAULT: 80
sampling_rate	The sampling rate at which the audio files should be digitalized expressed in hertz (Hz). TYPE: `int`, defaults to 16000 DEFAULT: 16000
hop_length	Length of the overlaping windows for the STFT used to obtain the Mel Frequency coefficients. TYPE: `int`, defaults to 160 DEFAULT: 160
chunk_length	The maximum number of chuncks of sampling_rate samples used to trim and pad longer or shorter audio sequences. TYPE: `int`, defaults to 30 DEFAULT: 30
n_fft	Size of the Fourier transform. TYPE: `int`, defaults to 400 DEFAULT: 400
padding_value	Padding value used to pad the audio. Should correspond to silences. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0

Source code in mindnlp/transformers/models/whisper/feature_extraction_whisper.py

class WhisperFeatureExtractor(SequenceFeatureExtractor):
    r"""
    Constructs a Whisper feature extractor.

    This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
    most of the main methods. Users should refer to this superclass for more information regarding those methods.

    This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time
    Fourier Transform` which should match pytorch's `torch.stft` equivalent.

    Args:
        feature_size (`int`, defaults to 80):
            The feature dimension of the extracted features.
        sampling_rate (`int`, defaults to 16000):
            The sampling rate at which the audio files should be digitalized expressed in hertz (Hz).
        hop_length (`int`, defaults to 160):
            Length of the overlaping windows for the STFT used to obtain the Mel Frequency coefficients.
        chunk_length (`int`, defaults to 30):
            The maximum number of chuncks of `sampling_rate` samples used to trim and pad longer or shorter audio
            sequences.
        n_fft (`int`, defaults to 400):
            Size of the Fourier transform.
        padding_value (`float`, *optional*, defaults to 0.0):
            Padding value used to pad the audio. Should correspond to silences.
    """
    model_input_names = ["input_features"]

    def __init__(
        self,
        feature_size=80,
        sampling_rate=16000,
        hop_length=160,
        chunk_length=30,
        n_fft=400,
        padding_value=0.0,
        return_attention_mask=False,  # pad inputs to max length with silence token (zero) and no attention mask
        **kwargs,
    ):
        """
        Initializes a WhisperFeatureExtractor object.

        Args:
            self: The instance of the class.
            feature_size (int): The size of the feature vector.
            sampling_rate (int): The sampling rate of the audio signal.
            hop_length (int): The hop length for the short-time Fourier transform.
            chunk_length (int): The length of each audio chunk in seconds.
            n_fft (int): The number of FFT points.
            padding_value (float): The value used for padding.
            return_attention_mask (bool): Flag indicating whether to return an attention mask.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(
            feature_size=feature_size,
            sampling_rate=sampling_rate,
            padding_value=padding_value,
            return_attention_mask=return_attention_mask,
            **kwargs,
        )
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.chunk_length = chunk_length
        self.n_samples = chunk_length * sampling_rate
        self.nb_max_frames = self.n_samples // hop_length
        self.sampling_rate = sampling_rate
        self.mel_filters = mel_filter_bank(
            num_frequency_bins=1 + n_fft // 2,
            num_mel_filters=feature_size,
            min_frequency=0.0,
            max_frequency=8000.0,
            sampling_rate=sampling_rate,
            norm="slaney",
            mel_scale="slaney",
        )

    def _np_extract_fbank_features(self, waveform: np.array) -> np.ndarray:
        """
        Compute the log-mel spectrogram of the provided audio, gives similar results to Whisper's original torch
        implementation with 1e-5 tolerance.
        """
        log_spec = spectrogram(
            waveform,
            window_function(self.n_fft, "hann"),
            frame_length=self.n_fft,
            hop_length=self.hop_length,
            power=2.0,
            mel_filters=self.mel_filters,
            log_mel="log10",
        )
        log_spec = log_spec[:, :-1]
        log_spec = np.maximum(log_spec, log_spec.max() - 8.0)
        log_spec = (log_spec + 4.0) / 4.0
        return log_spec

    @staticmethod
    # Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
    def zero_mean_unit_var_norm(
        input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
    ) -> List[np.ndarray]:
        """
        Every array in the list is normalized to have zero mean and unit variance
        """
        if attention_mask is not None:
            attention_mask = np.array(attention_mask, np.int32)
            normed_input_values = []

            for vector, length in zip(input_values, attention_mask.sum(-1)):
                normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
                if length < normed_slice.shape[0]:
                    normed_slice[length:] = padding_value

                normed_input_values.append(normed_slice)
        else:
            normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]

        return normed_input_values

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        truncation: bool = True,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_attention_mask: Optional[bool] = None,
        padding: Optional[str] = "max_length",
        max_length: Optional[int] = None,
        sampling_rate: Optional[int] = None,
        do_normalize: Optional[bool] = None,
        **kwargs,
    ) -> BatchFeature:
        """
        Main method to featurize and prepare for the model one or several sequence(s).

        Args:
            raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`):
                The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
                values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
                stereo, i.e. single float per timestep.
            truncation (`bool`, *optional*, default to `True`):
                Activates truncation to cut input sequences longer than *max_length* to *max_length*.
            pad_to_multiple_of (`int`, *optional*, defaults to None):
                If set will pad the sequence to a multiple of the provided value.

                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
                `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
            return_attention_mask (`bool`, *optional*):
                Whether to return the attention mask. If left to the default, will return the attention mask according
                to the specific feature_extractor's default.

                [What are attention masks?](../glossary#attention-mask)

                <Tip>

                For Whisper models, `attention_mask` should always be passed for batched inference, to avoid subtle
                bugs.

                </Tip>

            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
            sampling_rate (`int`, *optional*):
                The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
                `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
                pipeline.
            padding_value (`float`, defaults to 0.0):
                The value that is used to fill the padding values / vectors.
            do_normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
                improve the performance of the model.
        """
        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
                    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
                    f" was sampled with {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        if is_batched_numpy and len(raw_speech.shape) > 2:
            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
            raw_speech = raw_speech.astype(np.float32)

        # always return batch
        if not is_batched:
            raw_speech = [np.asarray([raw_speech]).T]

        batched_speech = BatchFeature({"input_features": raw_speech})

        # convert into correct format for padding

        padded_inputs = self.pad(
            batched_speech,
            padding=padding,
            max_length=max_length if max_length else self.n_samples,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask or do_normalize,
        )

        # zero-mean and unit-variance normalization
        if do_normalize:
            padded_inputs["input_features"] = self.zero_mean_unit_var_norm(
                padded_inputs["input_features"],
                attention_mask=padded_inputs["attention_mask"],
                padding_value=self.padding_value,
            )
            padded_inputs["input_features"] = np.stack(padded_inputs["input_features"], axis=0)

        # make sure list is in array format
        input_features = padded_inputs.get("input_features").transpose(2, 0, 1)

        input_features = [self._np_extract_fbank_features(waveform) for waveform in input_features[0]]

        if isinstance(input_features[0], List):
            padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features]
        else:
            padded_inputs["input_features"] = input_features

        if return_attention_mask:
            # rescale from sample (48000) to feature (3000)
            padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length]

        if return_tensors is not None:
            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

        return padded_inputs

mindnlp.transformers.models.whisper.feature_extraction_whisper.WhisperFeatureExtractor.__call__(raw_speech, truncation=True, pad_to_multiple_of=None, return_tensors=None, return_attention_mask=None, padding='max_length', max_length=None, sampling_rate=None, do_normalize=None, **kwargs)

Main method to featurize and prepare for the model one or several sequence(s).

PARAMETER	DESCRIPTION
raw_speech	The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not stereo, i.e. single float per timestep. TYPE: `np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`
truncation	Activates truncation to cut input sequences longer than max_length to max_length. TYPE: `bool`, *optional*, default to `True` DEFAULT: True
pad_to_multiple_of	If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. TYPE: `int`, *optional*, defaults to None DEFAULT: None
return_attention_mask	Whether to return the attention mask. If left to the default, will return the attention mask according to the specific feature_extractor's default. What are attention masks? For Whisper models, attention_mask should always be passed for batched inference, to avoid subtle bugs. TYPE: `bool`, *optional* DEFAULT: None
return_tensors	If set, will return tensors instead of list of python integers. Acceptable values are: 'tf': Return TensorFlow tf.constant objects. 'pt': Return PyTorch torch.Tensor objects. 'np': Return Numpy np.ndarray objects. TYPE: `str` or [`~utils.TensorType`], *optional* DEFAULT: None
sampling_rate	The sampling rate at which the raw_speech input was sampled. It is strongly recommended to pass sampling_rate at the forward call to prevent silent errors and allow automatic speech recognition pipeline. TYPE: `int`, *optional* DEFAULT: None
padding_value	The value that is used to fill the padding values / vectors. TYPE: `float`, defaults to 0.0
do_normalize	Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly improve the performance of the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: None

Source code in mindnlp/transformers/models/whisper/feature_extraction_whisper.py

def __call__(
    self,
    raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
    truncation: bool = True,
    pad_to_multiple_of: Optional[int] = None,
    return_tensors: Optional[Union[str, TensorType]] = None,
    return_attention_mask: Optional[bool] = None,
    padding: Optional[str] = "max_length",
    max_length: Optional[int] = None,
    sampling_rate: Optional[int] = None,
    do_normalize: Optional[bool] = None,
    **kwargs,
) -> BatchFeature:
    """
    Main method to featurize and prepare for the model one or several sequence(s).

    Args:
        raw_speech (`np.ndarray`, `List[float]`, `List[np.ndarray]`, `List[List[float]]`):
            The sequence or batch of sequences to be padded. Each sequence can be a numpy array, a list of float
            values, a list of numpy arrays or a list of list of float values. Must be mono channel audio, not
            stereo, i.e. single float per timestep.
        truncation (`bool`, *optional*, default to `True`):
            Activates truncation to cut input sequences longer than *max_length* to *max_length*.
        pad_to_multiple_of (`int`, *optional*, defaults to None):
            If set will pad the sequence to a multiple of the provided value.

            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
            `>= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
        return_attention_mask (`bool`, *optional*):
            Whether to return the attention mask. If left to the default, will return the attention mask according
            to the specific feature_extractor's default.

            [What are attention masks?](../glossary#attention-mask)

            <Tip>

            For Whisper models, `attention_mask` should always be passed for batched inference, to avoid subtle
            bugs.

            </Tip>

        return_tensors (`str` or [`~utils.TensorType`], *optional*):
            If set, will return tensors instead of list of python integers. Acceptable values are:

            - `'tf'`: Return TensorFlow `tf.constant` objects.
            - `'pt'`: Return PyTorch `torch.Tensor` objects.
            - `'np'`: Return Numpy `np.ndarray` objects.
        sampling_rate (`int`, *optional*):
            The sampling rate at which the `raw_speech` input was sampled. It is strongly recommended to pass
            `sampling_rate` at the forward call to prevent silent errors and allow automatic speech recognition
            pipeline.
        padding_value (`float`, defaults to 0.0):
            The value that is used to fill the padding values / vectors.
        do_normalize (`bool`, *optional*, defaults to `False`):
            Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
            improve the performance of the model.
    """
    if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
            raise ValueError(
                f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
                f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
                f" was sampled with {self.sampling_rate} and not {sampling_rate}."
            )
    else:
        logger.warning(
            "It is strongly recommended to pass the `sampling_rate` argument to this function. "
            "Failing to do so can result in silent errors that might be hard to debug."
        )

    is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
    if is_batched_numpy and len(raw_speech.shape) > 2:
        raise ValueError(f"Only mono-channel audio is supported for input to {self}")
    is_batched = is_batched_numpy or (
        isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
    )

    if is_batched:
        raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
    elif not is_batched and not isinstance(raw_speech, np.ndarray):
        raw_speech = np.asarray(raw_speech, dtype=np.float32)
    elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
        raw_speech = raw_speech.astype(np.float32)

    # always return batch
    if not is_batched:
        raw_speech = [np.asarray([raw_speech]).T]

    batched_speech = BatchFeature({"input_features": raw_speech})

    # convert into correct format for padding

    padded_inputs = self.pad(
        batched_speech,
        padding=padding,
        max_length=max_length if max_length else self.n_samples,
        truncation=truncation,
        pad_to_multiple_of=pad_to_multiple_of,
        return_attention_mask=return_attention_mask or do_normalize,
    )

    # zero-mean and unit-variance normalization
    if do_normalize:
        padded_inputs["input_features"] = self.zero_mean_unit_var_norm(
            padded_inputs["input_features"],
            attention_mask=padded_inputs["attention_mask"],
            padding_value=self.padding_value,
        )
        padded_inputs["input_features"] = np.stack(padded_inputs["input_features"], axis=0)

    # make sure list is in array format
    input_features = padded_inputs.get("input_features").transpose(2, 0, 1)

    input_features = [self._np_extract_fbank_features(waveform) for waveform in input_features[0]]

    if isinstance(input_features[0], List):
        padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features]
    else:
        padded_inputs["input_features"] = input_features

    if return_attention_mask:
        # rescale from sample (48000) to feature (3000)
        padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length]

    if return_tensors is not None:
        padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

    return padded_inputs

mindnlp.transformers.models.whisper.feature_extraction_whisper.WhisperFeatureExtractor.__init__(feature_size=80, sampling_rate=16000, hop_length=160, chunk_length=30, n_fft=400, padding_value=0.0, return_attention_mask=False, **kwargs)

Initializes a WhisperFeatureExtractor object.

PARAMETER	DESCRIPTION
self	The instance of the class.
feature_size	The size of the feature vector. TYPE: int DEFAULT: 80
sampling_rate	The sampling rate of the audio signal. TYPE: int DEFAULT: 16000
hop_length	The hop length for the short-time Fourier transform. TYPE: int DEFAULT: 160
chunk_length	The length of each audio chunk in seconds. TYPE: int DEFAULT: 30
n_fft	The number of FFT points. TYPE: int DEFAULT: 400
padding_value	The value used for padding. TYPE: float DEFAULT: 0.0
return_attention_mask	Flag indicating whether to return an attention mask. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/feature_extraction_whisper.py

def __init__(
    self,
    feature_size=80,
    sampling_rate=16000,
    hop_length=160,
    chunk_length=30,
    n_fft=400,
    padding_value=0.0,
    return_attention_mask=False,  # pad inputs to max length with silence token (zero) and no attention mask
    **kwargs,
):
    """
    Initializes a WhisperFeatureExtractor object.

    Args:
        self: The instance of the class.
        feature_size (int): The size of the feature vector.
        sampling_rate (int): The sampling rate of the audio signal.
        hop_length (int): The hop length for the short-time Fourier transform.
        chunk_length (int): The length of each audio chunk in seconds.
        n_fft (int): The number of FFT points.
        padding_value (float): The value used for padding.
        return_attention_mask (bool): Flag indicating whether to return an attention mask.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(
        feature_size=feature_size,
        sampling_rate=sampling_rate,
        padding_value=padding_value,
        return_attention_mask=return_attention_mask,
        **kwargs,
    )
    self.n_fft = n_fft
    self.hop_length = hop_length
    self.chunk_length = chunk_length
    self.n_samples = chunk_length * sampling_rate
    self.nb_max_frames = self.n_samples // hop_length
    self.sampling_rate = sampling_rate
    self.mel_filters = mel_filter_bank(
        num_frequency_bins=1 + n_fft // 2,
        num_mel_filters=feature_size,
        min_frequency=0.0,
        max_frequency=8000.0,
        sampling_rate=sampling_rate,
        norm="slaney",
        mel_scale="slaney",
    )

mindnlp.transformers.models.whisper.feature_extraction_whisper.WhisperFeatureExtractor.zero_mean_unit_var_norm(input_values, attention_mask, padding_value=0.0) staticmethod

Every array in the list is normalized to have zero mean and unit variance

Source code in mindnlp/transformers/models/whisper/feature_extraction_whisper.py

@staticmethod
# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
def zero_mean_unit_var_norm(
    input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
) -> List[np.ndarray]:
    """
    Every array in the list is normalized to have zero mean and unit variance
    """
    if attention_mask is not None:
        attention_mask = np.array(attention_mask, np.int32)
        normed_input_values = []

        for vector, length in zip(input_values, attention_mask.sum(-1)):
            normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
            if length < normed_slice.shape[0]:
                normed_slice[length:] = padding_value

            normed_input_values.append(normed_slice)
    else:
        normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]

    return normed_input_values

mindnlp.transformers.models.whisper.tokenization_whisper_fast

Tokenization classes for Whisper.

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast

Bases: PreTrainedTokenizerFast

Construct a "fast" Whisper tokenizer (backed by HuggingFace's tokenizers library).

This tokenizer inherits from [PreTrainedTokenizerFast] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`, *optional* DEFAULT: None
merges_file	Path to the merges file. TYPE: `str`, *optional* DEFAULT: None
normalizer_file	Path to the normalizer_file file. TYPE: `str`, *optional* DEFAULT: None
tokenizer_file	Path to tokenizers file (generally has a .json extension) that contains everything needed to load the tokenizer. TYPE: `str`, *optional* DEFAULT: None
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
bos_token	The beginning of sequence token. The decoder_start_token_id is used to set the first token as "<|startoftranscript|>" when generating. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
eos_token	The end of sequence token. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
add_prefix_space	Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. (Whisper tokenizer detect beginning of words by the preceding space). TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
language	The language of the transcription text. The corresponding language id token is appended to the start of the sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token "<|es|>" is appended to the start of sequence. This should be used for multilingual fine-tuning only. TYPE: `str`, *optional* DEFAULT: None
task	Task identifier to append at the start of sequence (if any). This should be used for mulitlingual fine-tuning, with "transcribe" for speech recognition and "translate" for speech translation. TYPE: `str`, *optional* DEFAULT: None
predict_timestamps	Whether to omit the <|notimestamps|> token at the start of the sequence. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

class WhisperTokenizerFast(PreTrainedTokenizerFast):
    """
    Construct a "fast" Whisper tokenizer (backed by HuggingFace's *tokenizers* library).

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        merges_file (`str`, *optional*):
            Path to the merges file.
        normalizer_file (`str`, *optional*):
            Path to the normalizer_file file.
        tokenizer_file (`str`, *optional*):
            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
            contains everything needed to load the tokenizer.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The beginning of sequence token. The `decoder_start_token_id` is used to set the first token as
            `"<|startoftranscript|>"` when generating.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        add_prefix_space (`bool`, *optional*, defaults to `False`):
            Whether or not to add an initial space to the input. This allows to treat the leading word just as any
            other word. (Whisper tokenizer detect beginning of words by the preceding space).
        language (`str`, *optional*):
            The language of the transcription text. The corresponding language id token is appended to the start of the
            sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token
            `"<|es|>"` is appended to the start of sequence. This should be used for multilingual fine-tuning only.
        task (`str`, *optional*):
            Task identifier to append at the start of sequence (if any). This should be used for mulitlingual
            fine-tuning, with `"transcribe"` for speech recognition and `"translate"` for speech translation.
        predict_timestamps (`bool`, *optional*, defaults to `False`):
            Whether to omit the `<|notimestamps|>` token at the start of the sequence.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = WhisperTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        normalizer_file=None,
        tokenizer_file=None,
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        add_prefix_space=False,
        language=None,
        task=None,
        predict_timestamps=False,
        **kwargs,
    ):
        """
        This method initializes an instance of the WhisperTokenizerFast class.

        Args:
            self: The instance of the class.
            vocab_file (str): The file path to the vocabulary file.
            merges_file (str): The file path to the merges file.
            normalizer_file (str): The file path to the normalizer file.
            tokenizer_file (str): The file path to the tokenizer file.
            unk_token (str): The unknown token. Default is 'endoftext'.
            bos_token (str): The beginning of sentence token. Default is 'endoftext'.
            eos_token (str): The end of sentence token. Default is 'endoftext'.
            add_prefix_space (bool): Indicates whether to add prefix space. Default is False.
            language (str): The language used.
            task (str): The task for the tokenizer.
            predict_timestamps (bool): Indicates whether timestamps should be predicted.

        Returns:
            None

        Raises:
            TypeError: If any of the parameters are of incorrect type.
            FileNotFoundError: If any of the specified files cannot be found.
            JSONDecodeError: If there is an issue with decoding JSON data.
            AttributeError: If there is an attribute error while setting tokenizer pre-tokenizer.
            ValueError: If there is an issue with the provided values.
            RegexError: If there is an issue with the regular expression compilation.
        """
        bos_token = (
            AddedToken(bos_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(bos_token, str)
            else bos_token
        )
        eos_token = (
            AddedToken(eos_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(eos_token, str)
            else eos_token
        )
        unk_token = (
            AddedToken(unk_token, lstrip=False, rstrip=False, normalized=False, special=True)
            if isinstance(unk_token, str)
            else unk_token
        )

        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )

        self.add_bos_token = kwargs.pop("add_bos_token", False)

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        if normalizer_file is not None:
            with open(normalizer_file, encoding="utf-8") as vocab_handle:
                self.english_spelling_normalizer = json.load(vocab_handle)
        else:
            self.english_spelling_normalizer = None

        self.add_prefix_space = add_prefix_space
        self.timestamp_pat = re.compile(r"<\|(\d+\.\d+)\|>")

        self.language = language
        self.task = task
        self.predict_timestamps = predict_timestamps

    # Copied from transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast._batch_encode_plus
    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        """
        Method _batch_encode_plus in the class WhisperTokenizerFast.

        Args:
            self (WhisperTokenizerFast): The instance of the WhisperTokenizerFast class.

        Returns:
            BatchEncoding: An object of type BatchEncoding containing the encoded inputs.

        Raises:
            AssertionError: If add_prefix_space is False and is_split_into_words is True,
                it raises an assertion error with the message indicating that the class needs to be instantiated with
                add_prefix_space=True to use it with pretokenized inputs.
        """
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    # Copied from transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast._encode_plus
    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        """
        Encode the input sequence into a batch encoding using the WhisperTokenizerFast class.

        Args:
            self (WhisperTokenizerFast): The instance of the WhisperTokenizerFast class.

        Returns:
            BatchEncoding: A batch encoding object containing the encoded input sequence.

        Raises:
            AssertionError: If the 'add_prefix_space' attribute is not set to True and the input is split into words.
                This indicates that the WhisperTokenizerFast instance needs to be instantiated with
                add_prefix_space=True for use with pretokenized inputs.
        """
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._decode_with_timestamps
    def _decode_with_timestamps(self, token_ids, skip_special_tokens=False, time_precision=0.02) -> str:
        """
        Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. This method decodes
        given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
        """
        timestamp_begin = self.all_special_ids[-1] + 1
        outputs = [[]]

        cur_max_timestamp = 0.0
        prev_segments_len = 0.0

        for token in token_ids:
            if token >= timestamp_begin:
                timestamp = float((token - timestamp_begin) * time_precision)

                if timestamp < cur_max_timestamp:
                    # next segment has started
                    prev_segments_len += cur_max_timestamp

                cur_max_timestamp = timestamp

                outputs.append(f"<|{(timestamp + prev_segments_len):.2f}|>")
                outputs.append([])
            else:
                outputs[-1].append(token)
        outputs = [
            s if isinstance(s, str) else self.decode(s, skip_special_tokens=skip_special_tokens) for s in outputs
        ]
        return "".join(outputs)

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._compute_offsets
    def _compute_offsets(self, token_ids, time_precision=0.02):
        """
        Compute offsets for a given tokenized input

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            time_precision (`float`, `optional`, defaults to 0.02):
                The time ratio to convert from token to time.
        """
        offsets = []
        # ensure torch tensor of token ids is placed on cpu
        if "torch" in str(type(token_ids)) and (hasattr(token_ids, "cpu") and callable(token_ids.cpu)):
            token_ids = token_ids.cpu()
        token_ids = np.array(token_ids)
        if token_ids.shape[0] > 1 and len(token_ids.shape) > 1:
            raise ValueError("Can only process a single input at a time")
        timestamp_begin = self.all_special_ids[-1] + 1
        timestamp_tokens = token_ids >= timestamp_begin

        consecutive = np.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0] + 1
        if consecutive.shape[0] == 0 and timestamp_tokens.sum() <= 1:
            # either there are no timestamps or there are no consecutive ones
            return []
        if np.where(timestamp_tokens)[0][-1] + 1 not in consecutive:
            # we add the final timestamp if it is not already in the list
            consecutive = np.append(consecutive, np.where(timestamp_tokens)[0][-1] + 1)

        last_slice = np.where(timestamp_tokens)[0][0]
        for current_slice in consecutive:
            sliced_tokens = token_ids[last_slice:current_slice]
            if len(sliced_tokens) > 1:
                start_timestamp_position = sliced_tokens[0].item() - timestamp_begin
                end_timestamp_position = sliced_tokens[-1].item() - timestamp_begin
                # strip timestamp tokens from the text output
                sliced_tokens = self._preprocess_token_ids(sliced_tokens)
                text = self._decode(sliced_tokens)
                text = self._filter_timestamp_ids(text)
                offsets.append(
                    {
                        "text": text,
                        "timestamp": (
                            start_timestamp_position * time_precision,
                            end_timestamp_position * time_precision,
                        ),
                    }
                )
            last_slice = current_slice

        return offsets

    @lru_cache(128)
    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.timestamp_ids
    def timestamp_ids(self, time_precision=0.02):
        """
        Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

        Args:
            time_precision (`float`, `optional`, defaults to 0.02):
                The time ratio to convert from token to time.
        """
        return self.convert_tokens_to_ids([("<|%.2f|>" % (i * time_precision)) for i in range(1500 + 1)])

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._preprocess_token_ids
    def _preprocess_token_ids(self, token_ids, skip_special_tokens: bool = False):
        """
        Pre-process the token ids for decoding by removing the prompt tokens ids and timestamp token ids.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Typically, obtained using the `__call__` method of the tokenizer.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens from the token ids. If `True`, the prompt token ids will be
                removed.
        """
        if skip_special_tokens:
            prompt_token_id = self.convert_tokens_to_ids("<|startofprev|>")
            decoder_start_token_id = self.convert_tokens_to_ids("<|startoftranscript|>")
            token_ids = self._strip_prompt(token_ids, prompt_token_id, decoder_start_token_id)

        return token_ids

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._filter_timestamp_ids
    def _filter_timestamp_ids(self, token_ids):
        """
        Filters out timestamp IDs from a given list of token IDs.

        Args:
            self (WhisperTokenizerFast): An instance of the WhisperTokenizerFast class.
            token_ids (str): A string containing the token IDs.

        Returns:
            None: This method modifies the 'token_ids' string in-place by removing any timestamp IDs.

        Raises:
            None.
        """
        return re.sub(self.timestamp_pat, "", token_ids)

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.decode
    def decode(
        self,
        token_ids,
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        output_offsets: bool = False,
        time_precision: float = 0.02,
        decode_with_timestamps: bool = False,
        normalize: bool = False,
        basic_normalize: bool = False,
        remove_diacritics: bool = False,
        **kwargs,
    ) -> str:
        """
        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
        tokens and clean up tokenization spaces.

        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding.
            clean_up_tokenization_spaces (`bool`, *optional*):
                Whether or not to clean up the tokenization spaces. If `None`, will default to
                `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
            output_offsets (`bool`, *optional*, defaults to `False`):
                Whether or not to output the offsets of the tokens. This should only be set if the model predicted
                timestamps.
            time_precision (`float`, `optional`, defaults to 0.02):
                The time ratio to convert from token to time.
            decode_with_timestamps (`bool`, *optional*, defaults to `False`):
                Whether or not to decode with timestamps included in the raw text.
            normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to apply the English text normalizer to the decoded text. Only applicable when the
                target text is in English. Otherwise, the basic text normalizer should be applied.
            basic_normalize (`bool`, *optional*, defaults to `False`):
                Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual
                target text.
            remove_diacritics (`bool`, *optional*, defaults to `False`):
                Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may
                destroy information in the decoded text, hence it should be used with caution.
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the underlying model specific decode method.

        Returns:
            `str`: The decoded sentence.
        """
        filtered_ids = self._preprocess_token_ids(
            token_ids,
            skip_special_tokens=skip_special_tokens,
        )

        text = super().decode(
            filtered_ids,
            skip_special_tokens=skip_special_tokens,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            normalize=normalize,
            basic_normalize=basic_normalize,
            remove_diacritics=remove_diacritics,
            **kwargs,
        )
        if decode_with_timestamps:
            # legacy method to decode timestamps when not included in the tokenizer vocabulary
            text = self._decode_with_timestamps(
                filtered_ids, time_precision=time_precision, skip_special_tokens=skip_special_tokens
            )
        else:
            text = self._filter_timestamp_ids(text)

        # retrieve offsets
        if output_offsets:
            offsets = self._compute_offsets(token_ids, time_precision=time_precision)
            return {"text": text, "offsets": offsets}
        return text

    def _decode(
        self, *args, normalize: bool = False, basic_normalize: bool = False, remove_diacritics: bool = False, **kwargs
    ) -> str:
        """
        Decodes the text passed as input with optional normalization and diacritics removal.

        Args:
            self (WhisperTokenizerFast): The instance of the WhisperTokenizerFast class.
            *args: Variable length argument list.
            normalize (bool): Flag to enable full normalization of the text.
                Defaults to False.
            basic_normalize (bool): Flag to enable basic normalization of the text.
                Defaults to False.
            remove_diacritics (bool): Flag to remove diacritics from the text.
                Defaults to False.
            **kwargs: Additional keyword arguments.

        Returns:
            str: The decoded and optionally normalized text.

        Raises:
            None.
        """
        text = super()._decode(*args, **kwargs)

        if normalize:
            clean_text = self._normalize(text)
            return clean_text
        if basic_normalize:
            clean_text = self._basic_normalize(text, remove_diacritics=remove_diacritics)
            return clean_text
        return text

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._normalize
    def _normalize(self, text):
        """
        Normalize the input text using the Whisper English normalizer.

        Args:
            self (WhisperTokenizerFast): An instance of the WhisperTokenizerFast class.
            text (str): The input text to be normalized.

        Returns:
            None.

        Raises:
            DeprecationWarning: If the method is called, a DeprecationWarning will be raised to notify that the
                private method `_normalize` is deprecated and will be removed in v5 of Transformers. Users are
                encouraged to use the `normalize` method to normalize the input string.

        Note:
            The `_normalize` method is a private method intended for internal use only. Its functionality will be
            replaced by the `normalize` method in future versions of Transformers.

        Example:
            ```python
            >>> tokenizer = WhisperTokenizerFast()
            >>> text = "Hello, World!"
            >>> tokenizer._normalize(text)
            .. warning::
                The `_normalize` method is deprecated and will be removed in v5 of Transformers.
                You can normalize an input string using the Whisper English normalizer using the `normalize` method.
            ```
        """
        warnings.warn(
            "The private method `_normalize` is deprecated and will be removed in v5 of Transformers."
            "You can normalize an input string using the Whisper English normalizer using the `normalize` method."
        )
        return self.normalize(text)

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._basic_normalize
    def _basic_normalize(self, text, remove_diacritics=False):
        """
        This method '_basic_normalize' in the class 'WhisperTokenizerFast' is deprecated and will be removed in v5 of
        Transformers. It is recommended to use the 'basic_normalize' method for string normalization.

        Args:
            self: Instance of the WhisperTokenizerFast class.
            text (str): Input text to be normalized.
            remove_diacritics (bool): Flag indicating whether diacritics should be removed during normalization.
                Defaults to False.

        Returns:
            None.

        Raises:
            DeprecationWarning: If the method '_basic_normalize' is called, a DeprecationWarning is raised indicating
                that the method is deprecated and will be removed in v5 of Transformers.
        """
        warnings.warn(
            "The private method `_basic_normalize` is deprecated and will be removed in v5 of Transformers."
            "You can normalize an input string using the Whisper basic normalizer using the `basic_normalize` method."
        )
        return self.basic_normalize(text, remove_diacritics=remove_diacritics)

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.normalize
    def normalize(self, text):
        """
        Normalize a given string using the `EnglishTextNormalizer` class, which preforms commons transformation on
        english text.
        """
        normalizer = EnglishTextNormalizer(self.english_spelling_normalizer)
        return normalizer(text)

    @staticmethod
    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.basic_normalize
    def basic_normalize(text, remove_diacritics=False):
        """
        Normalize a given string using the `BasicTextNormalizer` class, which preforms commons transformation on
        multilingual text.
        """
        normalizer = BasicTextNormalizer(remove_diacritics=remove_diacritics)
        return normalizer(text)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary to the specified directory.

        Args:
            self: Instance of the WhisperTokenizerFast class.
            save_directory (str): The directory where the vocabulary files will be saved.
            filename_prefix (Optional[str]): An optional prefix to be included in the filename. Default is None.

        Returns:
            Tuple[str]: A tuple containing the paths of the saved files and the path of the normalizer file.

        Raises:
            OSError: If an error occurs while saving the vocabulary files to the specified directory.
            ValueError: If the save_directory is invalid or inaccessible.
            TypeError: If the provided filename_prefix is not a string.
        """
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)

        normalizer_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["normalizer_file"]
        )

        if self.english_spelling_normalizer is not None:
            with open(normalizer_file, "w", encoding="utf-8") as f:
                f.write(
                    json.dumps(self.english_spelling_normalizer, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
                )

        return tuple(files) + (normalizer_file,)

    def set_prefix_tokens(self, language: str = None, task: str = None, predict_timestamps: bool = None):
        """
        Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to
        update the prefix tokens as required when fine-tuning.

        Example:
            ```python
            >>> # instantiate the tokenizer and set the prefix token to Spanish
            >>> tokenizer = WhisperTokenizerFast.from_pretrained("openai/whisper-tiny", language="spanish")
            >>> # now switch the prefix token from Spanish to French
            >>> tokenizer.set_prefix_tokens(language="french")
            ```

        Args:
            language (`str`, *optional*, defaults to `None`):
                The language of the transcription text.
            task (`str`, *optional*, defaults to `None`):
                Task identifier to append at the start of sequence (if any).
            predict_timestamps (`bool`, *optional*, defaults to `None`):
                Whether to omit the `<|notimestamps|>` token at the start of the sequence.
        """
        self.language = language if language is not None else self.language
        self.task = task if task is not None else self.task
        self.predict_timestamps = predict_timestamps if predict_timestamps is not None else self.predict_timestamps

        prefix_token_ids = self.prefix_tokens
        prefixes = self.convert_ids_to_tokens(prefix_token_ids)
        eos = self.eos_token
        eos_token_id = self.eos_token_id
        prefix_template = " ".join([f"{token}:0" for token in prefixes])
        self.backend_tokenizer.post_processor = processors.TemplateProcessing(
            single=f"{prefix_template} $A:0 {eos}:0",
            pair=f"{prefix_template} $A:0 $B:1 {eos}:1",
            special_tokens=[
                (eos, eos_token_id),
                *zip(prefixes, prefix_token_ids),
            ],
        )

    @property
    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.prefix_tokens
    def prefix_tokens(self) -> List[int]:
        """
        This method, prefix_tokens, is a member of the class WhisperTokenizerFast and is responsible for generating a
        list of token IDs that represent the prefix of a transcription or translation sequence. It takes the following
        parameter:

        Args:
            self:
                The instance of the WhisperTokenizerFast class. It is used to access the attributes and methods of the
                class within the prefix_tokens method.

        Returns:
            List[int]:
                Returns a list of integer token IDs representing the prefix of a transcription or translation
                sequence.

        Raises:
            ValueError:
                This exception is raised if the language provided is not supported or if the task provided is not
                recognized.
                The exception message provides details about the unsupported language or task and the valid options
                for language and task, respectively.
        """
        bos_token_id = self.convert_tokens_to_ids("<|startoftranscript|>")
        translate_token_id = self.convert_tokens_to_ids("<|translate|>")
        transcribe_token_id = self.convert_tokens_to_ids("<|transcribe|>")
        notimestamps_token_id = self.convert_tokens_to_ids("<|notimestamps|>")
        langs = tuple(LANGUAGES.keys())

        if self.language is not None:
            self.language = self.language.lower()
            if self.language in TO_LANGUAGE_CODE:
                language_id = TO_LANGUAGE_CODE[self.language]
            elif self.language in TO_LANGUAGE_CODE.values():
                language_id = self.language
            else:
                is_language_code = len(self.language) == 2
                raise ValueError(
                    f"Unsupported language: {self.language}. Language should be one of:"
                    f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}."
                )

        if self.task is not None:
            if self.task not in TASK_IDS:
                raise ValueError(f"Unsupported task: {self.task}. Task should be in: {TASK_IDS}")

        bos_sequence = [bos_token_id]
        if self.language is not None:
            bos_sequence.append(bos_token_id + 1 + langs.index(language_id))
        if self.task is not None:
            bos_sequence.append(transcribe_token_id if self.task == "transcribe" else translate_token_id)
        if not self.predict_timestamps:
            bos_sequence.append(notimestamps_token_id)
        return bos_sequence

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.build_inputs_with_special_tokens
    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
        """Build model inputs from a sequence by appending eos_token_id."""
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0 + [self.eos_token_id]
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_0 + token_ids_1 + [self.eos_token_id]

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_special_tokens_mask
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        prefix_ones = [1] * len(self.prefix_tokens)
        suffix_ones = [1]
        if token_ids_1 is None:
            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

    @property
    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.default_chat_template
    def default_chat_template(self):
        """
        A simple chat template that ignores role information and just concatenates messages with EOS tokens.
        """
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://hf-mirror.com/docs/transformers/main/chat_templating for more information.\n"
        )
        return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_decoder_prompt_ids
    def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
        """
        This method retrieves the decoder prompt IDs for the WhisperTokenizerFast class.

        Args:
            self (WhisperTokenizerFast): The instance of the WhisperTokenizerFast class.
            task (str, optional): The task associated with the decoder prompt IDs. Default is None.
            language (str, optional): The language for which the decoder prompt IDs are retrieved. Default is None.
            no_timestamps (bool, optional): A flag indicating whether timestamps should be predicted or not.
                Default is True.

        Returns:
            list: A list of tuples containing the rank and corresponding token of forced decoder IDs.

        Raises:
            None.
        """
        self.set_prefix_tokens(task=task, language=language, predict_timestamps=not no_timestamps)
        # prefix tokens are of the form: <|startoftranscript|> <|lang_id|> <|task|> <|notimestamps|>
        # we don't want to force the bos token at position 1, as this is the starting token
        # when we generate, so we slice the prefix tokens to: <|lang_id|> <|task|> <|notimestamps|>
        # to get the forced tokens
        forced_tokens = self.prefix_tokens[1:]
        forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_tokens)]
        return forced_decoder_ids

    def _decode_asr(self, model_outputs, *, return_timestamps, return_language, time_precision):
        """
        This method is used to decode ASR (Automatic Speech Recognition) outputs. It takes model outputs as input and
        decodes them based on specified parameters.

        Args:
            self (WhisperTokenizerFast): The instance of the WhisperTokenizerFast class.
            model_outputs (Any): The model outputs from the ASR system that need to be decoded.

            return_timestamps (bool): A flag indicating whether to return timestamps along with the decoded output.
            return_language (bool): A flag indicating whether to return the language information along with the decoded output.
            time_precision (str): The precision level for the timestamps, e.g., 'milliseconds', 'seconds'.

        Returns:
            None: This method does not return any value directly. The decoding results are processed within the method.

        Raises:
            ValueError: If the model_outputs are not in the expected format.
            RuntimeError: If there is an issue during the decoding process.
            KeyError: If there is a key error while accessing the model outputs.
        """
        return _decode_asr(
            self,
            model_outputs,
            return_timestamps=return_timestamps,
            return_language=return_language,
            time_precision=time_precision,
        )

    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_prompt_ids
    def get_prompt_ids(self, text: str, return_tensors="np"):
        """Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`]."""
        batch_encoding = self("<|startofprev|>", " " + text.strip(), add_special_tokens=False)

        # Check for special tokens
        prompt_text_ids = batch_encoding["input_ids"][1:]
        special_token_id = next((x for x in prompt_text_ids if x >= self.all_special_ids[0]), None)
        if special_token_id is not None:
            token = self.convert_ids_to_tokens(special_token_id)
            raise ValueError(f"Encountered text in the prompt corresponding to disallowed special token: {token}.")

        batch_encoding.convert_to_tensors(tensor_type=return_tensors)
        return batch_encoding["input_ids"]

    @staticmethod
    # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._strip_prompt
    def _strip_prompt(token_ids: List[int], prompt_token_id: int, decoder_start_token_id: int):
        """
        Removes the prompt from the given token IDs and returns the remaining tokens.

        Args:
            token_ids (List[int]): A list of token IDs.
                This list represents the sequence of tokens to process.
            prompt_token_id (int): The token ID that indicates the start of the prompt.
                The prompt is the initial text or instruction given to the model.
            decoder_start_token_id (int): The token ID that indicates the start of the decoder input.
                This token marks the beginning of the sequence of tokens to keep after stripping the prompt.

        Returns:
            List[int]: A list of token IDs with the prompt removed.
                If the token IDs contain the decoder start token, the resulting list starts from that token.
                If the token IDs do not contain the decoder start token, an empty list is returned.

        Raises:
            None.

        """
        has_prompt = isinstance(token_ids, list) and token_ids and token_ids[0] == prompt_token_id
        if has_prompt:
            if decoder_start_token_id in token_ids:
                return token_ids[token_ids.index(decoder_start_token_id) :]
            return []

        return token_ids

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.default_chat_template property

A simple chat template that ignores role information and just concatenates messages with EOS tokens.

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.prefix_tokens: List[int] property

This method, prefix_tokens, is a member of the class WhisperTokenizerFast and is responsible for generating a list of token IDs that represent the prefix of a transcription or translation sequence. It takes the following parameter:

PARAMETER	DESCRIPTION
self	The instance of the WhisperTokenizerFast class. It is used to access the attributes and methods of the class within the prefix_tokens method.

RETURNS	DESCRIPTION
List[int]	List[int]: Returns a list of integer token IDs representing the prefix of a transcription or translation sequence.

RAISES	DESCRIPTION
ValueError	This exception is raised if the language provided is not supported or if the task provided is not recognized. The exception message provides details about the unsupported language or task and the valid options for language and task, respectively.

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.__init__(vocab_file=None, merges_file=None, normalizer_file=None, tokenizer_file=None, unk_token='<|endoftext|>', bos_token='<|endoftext|>', eos_token='<|endoftext|>', add_prefix_space=False, language=None, task=None, predict_timestamps=False, **kwargs)

This method initializes an instance of the WhisperTokenizerFast class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The file path to the vocabulary file. TYPE: str DEFAULT: None
merges_file	The file path to the merges file. TYPE: str DEFAULT: None
normalizer_file	The file path to the normalizer file. TYPE: str DEFAULT: None
tokenizer_file	The file path to the tokenizer file. TYPE: str DEFAULT: None
unk_token	The unknown token. Default is 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
bos_token	The beginning of sentence token. Default is 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
eos_token	The end of sentence token. Default is 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
add_prefix_space	Indicates whether to add prefix space. Default is False. TYPE: bool DEFAULT: False
language	The language used. TYPE: str DEFAULT: None
task	The task for the tokenizer. TYPE: str DEFAULT: None
predict_timestamps	Indicates whether timestamps should be predicted. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
TypeError	If any of the parameters are of incorrect type.
FileNotFoundError	If any of the specified files cannot be found.
JSONDecodeError	If there is an issue with decoding JSON data.
AttributeError	If there is an attribute error while setting tokenizer pre-tokenizer.
ValueError	If there is an issue with the provided values.
RegexError	If there is an issue with the regular expression compilation.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def __init__(
    self,
    vocab_file=None,
    merges_file=None,
    normalizer_file=None,
    tokenizer_file=None,
    unk_token="<|endoftext|>",
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
    add_prefix_space=False,
    language=None,
    task=None,
    predict_timestamps=False,
    **kwargs,
):
    """
    This method initializes an instance of the WhisperTokenizerFast class.

    Args:
        self: The instance of the class.
        vocab_file (str): The file path to the vocabulary file.
        merges_file (str): The file path to the merges file.
        normalizer_file (str): The file path to the normalizer file.
        tokenizer_file (str): The file path to the tokenizer file.
        unk_token (str): The unknown token. Default is 'endoftext'.
        bos_token (str): The beginning of sentence token. Default is 'endoftext'.
        eos_token (str): The end of sentence token. Default is 'endoftext'.
        add_prefix_space (bool): Indicates whether to add prefix space. Default is False.
        language (str): The language used.
        task (str): The task for the tokenizer.
        predict_timestamps (bool): Indicates whether timestamps should be predicted.

    Returns:
        None

    Raises:
        TypeError: If any of the parameters are of incorrect type.
        FileNotFoundError: If any of the specified files cannot be found.
        JSONDecodeError: If there is an issue with decoding JSON data.
        AttributeError: If there is an attribute error while setting tokenizer pre-tokenizer.
        ValueError: If there is an issue with the provided values.
        RegexError: If there is an issue with the regular expression compilation.
    """
    bos_token = (
        AddedToken(bos_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(bos_token, str)
        else bos_token
    )
    eos_token = (
        AddedToken(eos_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(eos_token, str)
        else eos_token
    )
    unk_token = (
        AddedToken(unk_token, lstrip=False, rstrip=False, normalized=False, special=True)
        if isinstance(unk_token, str)
        else unk_token
    )

    super().__init__(
        vocab_file,
        merges_file,
        tokenizer_file=tokenizer_file,
        unk_token=unk_token,
        bos_token=bos_token,
        eos_token=eos_token,
        add_prefix_space=add_prefix_space,
        **kwargs,
    )

    self.add_bos_token = kwargs.pop("add_bos_token", False)

    pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
    if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
        pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
        pre_tok_state["add_prefix_space"] = add_prefix_space
        self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

    if normalizer_file is not None:
        with open(normalizer_file, encoding="utf-8") as vocab_handle:
            self.english_spelling_normalizer = json.load(vocab_handle)
    else:
        self.english_spelling_normalizer = None

    self.add_prefix_space = add_prefix_space
    self.timestamp_pat = re.compile(r"<\|(\d+\.\d+)\|>")

    self.language = language
    self.task = task
    self.predict_timestamps = predict_timestamps

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.basic_normalize(text, remove_diacritics=False) staticmethod

Normalize a given string using the BasicTextNormalizer class, which preforms commons transformation on multilingual text.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

@staticmethod
# Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.basic_normalize
def basic_normalize(text, remove_diacritics=False):
    """
    Normalize a given string using the `BasicTextNormalizer` class, which preforms commons transformation on
    multilingual text.
    """
    normalizer = BasicTextNormalizer(remove_diacritics=remove_diacritics)
    return normalizer(text)

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence by appending eos_token_id.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
    """Build model inputs from a sequence by appending eos_token_id."""
    if token_ids_1 is None:
        return self.prefix_tokens + token_ids_0 + [self.eos_token_id]
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_0 + token_ids_1 + [self.eos_token_id]

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.decode(token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=None, output_offsets=False, time_precision=0.02, decode_with_timestamps=False, normalize=False, basic_normalize=False, remove_diacritics=False, **kwargs)

Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces.

Similar to doing self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids)).

PARAMETER	DESCRIPTION
token_ids	List of tokenized input ids. Can be obtained using the __call__ method. TYPE: `Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`
skip_special_tokens	Whether or not to remove special tokens in the decoding. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
clean_up_tokenization_spaces	Whether or not to clean up the tokenization spaces. If None, will default to self.clean_up_tokenization_spaces (available in the tokenizer_config). TYPE: `bool`, *optional* DEFAULT: None
output_offsets	Whether or not to output the offsets of the tokens. This should only be set if the model predicted timestamps. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
time_precision	The time ratio to convert from token to time. TYPE: `float`, `optional`, defaults to 0.02 DEFAULT: 0.02
decode_with_timestamps	Whether or not to decode with timestamps included in the raw text. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
normalize	Whether or not to apply the English text normalizer to the decoded text. Only applicable when the target text is in English. Otherwise, the basic text normalizer should be applied. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
basic_normalize	Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual target text. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
remove_diacritics	Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may destroy information in the decoded text, hence it should be used with caution. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Will be passed to the underlying model specific decode method. TYPE: additional keyword arguments, *optional* DEFAULT: {}

RETURNS	DESCRIPTION
str	str: The decoded sentence.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def decode(
    self,
    token_ids,
    skip_special_tokens: bool = False,
    clean_up_tokenization_spaces: bool = None,
    output_offsets: bool = False,
    time_precision: float = 0.02,
    decode_with_timestamps: bool = False,
    normalize: bool = False,
    basic_normalize: bool = False,
    remove_diacritics: bool = False,
    **kwargs,
) -> str:
    """
    Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
    tokens and clean up tokenization spaces.

    Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

    Args:
        token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
            List of tokenized input ids. Can be obtained using the `__call__` method.
        skip_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to remove special tokens in the decoding.
        clean_up_tokenization_spaces (`bool`, *optional*):
            Whether or not to clean up the tokenization spaces. If `None`, will default to
            `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
        output_offsets (`bool`, *optional*, defaults to `False`):
            Whether or not to output the offsets of the tokens. This should only be set if the model predicted
            timestamps.
        time_precision (`float`, `optional`, defaults to 0.02):
            The time ratio to convert from token to time.
        decode_with_timestamps (`bool`, *optional*, defaults to `False`):
            Whether or not to decode with timestamps included in the raw text.
        normalize (`bool`, *optional*, defaults to `False`):
            Whether or not to apply the English text normalizer to the decoded text. Only applicable when the
            target text is in English. Otherwise, the basic text normalizer should be applied.
        basic_normalize (`bool`, *optional*, defaults to `False`):
            Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual
            target text.
        remove_diacritics (`bool`, *optional*, defaults to `False`):
            Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may
            destroy information in the decoded text, hence it should be used with caution.
        kwargs (additional keyword arguments, *optional*):
            Will be passed to the underlying model specific decode method.

    Returns:
        `str`: The decoded sentence.
    """
    filtered_ids = self._preprocess_token_ids(
        token_ids,
        skip_special_tokens=skip_special_tokens,
    )

    text = super().decode(
        filtered_ids,
        skip_special_tokens=skip_special_tokens,
        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
        normalize=normalize,
        basic_normalize=basic_normalize,
        remove_diacritics=remove_diacritics,
        **kwargs,
    )
    if decode_with_timestamps:
        # legacy method to decode timestamps when not included in the tokenizer vocabulary
        text = self._decode_with_timestamps(
            filtered_ids, time_precision=time_precision, skip_special_tokens=skip_special_tokens
        )
    else:
        text = self._filter_timestamp_ids(text)

    # retrieve offsets
    if output_offsets:
        offsets = self._compute_offsets(token_ids, time_precision=time_precision)
        return {"text": text, "offsets": offsets}
    return text

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.get_decoder_prompt_ids(task=None, language=None, no_timestamps=True)

This method retrieves the decoder prompt IDs for the WhisperTokenizerFast class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperTokenizerFast class. TYPE: WhisperTokenizerFast
task	The task associated with the decoder prompt IDs. Default is None. TYPE: str DEFAULT: None
language	The language for which the decoder prompt IDs are retrieved. Default is None. TYPE: str DEFAULT: None
no_timestamps	A flag indicating whether timestamps should be predicted or not. Default is True. TYPE: bool DEFAULT: True

RETURNS	DESCRIPTION
list	A list of tuples containing the rank and corresponding token of forced decoder IDs.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def get_decoder_prompt_ids(self, task=None, language=None, no_timestamps=True):
    """
    This method retrieves the decoder prompt IDs for the WhisperTokenizerFast class.

    Args:
        self (WhisperTokenizerFast): The instance of the WhisperTokenizerFast class.
        task (str, optional): The task associated with the decoder prompt IDs. Default is None.
        language (str, optional): The language for which the decoder prompt IDs are retrieved. Default is None.
        no_timestamps (bool, optional): A flag indicating whether timestamps should be predicted or not.
            Default is True.

    Returns:
        list: A list of tuples containing the rank and corresponding token of forced decoder IDs.

    Raises:
        None.
    """
    self.set_prefix_tokens(task=task, language=language, predict_timestamps=not no_timestamps)
    # prefix tokens are of the form: <|startoftranscript|> <|lang_id|> <|task|> <|notimestamps|>
    # we don't want to force the bos token at position 1, as this is the starting token
    # when we generate, so we slice the prefix tokens to: <|lang_id|> <|task|> <|notimestamps|>
    # to get the forced tokens
    forced_tokens = self.prefix_tokens[1:]
    forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_tokens)]
    return forced_decoder_ids

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.get_prompt_ids(text, return_tensors='np')

Converts prompt text to IDs that can be passed to [~WhisperForConditionalGeneration.generate].

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def get_prompt_ids(self, text: str, return_tensors="np"):
    """Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`]."""
    batch_encoding = self("<|startofprev|>", " " + text.strip(), add_special_tokens=False)

    # Check for special tokens
    prompt_text_ids = batch_encoding["input_ids"][1:]
    special_token_id = next((x for x in prompt_text_ids if x >= self.all_special_ids[0]), None)
    if special_token_id is not None:
        token = self.convert_ids_to_tokens(special_token_id)
        raise ValueError(f"Encountered text in the prompt corresponding to disallowed special token: {token}.")

    batch_encoding.convert_to_tensors(tensor_type=return_tensors)
    return batch_encoding["input_ids"]

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    prefix_ones = [1] * len(self.prefix_tokens)
    suffix_ones = [1]
    if token_ids_1 is None:
        return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
    return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.normalize(text)

Normalize a given string using the EnglishTextNormalizer class, which preforms commons transformation on english text.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def normalize(self, text):
    """
    Normalize a given string using the `EnglishTextNormalizer` class, which preforms commons transformation on
    english text.
    """
    normalizer = EnglishTextNormalizer(self.english_spelling_normalizer)
    return normalizer(text)

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary to the specified directory.

PARAMETER	DESCRIPTION
self	Instance of the WhisperTokenizerFast class.
save_directory	The directory where the vocabulary files will be saved. TYPE: str
filename_prefix	An optional prefix to be included in the filename. Default is None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the paths of the saved files and the path of the normalizer file.

RAISES	DESCRIPTION
OSError	If an error occurs while saving the vocabulary files to the specified directory.
ValueError	If the save_directory is invalid or inaccessible.
TypeError	If the provided filename_prefix is not a string.

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary to the specified directory.

    Args:
        self: Instance of the WhisperTokenizerFast class.
        save_directory (str): The directory where the vocabulary files will be saved.
        filename_prefix (Optional[str]): An optional prefix to be included in the filename. Default is None.

    Returns:
        Tuple[str]: A tuple containing the paths of the saved files and the path of the normalizer file.

    Raises:
        OSError: If an error occurs while saving the vocabulary files to the specified directory.
        ValueError: If the save_directory is invalid or inaccessible.
        TypeError: If the provided filename_prefix is not a string.
    """
    files = self._tokenizer.model.save(save_directory, name=filename_prefix)

    normalizer_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["normalizer_file"]
    )

    if self.english_spelling_normalizer is not None:
        with open(normalizer_file, "w", encoding="utf-8") as f:
            f.write(
                json.dumps(self.english_spelling_normalizer, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
            )

    return tuple(files) + (normalizer_file,)

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.set_prefix_tokens(language=None, task=None, predict_timestamps=None)

Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to update the prefix tokens as required when fine-tuning.

Example

>>> # instantiate the tokenizer and set the prefix token to Spanish
>>> tokenizer = WhisperTokenizerFast.from_pretrained("openai/whisper-tiny", language="spanish")
>>> # now switch the prefix token from Spanish to French
>>> tokenizer.set_prefix_tokens(language="french")

PARAMETER	DESCRIPTION
language	The language of the transcription text. TYPE: `str`, *optional*, defaults to `None` DEFAULT: None
task	Task identifier to append at the start of sequence (if any). TYPE: `str`, *optional*, defaults to `None` DEFAULT: None
predict_timestamps	Whether to omit the <|notimestamps|> token at the start of the sequence. TYPE: `bool`, *optional*, defaults to `None` DEFAULT: None

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

def set_prefix_tokens(self, language: str = None, task: str = None, predict_timestamps: bool = None):
    """
    Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to
    update the prefix tokens as required when fine-tuning.

    Example:
        ```python
        >>> # instantiate the tokenizer and set the prefix token to Spanish
        >>> tokenizer = WhisperTokenizerFast.from_pretrained("openai/whisper-tiny", language="spanish")
        >>> # now switch the prefix token from Spanish to French
        >>> tokenizer.set_prefix_tokens(language="french")
        ```

    Args:
        language (`str`, *optional*, defaults to `None`):
            The language of the transcription text.
        task (`str`, *optional*, defaults to `None`):
            Task identifier to append at the start of sequence (if any).
        predict_timestamps (`bool`, *optional*, defaults to `None`):
            Whether to omit the `<|notimestamps|>` token at the start of the sequence.
    """
    self.language = language if language is not None else self.language
    self.task = task if task is not None else self.task
    self.predict_timestamps = predict_timestamps if predict_timestamps is not None else self.predict_timestamps

    prefix_token_ids = self.prefix_tokens
    prefixes = self.convert_ids_to_tokens(prefix_token_ids)
    eos = self.eos_token
    eos_token_id = self.eos_token_id
    prefix_template = " ".join([f"{token}:0" for token in prefixes])
    self.backend_tokenizer.post_processor = processors.TemplateProcessing(
        single=f"{prefix_template} $A:0 {eos}:0",
        pair=f"{prefix_template} $A:0 $B:1 {eos}:1",
        special_tokens=[
            (eos, eos_token_id),
            *zip(prefixes, prefix_token_ids),
        ],
    )

mindnlp.transformers.models.whisper.tokenization_whisper_fast.WhisperTokenizerFast.timestamp_ids(time_precision=0.02) cached

Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

PARAMETER	DESCRIPTION
time_precision	The time ratio to convert from token to time. TYPE: `float`, `optional`, defaults to 0.02 DEFAULT: 0.02

Source code in mindnlp/transformers/models/whisper/tokenization_whisper_fast.py

@lru_cache(128)
# Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.timestamp_ids
def timestamp_ids(self, time_precision=0.02):
    """
    Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache.

    Args:
        time_precision (`float`, `optional`, defaults to 0.02):
            The time ratio to convert from token to time.
    """
    return self.convert_tokens_to_ids([("<|%.2f|>" % (i * time_precision)) for i in range(1500 + 1)])