deeppavlov/core/layers/tf_attention_mechanisms.py
def light_bahdanau_attention(key, context, hidden_size, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
    """
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    # projected_context: [None, max_num_tokens, hidden_size]
    projected_context = \
        tf.layers.dense(r_context, hidden_size, kernel_initializer=xav())
    concat_h_state = tf.concat([projected_context, r_projected_key], -1)

    projected_state = \
        tf.layers.dense(concat_h_state, hidden_size, use_bias=False,
                        kernel_initializer=xav())
    score = \
        tf.layers.dense(tf.tanh(projected_state), units=1, use_bias=False,
                        kernel_initializer=xav())

    attn = tf.nn.softmax(score, dim=1)

    if projected_align:
        log.info("Using projected attention alignment")
        t_context = tf.transpose(projected_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, hidden_size])
    else:
        log.info("Using without projected attention alignment")
        t_context = tf.transpose(r_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, token_size])
    return output
def light_bahdanau_attention(key, context, hidden_size, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
    """
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]
    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    # projected_context: [None, max_num_tokens, hidden_size]
    projected_context = \
        tf.layers.dense(r_context, hidden_size, kernel_initializer=xav())
    concat_h_state = tf.concat([projected_context, r_projected_key], -1)

    projected_state = \
        tf.layers.dense(concat_h_state, hidden_size, use_bias=False,
                        kernel_initializer=xav())
    score = \
        tf.layers.dense(tf.tanh(projected_state), units=1, use_bias=False,
                        kernel_initializer=xav())

    attn = tf.nn.softmax(score, dim=1)

    if projected_align:
        log.info("Using projected attention alignment")
        t_context = tf.transpose(projected_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, hidden_size])
    else:
        log.info("Using without projected attention alignment")
        t_context = tf.transpose(r_context, [0, 2, 1])
        output = tf.reshape(tf.matmul(t_context, attn),
                            shape=[batch_size, -1, token_size])
    return output
def light_bahdanau_attention ( key , context , hidden_size , projected_align = False ) : batch_size = tf . shape ( context ) [ 0 ] max_num_tokens , token_size = context . get_shape ( ) . as_list ( ) [ - 2 : ] r_context = tf . reshape ( context , shape = [ - 1 , max_num_tokens , token_size ] ) # projected_key: [None, None, hidden_size] projected_key = tf . layers . dense ( key , hidden_size , kernel_initializer = xav ( ) ) r_projected_key = tf . tile ( tf . reshape ( projected_key , shape = [ - 1 , 1 , hidden_size ] ) , [ 1 , max_num_tokens , 1 ] ) # projected_context: [None, max_num_tokens, hidden_size] projected_context = tf . layers . dense ( r_context , hidden_size , kernel_initializer = xav ( ) ) concat_h_state = tf . concat ( [ projected_context , r_projected_key ] , - 1 ) projected_state = tf . layers . dense ( concat_h_state , hidden_size , use_bias = False , kernel_initializer = xav ( ) ) score = tf . layers . dense ( tf . tanh ( projected_state ) , units = 1 , use_bias = False , kernel_initializer = xav ( ) ) attn = tf . nn . softmax ( score , dim = 1 ) if projected_align : log . info ( "Using projected attention alignment" ) t_context = tf . transpose ( projected_context , [ 0 , 2 , 1 ] ) output = tf . reshape ( tf . matmul ( t_context , attn ) , shape = [ batch_size , - 1 , hidden_size ] ) else : log . info ( "Using without projected attention alignment" ) t_context = tf . transpose ( r_context , [ 0 , 2 , 1 ] ) output = tf . reshape ( tf . matmul ( t_context , attn ) , shape = [ batch_size , - 1 , token_size ] ) return output
It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]
It is a implementation of the Bahdanau et al . attention mechanism . Based on the paper : https : // arxiv . org / abs / 1409 . 0473 Neural Machine Translation by Jointly Learning to Align and Translate Args : key : A tensorflow tensor with dimensionality [ None None key_size ] context : A tensorflow tensor with dimensionality [ None None max_num_tokens token_size ] hidden_size : Number of units in hidden representation projected_align : Using dense layer for hidden representation of context . If true between input and attention mechanism insert a dense layer with dimensionality [ hidden_size ] . If false a dense layer is not used . Returns : output : Tensor at the output with dimensionality [ None None hidden_size ]