deeppavlov/core/layers/tf_csoftmax_attention.py
def attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, key, cum_att):
    """ It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]
    """
    with tf.name_scope('attention_step'):
        sketch_dims = hidden_for_sketch.get_shape().as_list()
        batch_size = sketch_dims[0]
        num_tokens = sketch_dims[1]
        hidden_size = sketch_dims[2]
        attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list()
        attn_alignment_hidden_size = attn_alignment_dims[2]

        repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1,num_tokens, 1))
        concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-1)


        concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2*hidden_size]) # dirty trick
        reduce_mem = tf.layers.dense(concat_mem, hidden_size)

        projected_key = tf.layers.dense(key, hidden_size)
        t_key = tf.reshape(projected_key,[-1, hidden_size, 1])

        score = tf.reshape(tf.matmul(reduce_mem, t_key), [-1, num_tokens])

        inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-1, num_tokens])
        att = csoftmax(score, inv_cum_att)

        t_reduce_mem = tf.transpose(reduce_mem, [0,2,1])
        t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0,2,1])

        r_att = tf.reshape(att, [-1, num_tokens, 1])

        next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-1)
        aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-1)
    return next_sketch, att, aligned_hidden_sketch
def attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, key, cum_att):
    """ It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]
    """
    with tf.name_scope('attention_step'):
        sketch_dims = hidden_for_sketch.get_shape().as_list()
        batch_size = sketch_dims[0]
        num_tokens = sketch_dims[1]
        hidden_size = sketch_dims[2]
        attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list()
        attn_alignment_hidden_size = attn_alignment_dims[2]

        repeated_sketch = tf.tile(tf.reshape(sketch, [-1, 1, hidden_size]), (1,num_tokens, 1))
        concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-1)


        concat_mem = tf.reshape(concat_mem, [-1, num_tokens, 2*hidden_size]) # dirty trick
        reduce_mem = tf.layers.dense(concat_mem, hidden_size)

        projected_key = tf.layers.dense(key, hidden_size)
        t_key = tf.reshape(projected_key,[-1, hidden_size, 1])

        score = tf.reshape(tf.matmul(reduce_mem, t_key), [-1, num_tokens])

        inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-1, num_tokens])
        att = csoftmax(score, inv_cum_att)

        t_reduce_mem = tf.transpose(reduce_mem, [0,2,1])
        t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [0,2,1])

        r_att = tf.reshape(att, [-1, num_tokens, 1])

        next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-1)
        aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-1)
    return next_sketch, att, aligned_hidden_sketch
def attention_gen_step ( hidden_for_sketch , hidden_for_attn_alignment , sketch , key , cum_att ) : with tf . name_scope ( 'attention_step' ) : sketch_dims = hidden_for_sketch . get_shape ( ) . as_list ( ) batch_size = sketch_dims [ 0 ] num_tokens = sketch_dims [ 1 ] hidden_size = sketch_dims [ 2 ] attn_alignment_dims = hidden_for_attn_alignment . get_shape ( ) . as_list ( ) attn_alignment_hidden_size = attn_alignment_dims [ 2 ] repeated_sketch = tf . tile ( tf . reshape ( sketch , [ - 1 , 1 , hidden_size ] ) , ( 1 , num_tokens , 1 ) ) concat_mem = tf . concat ( [ hidden_for_sketch , repeated_sketch ] , - 1 ) concat_mem = tf . reshape ( concat_mem , [ - 1 , num_tokens , 2 * hidden_size ] ) # dirty trick reduce_mem = tf . layers . dense ( concat_mem , hidden_size ) projected_key = tf . layers . dense ( key , hidden_size ) t_key = tf . reshape ( projected_key , [ - 1 , hidden_size , 1 ] ) score = tf . reshape ( tf . matmul ( reduce_mem , t_key ) , [ - 1 , num_tokens ] ) inv_cum_att = tf . reshape ( tf . ones_like ( cum_att ) - cum_att , [ - 1 , num_tokens ] ) att = csoftmax ( score , inv_cum_att ) t_reduce_mem = tf . transpose ( reduce_mem , [ 0 , 2 , 1 ] ) t_hidden_for_attn_alignment = tf . transpose ( hidden_for_attn_alignment , [ 0 , 2 , 1 ] ) r_att = tf . reshape ( att , [ - 1 , num_tokens , 1 ] ) next_sketch = tf . squeeze ( tf . matmul ( t_reduce_mem , r_att ) , - 1 ) aligned_hidden_sketch = tf . squeeze ( tf . matmul ( t_hidden_for_attn_alignment , r_att ) , - 1 ) return next_sketch , att , aligned_hidden_sketch
It is a implementation one step of block of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]
It is a implementation one step of block of the Luong et al . attention mechanism with general score and the constrained softmax ( csoftmax ) . Based on the papers : https : // arxiv . org / abs / 1508 . 04025 Effective Approaches to Attention - based Neural Machine Translation https : // andre - martins . github . io / docs / emnlp2017_final . pdf Learning What s Easy : Fully Differentiable Neural Easy - First Taggers Args : hidden_for_sketch : A tensorflow tensor for a sketch computing . This tensor have dimensionality [ None max_num_tokens sketch_hidden_size ] hidden_for_attn_alignment : A tensorflow tensor is aligned for output during a performing . This tensor have dimensionality [ None max_num_tokens hidden_size_for_attn_alignment ] sketch : A previous step sketch tensor for a sketch computing . This tensor have dimensionality [ None sketch_hidden_size ] key : A tensorflow tensor with dimensionality [ None None key_size ] cum_att : A cumulative attention tensor with dimensionality [ None max_num_tokens ] Returns : next_sketch : Tensor of the current step sketch with dimensionality [ None sketch_hidden_size ] att : Tensor of the current step attention with dimensionality [ None max_num_tokens ] aligned_hidden_sketch : Tensor of aligned hidden state of current step with dimensionality [ None hidden_size_for_attn_alignment ]