deeppavlov/core/layers/tf_csoftmax_attention.py
def attention_gen_block(hidden_for_sketch, hidden_for_attn_alignment, key, attention_depth):
    """ It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        attention_depth: Number of usage csoftmax
    Returns:
        final_aligned_hiddens: Tensor at the output with dimensionality [1, attention_depth, hidden_size_for_attn_alignment]
    """
    with tf.name_scope('attention_block'):
        sketch_dims = tf.shape(hidden_for_sketch)
        batch_size = sketch_dims[0]
        num_tokens = sketch_dims[1]
        hidden_size = sketch_dims[2]

        attn_alignment_dims = tf.shape(hidden_for_attn_alignment)
        attn_alignment_hidden_size = attn_alignment_dims[2]

        sketches = [tf.zeros(shape=[batch_size, hidden_size], dtype=tf.float32)]
        aligned_hiddens = []
        cum_att = tf.zeros(shape=[batch_size, num_tokens])  # cumulative attention
        for i in range(attention_depth):
            sketch, cum_att_, aligned_hidden = attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-1], key, cum_att)
            sketches.append(sketch) #sketch
            aligned_hiddens.append(aligned_hidden) #sketch
            cum_att += cum_att_
        final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [1, 0, 2]),[1, attention_depth, attn_alignment_hidden_size])
    return final_aligned_hiddens
def attention_gen_block(hidden_for_sketch, hidden_for_attn_alignment, key, attention_depth):
    """ It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        attention_depth: Number of usage csoftmax
    Returns:
        final_aligned_hiddens: Tensor at the output with dimensionality [1, attention_depth, hidden_size_for_attn_alignment]
    """
    with tf.name_scope('attention_block'):
        sketch_dims = tf.shape(hidden_for_sketch)
        batch_size = sketch_dims[0]
        num_tokens = sketch_dims[1]
        hidden_size = sketch_dims[2]

        attn_alignment_dims = tf.shape(hidden_for_attn_alignment)
        attn_alignment_hidden_size = attn_alignment_dims[2]

        sketches = [tf.zeros(shape=[batch_size, hidden_size], dtype=tf.float32)]
        aligned_hiddens = []
        cum_att = tf.zeros(shape=[batch_size, num_tokens])  # cumulative attention
        for i in range(attention_depth):
            sketch, cum_att_, aligned_hidden = attention_gen_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-1], key, cum_att)
            sketches.append(sketch) #sketch
            aligned_hiddens.append(aligned_hidden) #sketch
            cum_att += cum_att_
        final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [1, 0, 2]),[1, attention_depth, attn_alignment_hidden_size])
    return final_aligned_hiddens
def attention_gen_block ( hidden_for_sketch , hidden_for_attn_alignment , key , attention_depth ) : with tf . name_scope ( 'attention_block' ) : sketch_dims = tf . shape ( hidden_for_sketch ) batch_size = sketch_dims [ 0 ] num_tokens = sketch_dims [ 1 ] hidden_size = sketch_dims [ 2 ] attn_alignment_dims = tf . shape ( hidden_for_attn_alignment ) attn_alignment_hidden_size = attn_alignment_dims [ 2 ] sketches = [ tf . zeros ( shape = [ batch_size , hidden_size ] , dtype = tf . float32 ) ] aligned_hiddens = [ ] cum_att = tf . zeros ( shape = [ batch_size , num_tokens ] ) # cumulative attention for i in range ( attention_depth ) : sketch , cum_att_ , aligned_hidden = attention_gen_step ( hidden_for_sketch , hidden_for_attn_alignment , sketches [ - 1 ] , key , cum_att ) sketches . append ( sketch ) #sketch aligned_hiddens . append ( aligned_hidden ) #sketch cum_att += cum_att_ final_aligned_hiddens = tf . reshape ( tf . transpose ( tf . stack ( aligned_hiddens ) , [ 1 , 0 , 2 ] ) , [ 1 , attention_depth , attn_alignment_hidden_size ] ) return final_aligned_hiddens
It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        attention_depth: Number of usage csoftmax
    Returns:
        final_aligned_hiddens: Tensor at the output with dimensionality [1, attention_depth, hidden_size_for_attn_alignment]
It is a implementation of the Luong et al . attention mechanism with general score and the constrained softmax ( csoftmax ) . Based on the papers : https : // arxiv . org / abs / 1508 . 04025 Effective Approaches to Attention - based Neural Machine Translation https : // andre - martins . github . io / docs / emnlp2017_final . pdf Learning What s Easy : Fully Differentiable Neural Easy - First Taggers Args : hidden_for_sketch : A tensorflow tensor for a sketch computing . This tensor have dimensionality [ None max_num_tokens sketch_hidden_size ] hidden_for_attn_alignment : A tensorflow tensor is aligned for output during a performing . This tensor have dimensionality [ None max_num_tokens hidden_size_for_attn_alignment ] key : A tensorflow tensor with dimensionality [ None None key_size ] attention_depth : Number of usage csoftmax Returns : final_aligned_hiddens : Tensor at the output with dimensionality [ 1 attention_depth hidden_size_for_attn_alignment ]