deeppavlov/models/squad/utils.py
def dot_attention(inputs, memory, mask, att_size, keep_prob=1.0, scope="dot_attention"):
    """Computes attention vector for each item in inputs:
       attention vector is a weighted sum of memory items.
       Dot product between input and memory vector is used as similarity measure.

       Gate mechanism is applied to attention vectors to produce output.

    Args:
        inputs: Tensor [batch_size x input_len x feature_size]
        memory: Tensor [batch_size x memory_len x feature_size]
        mask: inputs mask
        att_size: hidden size of attention
        keep_prob: dropout keep_prob
        scope:

    Returns:
        attention vectors [batch_size x input_len x (feature_size + feature_size)]

    """
    with tf.variable_scope(scope):
        BS, IL, IH = tf.unstack(tf.shape(inputs))
        BS, ML, MH = tf.unstack(tf.shape(memory))

        d_inputs = tf.nn.dropout(inputs, keep_prob=keep_prob, noise_shape=[BS, 1, IH])
        d_memory = tf.nn.dropout(memory, keep_prob=keep_prob, noise_shape=[BS, 1, MH])

        with tf.variable_scope("attention"):
            inputs_att = tf.layers.dense(d_inputs, att_size, use_bias=False, activation=tf.nn.relu)
            memory_att = tf.layers.dense(d_memory, att_size, use_bias=False, activation=tf.nn.relu)
            logits = tf.matmul(inputs_att, tf.transpose(memory_att, [0, 2, 1])) / (att_size ** 0.5)
            mask = tf.tile(tf.expand_dims(mask, axis=1), [1, IL, 1])
            att_weights = tf.nn.softmax(softmax_mask(logits, mask))
            outputs = tf.matmul(att_weights, memory)
            res = tf.concat([inputs, outputs], axis=2)

        with tf.variable_scope("gate"):
            dim = res.get_shape().as_list()[-1]
            d_res = tf.nn.dropout(res, keep_prob=keep_prob, noise_shape=[BS, 1, IH + MH])
            gate = tf.layers.dense(d_res, dim, use_bias=False, activation=tf.nn.sigmoid)
            return res * gate
def dot_attention(inputs, memory, mask, att_size, keep_prob=1.0, scope="dot_attention"):
    """Computes attention vector for each item in inputs:
       attention vector is a weighted sum of memory items.
       Dot product between input and memory vector is used as similarity measure.

       Gate mechanism is applied to attention vectors to produce output.

    Args:
        inputs: Tensor [batch_size x input_len x feature_size]
        memory: Tensor [batch_size x memory_len x feature_size]
        mask: inputs mask
        att_size: hidden size of attention
        keep_prob: dropout keep_prob
        scope:

    Returns:
        attention vectors [batch_size x input_len x (feature_size + feature_size)]

    """
    with tf.variable_scope(scope):
        BS, IL, IH = tf.unstack(tf.shape(inputs))
        BS, ML, MH = tf.unstack(tf.shape(memory))

        d_inputs = tf.nn.dropout(inputs, keep_prob=keep_prob, noise_shape=[BS, 1, IH])
        d_memory = tf.nn.dropout(memory, keep_prob=keep_prob, noise_shape=[BS, 1, MH])

        with tf.variable_scope("attention"):
            inputs_att = tf.layers.dense(d_inputs, att_size, use_bias=False, activation=tf.nn.relu)
            memory_att = tf.layers.dense(d_memory, att_size, use_bias=False, activation=tf.nn.relu)
            logits = tf.matmul(inputs_att, tf.transpose(memory_att, [0, 2, 1])) / (att_size ** 0.5)
            mask = tf.tile(tf.expand_dims(mask, axis=1), [1, IL, 1])
            att_weights = tf.nn.softmax(softmax_mask(logits, mask))
            outputs = tf.matmul(att_weights, memory)
            res = tf.concat([inputs, outputs], axis=2)

        with tf.variable_scope("gate"):
            dim = res.get_shape().as_list()[-1]
            d_res = tf.nn.dropout(res, keep_prob=keep_prob, noise_shape=[BS, 1, IH + MH])
            gate = tf.layers.dense(d_res, dim, use_bias=False, activation=tf.nn.sigmoid)
            return res * gate
def dot_attention ( inputs , memory , mask , att_size , keep_prob = 1.0 , scope = "dot_attention" ) : with tf . variable_scope ( scope ) : BS , IL , IH = tf . unstack ( tf . shape ( inputs ) ) BS , ML , MH = tf . unstack ( tf . shape ( memory ) ) d_inputs = tf . nn . dropout ( inputs , keep_prob = keep_prob , noise_shape = [ BS , 1 , IH ] ) d_memory = tf . nn . dropout ( memory , keep_prob = keep_prob , noise_shape = [ BS , 1 , MH ] ) with tf . variable_scope ( "attention" ) : inputs_att = tf . layers . dense ( d_inputs , att_size , use_bias = False , activation = tf . nn . relu ) memory_att = tf . layers . dense ( d_memory , att_size , use_bias = False , activation = tf . nn . relu ) logits = tf . matmul ( inputs_att , tf . transpose ( memory_att , [ 0 , 2 , 1 ] ) ) / ( att_size ** 0.5 ) mask = tf . tile ( tf . expand_dims ( mask , axis = 1 ) , [ 1 , IL , 1 ] ) att_weights = tf . nn . softmax ( softmax_mask ( logits , mask ) ) outputs = tf . matmul ( att_weights , memory ) res = tf . concat ( [ inputs , outputs ] , axis = 2 ) with tf . variable_scope ( "gate" ) : dim = res . get_shape ( ) . as_list ( ) [ - 1 ] d_res = tf . nn . dropout ( res , keep_prob = keep_prob , noise_shape = [ BS , 1 , IH + MH ] ) gate = tf . layers . dense ( d_res , dim , use_bias = False , activation = tf . nn . sigmoid ) return res * gate
Computes attention vector for each item in inputs:
       attention vector is a weighted sum of memory items.
       Dot product between input and memory vector is used as similarity measure.

       Gate mechanism is applied to attention vectors to produce output.

    Args:
        inputs: Tensor [batch_size x input_len x feature_size]
        memory: Tensor [batch_size x memory_len x feature_size]
        mask: inputs mask
        att_size: hidden size of attention
        keep_prob: dropout keep_prob
        scope:

    Returns:
        attention vectors [batch_size x input_len x (feature_size + feature_size)]
Computes attention vector for each item in inputs : attention vector is a weighted sum of memory items . Dot product between input and memory vector is used as similarity measure .