baselines/acer/acer.py
def q_retrace(R, D, q_i, v, rho_i, nenvs, nsteps, gamma):
    """
    Calculates q_retrace targets

    :param R: Rewards
    :param D: Dones
    :param q_i: Q values for actions taken
    :param v: V values
    :param rho_i: Importance weight for each action
    :return: Q_retrace values
    """
    rho_bar = batch_to_seq(tf.minimum(1.0, rho_i), nenvs, nsteps, True)  # list of len steps, shape [nenvs]
    rs = batch_to_seq(R, nenvs, nsteps, True)  # list of len steps, shape [nenvs]
    ds = batch_to_seq(D, nenvs, nsteps, True)  # list of len steps, shape [nenvs]
    q_is = batch_to_seq(q_i, nenvs, nsteps, True)
    vs = batch_to_seq(v, nenvs, nsteps + 1, True)
    v_final = vs[-1]
    qret = v_final
    qrets = []
    for i in range(nsteps - 1, -1, -1):
        check_shape([qret, ds[i], rs[i], rho_bar[i], q_is[i], vs[i]], [[nenvs]] * 6)
        qret = rs[i] + gamma * qret * (1.0 - ds[i])
        qrets.append(qret)
        qret = (rho_bar[i] * (qret - q_is[i])) + vs[i]
    qrets = qrets[::-1]
    qret = seq_to_batch(qrets, flat=True)
    return qret
def q_retrace(R, D, q_i, v, rho_i, nenvs, nsteps, gamma):
    """
    Calculates q_retrace targets

    :param R: Rewards
    :param D: Dones
    :param q_i: Q values for actions taken
    :param v: V values
    :param rho_i: Importance weight for each action
    :return: Q_retrace values
    """
    rho_bar = batch_to_seq(tf.minimum(1.0, rho_i), nenvs, nsteps, True)  # list of len steps, shape [nenvs]
    rs = batch_to_seq(R, nenvs, nsteps, True)  # list of len steps, shape [nenvs]
    ds = batch_to_seq(D, nenvs, nsteps, True)  # list of len steps, shape [nenvs]
    q_is = batch_to_seq(q_i, nenvs, nsteps, True)
    vs = batch_to_seq(v, nenvs, nsteps + 1, True)
    v_final = vs[-1]
    qret = v_final
    qrets = []
    for i in range(nsteps - 1, -1, -1):
        check_shape([qret, ds[i], rs[i], rho_bar[i], q_is[i], vs[i]], [[nenvs]] * 6)
        qret = rs[i] + gamma * qret * (1.0 - ds[i])
        qrets.append(qret)
        qret = (rho_bar[i] * (qret - q_is[i])) + vs[i]
    qrets = qrets[::-1]
    qret = seq_to_batch(qrets, flat=True)
    return qret
def q_retrace ( R , D , q_i , v , rho_i , nenvs , nsteps , gamma ) : rho_bar = batch_to_seq ( tf . minimum ( 1.0 , rho_i ) , nenvs , nsteps , True ) # list of len steps, shape [nenvs] rs = batch_to_seq ( R , nenvs , nsteps , True ) # list of len steps, shape [nenvs] ds = batch_to_seq ( D , nenvs , nsteps , True ) # list of len steps, shape [nenvs] q_is = batch_to_seq ( q_i , nenvs , nsteps , True ) vs = batch_to_seq ( v , nenvs , nsteps + 1 , True ) v_final = vs [ - 1 ] qret = v_final qrets = [ ] for i in range ( nsteps - 1 , - 1 , - 1 ) : check_shape ( [ qret , ds [ i ] , rs [ i ] , rho_bar [ i ] , q_is [ i ] , vs [ i ] ] , [ [ nenvs ] ] * 6 ) qret = rs [ i ] + gamma * qret * ( 1.0 - ds [ i ] ) qrets . append ( qret ) qret = ( rho_bar [ i ] * ( qret - q_is [ i ] ) ) + vs [ i ] qrets = qrets [ : : - 1 ] qret = seq_to_batch ( qrets , flat = True ) return qret
Calculates q_retrace targets

    :param R: Rewards
    :param D: Dones
    :param q_i: Q values for actions taken
    :param v: V values
    :param rho_i: Importance weight for each action
    :return: Q_retrace values
Calculates q_retrace targets