baselines/ppo1/pposgd_simple.py
def add_vtarg_and_adv(seg, gamma, lam):
    """
    Compute target value using TD(lambda) estimator, and advantage with GAE(lambda)
    """
    new = np.append(seg["new"], 0) # last element is only used for last vtarg, but we already zeroed it if last new = 1
    vpred = np.append(seg["vpred"], seg["nextvpred"])
    T = len(seg["rew"])
    seg["adv"] = gaelam = np.empty(T, 'float32')
    rew = seg["rew"]
    lastgaelam = 0
    for t in reversed(range(T)):
        nonterminal = 1-new[t+1]
        delta = rew[t] + gamma * vpred[t+1] * nonterminal - vpred[t]
        gaelam[t] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam
    seg["tdlamret"] = seg["adv"] + seg["vpred"]
def add_vtarg_and_adv(seg, gamma, lam):
    """
    Compute target value using TD(lambda) estimator, and advantage with GAE(lambda)
    """
    new = np.append(seg["new"], 0) # last element is only used for last vtarg, but we already zeroed it if last new = 1
    vpred = np.append(seg["vpred"], seg["nextvpred"])
    T = len(seg["rew"])
    seg["adv"] = gaelam = np.empty(T, 'float32')
    rew = seg["rew"]
    lastgaelam = 0
    for t in reversed(range(T)):
        nonterminal = 1-new[t+1]
        delta = rew[t] + gamma * vpred[t+1] * nonterminal - vpred[t]
        gaelam[t] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam
    seg["tdlamret"] = seg["adv"] + seg["vpred"]
def add_vtarg_and_adv ( seg , gamma , lam ) : new = np . append ( seg [ "new" ] , 0 ) # last element is only used for last vtarg, but we already zeroed it if last new = 1 vpred = np . append ( seg [ "vpred" ] , seg [ "nextvpred" ] ) T = len ( seg [ "rew" ] ) seg [ "adv" ] = gaelam = np . empty ( T , 'float32' ) rew = seg [ "rew" ] lastgaelam = 0 for t in reversed ( range ( T ) ) : nonterminal = 1 - new [ t + 1 ] delta = rew [ t ] + gamma * vpred [ t + 1 ] * nonterminal - vpred [ t ] gaelam [ t ] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam seg [ "tdlamret" ] = seg [ "adv" ] + seg [ "vpred" ]
Compute target value using TD(lambda) estimator, and advantage with GAE(lambda)
Compute target value using TD ( lambda ) estimator and advantage with GAE ( lambda )