tensorforce/core/memories/prioritized_replay.py
def tf_retrieve_indices(self, buffer_elements, priority_indices):
        """
        Fetches experiences for given indices by combining entries from buffer
        which have no priorities, and entries from priority memory.

        Args:
            buffer_elements: Number of buffer elements to retrieve
            priority_indices: Index tensor for priority memory

        Returns: Batch of experiences
        """
        states = dict()
        buffer_start = self.buffer_index - buffer_elements
        buffer_end = self.buffer_index

        # Fetch entries from respective memories, concat.
        for name in sorted(self.states_memory):
            buffer_state_memory = self.states_buffer[name]
            # Slicing is more efficient than gathering, and buffer elements are always
            # fetched using contiguous indices.
            buffer_states = buffer_state_memory[buffer_start:buffer_end]
            # Memory indices are obtained via priority sampling, hence require gather.
            memory_states = tf.gather(params=self.states_memory[name], indices=priority_indices)
            states[name] = tf.concat(values=(buffer_states, memory_states), axis=0)

        internals = dict()
        for name in sorted(self.internals_memory):
            internal_buffer_memory = self.internals_buffer[name]
            buffer_internals = internal_buffer_memory[buffer_start:buffer_end]
            memory_internals = tf.gather(params=self.internals_memory[name], indices=priority_indices)
            internals[name] = tf.concat(values=(buffer_internals, memory_internals), axis=0)

        actions = dict()
        for name in sorted(self.actions_memory):
            action_buffer_memory = self.actions_buffer[name]
            buffer_action = action_buffer_memory[buffer_start:buffer_end]
            memory_action = tf.gather(params=self.actions_memory[name], indices=priority_indices)
            actions[name] = tf.concat(values=(buffer_action, memory_action), axis=0)

        buffer_terminal = self.terminal_buffer[buffer_start:buffer_end]
        priority_terminal = tf.gather(params=self.terminal_memory, indices=priority_indices)
        terminal = tf.concat(values=(buffer_terminal, priority_terminal), axis=0)

        buffer_reward = self.reward_buffer[buffer_start:buffer_end]
        priority_reward = tf.gather(params=self.reward_memory, indices=priority_indices)
        reward = tf.concat(values=(buffer_reward, priority_reward), axis=0)

        if self.include_next_states:
            assert util.rank(priority_indices) == 1
            next_priority_indices = (priority_indices + 1) % self.capacity
            next_buffer_start = (buffer_start + 1) % self.buffer_size
            next_buffer_end = (buffer_end + 1) % self.buffer_size

            next_states = dict()
            for name in sorted(self.states_memory):
                buffer_state_memory = self.states_buffer[name]
                buffer_next_states = buffer_state_memory[next_buffer_start:next_buffer_end]
                memory_next_states = tf.gather(params=self.states_memory[name], indices=next_priority_indices)
                next_states[name] = tf.concat(values=(buffer_next_states, memory_next_states), axis=0)

            next_internals = dict()
            for name in sorted(self.internals_memory):
                buffer_internal_memory = self.internals_buffer[name]
                buffer_next_internals = buffer_internal_memory[next_buffer_start:next_buffer_end]
                memory_next_internals = tf.gather(params=self.internals_memory[name], indices=next_priority_indices)
                next_internals[name] = tf.concat(values=(buffer_next_internals, memory_next_internals), axis=0)

            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals
            )
        else:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )
def tf_retrieve_indices(self, buffer_elements, priority_indices):
        """
        Fetches experiences for given indices by combining entries from buffer
        which have no priorities, and entries from priority memory.

        Args:
            buffer_elements: Number of buffer elements to retrieve
            priority_indices: Index tensor for priority memory

        Returns: Batch of experiences
        """
        states = dict()
        buffer_start = self.buffer_index - buffer_elements
        buffer_end = self.buffer_index

        # Fetch entries from respective memories, concat.
        for name in sorted(self.states_memory):
            buffer_state_memory = self.states_buffer[name]
            # Slicing is more efficient than gathering, and buffer elements are always
            # fetched using contiguous indices.
            buffer_states = buffer_state_memory[buffer_start:buffer_end]
            # Memory indices are obtained via priority sampling, hence require gather.
            memory_states = tf.gather(params=self.states_memory[name], indices=priority_indices)
            states[name] = tf.concat(values=(buffer_states, memory_states), axis=0)

        internals = dict()
        for name in sorted(self.internals_memory):
            internal_buffer_memory = self.internals_buffer[name]
            buffer_internals = internal_buffer_memory[buffer_start:buffer_end]
            memory_internals = tf.gather(params=self.internals_memory[name], indices=priority_indices)
            internals[name] = tf.concat(values=(buffer_internals, memory_internals), axis=0)

        actions = dict()
        for name in sorted(self.actions_memory):
            action_buffer_memory = self.actions_buffer[name]
            buffer_action = action_buffer_memory[buffer_start:buffer_end]
            memory_action = tf.gather(params=self.actions_memory[name], indices=priority_indices)
            actions[name] = tf.concat(values=(buffer_action, memory_action), axis=0)

        buffer_terminal = self.terminal_buffer[buffer_start:buffer_end]
        priority_terminal = tf.gather(params=self.terminal_memory, indices=priority_indices)
        terminal = tf.concat(values=(buffer_terminal, priority_terminal), axis=0)

        buffer_reward = self.reward_buffer[buffer_start:buffer_end]
        priority_reward = tf.gather(params=self.reward_memory, indices=priority_indices)
        reward = tf.concat(values=(buffer_reward, priority_reward), axis=0)

        if self.include_next_states:
            assert util.rank(priority_indices) == 1
            next_priority_indices = (priority_indices + 1) % self.capacity
            next_buffer_start = (buffer_start + 1) % self.buffer_size
            next_buffer_end = (buffer_end + 1) % self.buffer_size

            next_states = dict()
            for name in sorted(self.states_memory):
                buffer_state_memory = self.states_buffer[name]
                buffer_next_states = buffer_state_memory[next_buffer_start:next_buffer_end]
                memory_next_states = tf.gather(params=self.states_memory[name], indices=next_priority_indices)
                next_states[name] = tf.concat(values=(buffer_next_states, memory_next_states), axis=0)

            next_internals = dict()
            for name in sorted(self.internals_memory):
                buffer_internal_memory = self.internals_buffer[name]
                buffer_next_internals = buffer_internal_memory[next_buffer_start:next_buffer_end]
                memory_next_internals = tf.gather(params=self.internals_memory[name], indices=next_priority_indices)
                next_internals[name] = tf.concat(values=(buffer_next_internals, memory_next_internals), axis=0)

            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals
            )
        else:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )
def tf_retrieve_indices ( self , buffer_elements , priority_indices ) : states = dict ( ) buffer_start = self . buffer_index - buffer_elements buffer_end = self . buffer_index # Fetch entries from respective memories, concat. for name in sorted ( self . states_memory ) : buffer_state_memory = self . states_buffer [ name ] # Slicing is more efficient than gathering, and buffer elements are always # fetched using contiguous indices. buffer_states = buffer_state_memory [ buffer_start : buffer_end ] # Memory indices are obtained via priority sampling, hence require gather. memory_states = tf . gather ( params = self . states_memory [ name ] , indices = priority_indices ) states [ name ] = tf . concat ( values = ( buffer_states , memory_states ) , axis = 0 ) internals = dict ( ) for name in sorted ( self . internals_memory ) : internal_buffer_memory = self . internals_buffer [ name ] buffer_internals = internal_buffer_memory [ buffer_start : buffer_end ] memory_internals = tf . gather ( params = self . internals_memory [ name ] , indices = priority_indices ) internals [ name ] = tf . concat ( values = ( buffer_internals , memory_internals ) , axis = 0 ) actions = dict ( ) for name in sorted ( self . actions_memory ) : action_buffer_memory = self . actions_buffer [ name ] buffer_action = action_buffer_memory [ buffer_start : buffer_end ] memory_action = tf . gather ( params = self . actions_memory [ name ] , indices = priority_indices ) actions [ name ] = tf . concat ( values = ( buffer_action , memory_action ) , axis = 0 ) buffer_terminal = self . terminal_buffer [ buffer_start : buffer_end ] priority_terminal = tf . gather ( params = self . terminal_memory , indices = priority_indices ) terminal = tf . concat ( values = ( buffer_terminal , priority_terminal ) , axis = 0 ) buffer_reward = self . reward_buffer [ buffer_start : buffer_end ] priority_reward = tf . gather ( params = self . reward_memory , indices = priority_indices ) reward = tf . concat ( values = ( buffer_reward , priority_reward ) , axis = 0 ) if self . include_next_states : assert util . rank ( priority_indices ) == 1 next_priority_indices = ( priority_indices + 1 ) % self . capacity next_buffer_start = ( buffer_start + 1 ) % self . buffer_size next_buffer_end = ( buffer_end + 1 ) % self . buffer_size next_states = dict ( ) for name in sorted ( self . states_memory ) : buffer_state_memory = self . states_buffer [ name ] buffer_next_states = buffer_state_memory [ next_buffer_start : next_buffer_end ] memory_next_states = tf . gather ( params = self . states_memory [ name ] , indices = next_priority_indices ) next_states [ name ] = tf . concat ( values = ( buffer_next_states , memory_next_states ) , axis = 0 ) next_internals = dict ( ) for name in sorted ( self . internals_memory ) : buffer_internal_memory = self . internals_buffer [ name ] buffer_next_internals = buffer_internal_memory [ next_buffer_start : next_buffer_end ] memory_next_internals = tf . gather ( params = self . internals_memory [ name ] , indices = next_priority_indices ) next_internals [ name ] = tf . concat ( values = ( buffer_next_internals , memory_next_internals ) , axis = 0 ) return dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward , next_states = next_states , next_internals = next_internals ) else : return dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward )
Fetches experiences for given indices by combining entries from buffer
        which have no priorities, and entries from priority memory.

        Args:
            buffer_elements: Number of buffer elements to retrieve
            priority_indices: Index tensor for priority memory

        Returns: Batch of experiences
Fetches experiences for given indices by combining entries from buffer which have no priorities and entries from priority memory .