add 'resources_per_worker' config for easily managing cpus/gpus of each env worker (#148)

agent_system/environments/env_manager.py

@@ -22,6 +22,7 @@ import os
 from agent_system.environments.prompts import *
 from agent_system.environments.base import EnvironmentManagerBase, to_numpy
 from agent_system.memory import SimpleMemory
+from omegaconf import OmegaConf
 
 def parse_gamefile(infos):
     gamefile = []
@@ -518,10 +519,12 @@ def make_envs(config):
     if not isinstance(config.env.rollout.n, int):
         raise ValueError("config.env.rollout.n should be an integer")
     group_n = config.env.rollout.n if config.env.rollout.n > 0 else 1
+    resources_per_worker = OmegaConf.to_container(config.env.resources_per_worker, resolve=True)
+
     if "gym_cards" in config.env.env_name.lower():
         from agent_system.environments.env_package.gym_cards import build_gymcards_envs, gym_projection
-        _envs = build_gymcards_envs(env_name=config.env.env_name, seed=config.env.seed, env_num=config.data.train_batch_size, group_n=group_n, is_train=True)
-        _val_envs = build_gymcards_envs(env_name=config.env.env_name, seed=config.env.seed + 1000, env_num=config.data.val_batch_size, group_n=1, is_train=False)
+        _envs = build_gymcards_envs(env_name=config.env.env_name, seed=config.env.seed, env_num=config.data.train_batch_size, group_n=group_n, is_train=True, resources_per_worker=resources_per_worker)
+        _val_envs = build_gymcards_envs(env_name=config.env.env_name, seed=config.env.seed + 1000, env_num=config.data.val_batch_size, group_n=1, is_train=False, resources_per_worker=resources_per_worker)
         
         projection_f = partial(gym_projection, env_name=config.env.env_name)
         envs = GymCardEnvironmentManager(_envs, projection_f, config)
@@ -539,8 +542,8 @@ def make_envs(config):
         env_kwargs = {
             'eval_dataset': 'eval_in_distribution', # 'eval_in_distribution' or 'eval_out_of_distribution'
         }
-        _envs = build_alfworld_envs(alf_config_path, config.env.seed, config.data.train_batch_size, group_n, is_train=True, env_kwargs=env_kwargs)
-        _val_envs = build_alfworld_envs(alf_config_path, config.env.seed + 1000, config.data.val_batch_size, 1, is_train=False, env_kwargs=env_kwargs)
+        _envs = build_alfworld_envs(alf_config_path, config.env.seed, config.data.train_batch_size, group_n, is_train=True, env_kwargs=env_kwargs, resources_per_worker=resources_per_worker)
+        _val_envs = build_alfworld_envs(alf_config_path, config.env.seed + 1000, config.data.val_batch_size, 1, is_train=False, env_kwargs=env_kwargs, resources_per_worker=resources_per_worker)
         
         projection_f = partial(alfworld_projection)
         envs = AlfWorldEnvironmentManager(_envs, projection_f, config)
@@ -554,8 +557,8 @@ def make_envs(config):
             'max_steps': config.env.max_steps,
             'search_depth': config.env.sokoban.search_depth
         }
-        _envs = build_sokoban_envs(config.env.seed, config.data.train_batch_size, group_n, mode=config.env.sokoban.mode, is_train=True, env_kwargs=env_kwargs)
-        _val_envs = build_sokoban_envs(config.env.seed + 1000, config.data.val_batch_size, 1, mode=config.env.sokoban.mode, is_train=False, env_kwargs=env_kwargs)
+        _envs = build_sokoban_envs(config.env.seed, config.data.train_batch_size, group_n, mode=config.env.sokoban.mode, is_train=True, env_kwargs=env_kwargs, resources_per_worker=resources_per_worker)
+        _val_envs = build_sokoban_envs(config.env.seed + 1000, config.data.val_batch_size, 1, mode=config.env.sokoban.mode, is_train=False, env_kwargs=env_kwargs, resources_per_worker=resources_per_worker)
         
         projection_f = partial(sokoban_projection)
         envs = SokobanEnvironmentManager(_envs, projection_f, config)
@@ -576,8 +579,8 @@ def make_envs(config):
                     'file_path': file_path,
                     'attr_path': attr_path
                     }
-        _envs = build_webshop_envs(seed=config.env.seed, env_num=config.data.train_batch_size, group_n=group_n, is_train=True, env_kwargs=env_kwargs)
-        _val_envs = build_webshop_envs(seed=config.env.seed + 1000, env_num=config.data.val_batch_size, group_n=1, is_train=False, env_kwargs=env_kwargs)
+        _envs = build_webshop_envs(seed=config.env.seed, env_num=config.data.train_batch_size, group_n=group_n, is_train=True, env_kwargs=env_kwargs, resources_per_worker=resources_per_worker)
+        _val_envs = build_webshop_envs(seed=config.env.seed + 1000, env_num=config.data.val_batch_size, group_n=1, is_train=False, env_kwargs=env_kwargs, resources_per_worker=resources_per_worker)
 
         projection_f = partial(webshop_projection)
         envs = WebshopEnvironmentManager(_envs, projection_f, config)
@@ -587,8 +590,8 @@ def make_envs(config):
         return envs, val_envs
     elif "appworld" in config.env.env_name.lower():
         from agent_system.environments.env_package.appworld import build_appworld_envs, appworld_projection
-        _envs = build_appworld_envs(dataset_name='train', seed=config.env.seed, env_num=config.data.train_batch_size, group_n=group_n, start_server_id=0)
-        _val_envs = build_appworld_envs(dataset_name='test_normal', seed=config.env.seed + 1000, env_num=config.data.val_batch_size, group_n=1, start_server_id=config.data.train_batch_size*group_n)
+        _envs = build_appworld_envs(dataset_name='train', seed=config.env.seed, env_num=config.data.train_batch_size, group_n=group_n, start_server_id=0, resources_per_worker=resources_per_worker)
+        _val_envs = build_appworld_envs(dataset_name='test_normal', seed=config.env.seed + 1000, env_num=config.data.val_batch_size, group_n=1, start_server_id=config.data.train_batch_size*group_n, resources_per_worker=resources_per_worker)
         
         projection_f = partial(appworld_projection)
         envs = AppWorldEnvironmentManager(_envs, projection_f, config)

agent_system/environments/env_package/alfworld/envs.py

@@ -52,7 +52,6 @@ def compute_reward(info, multi_modal=False):
         reward = 10.0 * float(info['won'])
     return reward
 
-@ray.remote(num_cpus=0.2)
 class AlfworldWorker:
     """
     Ray remote actor that replaces the worker function.
@@ -84,7 +83,7 @@ class AlfworldWorker:
         return image
 
 class AlfworldEnvs(gym.Env):
-    def __init__(self, alf_config_path, seed=0, env_num=1, group_n=1, is_train=True, env_kwargs={}):
+    def __init__(self, alf_config_path, seed, env_num, group_n, resources_per_worker, is_train=True, env_kwargs={}):
         super().__init__()
         
         # Initialize Ray if not already initialized
@@ -100,9 +99,10 @@ class AlfworldEnvs(gym.Env):
         self.group_n = group_n
 
         # Create Ray remote actors instead of processes
+        env_worker = ray.remote(**resources_per_worker)(AlfworldWorker)
         self.workers = []
         for i in range(self.num_processes):
-            worker = AlfworldWorker.remote(config, seed + (i // self.group_n), base_env)
+            worker = env_worker.remote(config, seed + (i // self.group_n), base_env)
             self.workers.append(worker)
 
         self.prev_admissible_commands = [None for _ in range(self.num_processes)]
@@ -202,5 +202,5 @@ class AlfworldEnvs(gym.Env):
         for worker in self.workers:
             ray.kill(worker)
 
-def build_alfworld_envs(alf_config_path, seed, env_num, group_n, is_train=True, env_kwargs={}):
-    return AlfworldEnvs(alf_config_path, seed, env_num, group_n, is_train, env_kwargs)
+def build_alfworld_envs(alf_config_path, seed, env_num, group_n, resources_per_worker, is_train=True, env_kwargs={}):
+    return AlfworldEnvs(alf_config_path, seed, env_num, group_n, resources_per_worker, is_train, env_kwargs)

agent_system/environments/env_package/appworld/envs.py

@@ -39,7 +39,6 @@ def load_available_ports(port_file="appworld_ports.ports"):
     
     return ports
 
-@ray.remote(num_cpus=0.1)
 class AppWorldWorker:
     """
     Ray Actor that holds an instance of AppWorld and operates the environment
@@ -115,6 +114,7 @@ class AppWorldEnvs:
                  env_num,
                  group_n,
                  start_server_id,
+                 resources_per_worker,
                  port_file="appworld_ports.ports"
                  ):
         super().__init__()
@@ -145,10 +145,11 @@ class AppWorldEnvs:
             ray.init()
 
         # Create Ray actors (workers)
+        env_worker = ray.remote(**resources_per_worker)(AppWorldWorker)
         self.workers = []
         for i in range(self.num_processes):
             port = self.available_ports[i]
-            worker = AppWorldWorker.remote(
+            worker = env_worker.remote(
                 worker_id=start_server_id + i,
                 max_interactions=self.max_interactions,
                 port=port
@@ -240,6 +241,7 @@ def build_appworld_envs(dataset_name="train",
                         env_num=1, 
                         group_n=1,
                         start_server_id=0,
+                        resources_per_worker={"num_cpus": 0.1},
                         ):
 
     return AppWorldEnvs(
@@ -249,4 +251,5 @@ def build_appworld_envs(dataset_name="train",
         env_num=env_num,
         group_n=group_n,
         start_server_id=start_server_id,
+        resources_per_worker=resources_per_worker
     )

agent_system/environments/env_package/gym_cards/envs.py

@@ -18,7 +18,7 @@ import ray
 import numpy as np
 from gym_cards.envs import Point24Env, EZPointEnv, BlackjackEnv, NumberLineEnv
 
-@ray.remote(num_cpus=0.2)
+
 class GymCardsWorker:
     """
     Ray remote actor that replaces the worker function.
@@ -66,6 +66,7 @@ class GymMultiProcessEnv(gym.Env):
                  seed=0,
                  env_num=1,
                  group_n=1,
+                 resources_per_worker={"num_cpus": 0.1},
                  is_train=True):
         super().__init__()
 
@@ -80,11 +81,12 @@ class GymMultiProcessEnv(gym.Env):
         self.num_processes = env_num * group_n
 
         np.random.seed(seed)
-
+    
         # Create Ray remote actors instead of processes
+        env_worker = ray.remote(**resources_per_worker)(GymCardsWorker)
         self.workers = []
         for _ in range(self.num_processes):
-            worker = GymCardsWorker.remote(self.env_id)
+            worker = env_worker.remote(self.env_id)
             self.workers.append(worker)
 
     def step(self, actions):
@@ -162,6 +164,7 @@ def build_gymcards_envs(env_name,
                         seed,
                         env_num,
                         group_n,
+                        resources_per_worker,
                         is_train=True):
     """
     Externally exposed constructor function to create parallel Gym environments.
@@ -176,5 +179,6 @@ def build_gymcards_envs(env_name,
         seed=seed,
         env_num=env_num,
         group_n=group_n,
+        resources_per_worker=resources_per_worker,
         is_train=is_train,
     )

agent_system/environments/env_package/sokoban/envs.py

@@ -18,7 +18,6 @@ import gym
 from agent_system.environments.env_package.sokoban.sokoban import SokobanEnv
 import numpy as np
 
-@ray.remote(num_cpus=0.2)
 class SokobanWorker:
     """
     Ray remote actor that replaces the worker function.
@@ -57,6 +56,7 @@ class SokobanMultiProcessEnv(gym.Env):
                  env_num=1, 
                  group_n=1, 
                  mode='rgb_array',
+                 resources_per_worker={"num_cpus": 0.1},
                  is_train=True,
                  env_kwargs=None):
         """
@@ -82,9 +82,10 @@ class SokobanMultiProcessEnv(gym.Env):
             env_kwargs = {}
 
         # Create Ray remote actors instead of processes
+        env_worker = ray.remote(**resources_per_worker)(SokobanWorker)
         self.workers = []
         for i in range(self.num_processes):
-            worker = SokobanWorker.remote(self.mode, env_kwargs)
+            worker = env_worker.remote(self.mode, env_kwargs)
             self.workers.append(worker)
 
     def step(self, actions):
@@ -178,6 +179,7 @@ def build_sokoban_envs(
         env_num=1,
         group_n=1,
         mode='rgb_array',
+        resources_per_worker={"num_cpus": 0.1},
         is_train=True,
         env_kwargs=None):
-    return SokobanMultiProcessEnv(seed, env_num, group_n, mode, is_train, env_kwargs=env_kwargs)
+    return SokobanMultiProcessEnv(seed, env_num, group_n, mode, resources_per_worker, is_train, env_kwargs=env_kwargs)

agent_system/environments/env_package/webshop/envs.py

@@ -21,7 +21,6 @@ import numpy as np
 # Ray remote worker actor -----------------------------------------------------
 # -----------------------------------------------------------------------------
 
-@ray.remote(num_cpus=0.2)
 class WebshopWorker:
     """Ray remote actor that replaces the worker function.
     Each actor hosts a *WebAgentTextEnv* instance.
@@ -94,9 +93,10 @@ class WebshopMultiProcessEnv(gym.Env):
     """
     def __init__(
         self,
-        seed: int = 0,
-        env_num: int = 1,
-        group_n: int = 1,
+        seed: int,
+        env_num: int,
+        group_n: int,
+        resources_per_worker: dict,
         is_train: bool = True,
         env_kwargs: dict = None,
     ) -> None:
@@ -117,10 +117,10 @@ class WebshopMultiProcessEnv(gym.Env):
         self._env_kwargs = env_kwargs if env_kwargs is not None else {'observation_mode': 'text', 'num_products': None}
 
         # -------------------------- Ray actors setup --------------------------
+        env_worker = ray.remote(**resources_per_worker)(WebshopWorker)
         self._workers = []
-
         for i in range(self.num_processes):
-            worker = WebshopWorker.remote(seed + (i // self.group_n), self._env_kwargs)
+            worker = env_worker.remote(seed + (i // self.group_n), self._env_kwargs)
             self._workers.append(worker)
 
         # Get goals from the first worker
@@ -239,9 +239,10 @@ class WebshopMultiProcessEnv(gym.Env):
 # -----------------------------------------------------------------------------
 
 def build_webshop_envs(
-    seed: int = 0,
-    env_num: int = 1,
-    group_n: int = 1,
+    seed: int,
+    env_num: int,
+    group_n: int,
+    resources_per_worker: dict,
     is_train: bool = True,
     env_kwargs: dict = None,
 ):
@@ -250,6 +251,7 @@ def build_webshop_envs(
         seed=seed,
         env_num=env_num,
         group_n=group_n,
+        resources_per_worker=resources_per_worker,
         is_train=is_train,
         env_kwargs=env_kwargs,
     )

examples/dapo_trainer/run_alfworld.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -60,6 +62,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=50 \
     env.rollout.n=${group_size} \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_alfworld' \

examples/dapo_trainer/run_webshop.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -60,6 +62,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=${group_size} \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

examples/gigpo_dynamic_trainer/run_alfworld.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -64,6 +66,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=50 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_alfworld' \

examples/gigpo_dynamic_trainer/run_sokoban.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=32
 val_data_size=128
 group_size=8
@@ -66,6 +68,7 @@ python3 -m verl.trainer.main_ppo \
     env.max_steps=15 \
     env.rollout.n=$group_size \
     env.sokoban.mode='rgb_array' \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_sokoban' \

examples/gigpo_dynamic_trainer/run_webshop.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -64,6 +66,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

examples/gigpo_trainer/run_alfworld.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=50 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_alfworld' \

examples/gigpo_trainer/run_alfworld_lora.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=50 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_alfworld' \

examples/gigpo_trainer/run_blackjack.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=32
 val_data_size=128
 group_size=8
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_blackjack' \

examples/gigpo_trainer/run_ezpoints.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=8 \
     env.rollout.n=${group_size} \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_ezpoints' \

examples/gigpo_trainer/run_numberline.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=10 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_numberLine' \

examples/gigpo_trainer/run_sokoban.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=32
 val_data_size=128
 group_size=8
@@ -57,6 +59,7 @@ python3 -m verl.trainer.main_ppo \
     env.max_steps=15 \
     env.rollout.n=$group_size \
     env.sokoban.mode='rgb_array' \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_sokoban' \

examples/gigpo_trainer/run_webshop.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

examples/gigpo_trainer/run_webshop_lora.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -57,6 +59,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

examples/gigpo_trainer/run_webshop_qwen3.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -54,6 +56,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

examples/grpo_trainer/run_alfworld.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -52,6 +54,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=50 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_alfworld' \

examples/grpo_trainer/run_balckjack.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=32
 val_data_size=128
 group_size=8
@@ -52,6 +54,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_blackjack' \

examples/grpo_trainer/run_sokoban.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=32
 val_data_size=128
 group_size=8
@@ -53,6 +55,7 @@ python3 -m verl.trainer.main_ppo \
     env.max_steps=15 \
     env.rollout.n=$group_size \
     env.sokoban.mode='rgb_array' \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_sokoban' \

examples/grpo_trainer/run_webshop.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=16
 val_data_size=128
 group_size=8
@@ -52,6 +54,7 @@ python3 -m verl.trainer.main_ppo \
     env.seed=0 \
     env.max_steps=15 \
     env.rollout.n=$group_size \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

examples/ppo_trainer/run_alfworld.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=128 # match GRPO and GiGPO configuration (16 × 8)
 val_data_size=128
 
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.env_name=alfworld/AlfredTWEnv \
     env.seed=0 \
     env.max_steps=50 \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_alfworld' \

examples/ppo_trainer/run_webshop.sh

@@ -2,6 +2,8 @@ set -x
 ENGINE=${1:-vllm}
 export VLLM_ATTENTION_BACKEND=XFORMERS
 
+num_cpus_per_env_worker=0.1 # The CPU resource allocated for each environment worker. If you want to use less CPU resources, you can decrease this value.
+
 train_data_size=128 # match GRPO and GiGPO configuration (16 × 8)
 val_data_size=128
 
@@ -56,6 +58,7 @@ python3 -m verl.trainer.main_ppo \
     env.env_name=Webshop \
     env.seed=0 \
     env.max_steps=15 \
+    env.resources_per_worker.num_cpus=$num_cpus_per_env_worker \
     trainer.critic_warmup=0 \
     trainer.logger=['console','wandb'] \
     trainer.project_name='verl_agent_webshop' \

verl/trainer/config/ppo_trainer.yaml

@@ -288,6 +288,9 @@ env:
   seed: 0
   max_steps: 50
   history_length: 2
+  resources_per_worker: # resources for each env worker
+    num_cpus: 0.1
+    num_gpus: 0
   rollout:
     n: -1 # the group number of envs (for GRPO and GiGPO). -1 means disable env grouping.