new sql

Author: Pengxiang-Li

examples/osworld/async/run_trainer_debug_w_rollout_stepwise_train_pt.sh

@@ -3,15 +3,21 @@ pip install cryptography
 
 set -x
 ENGINE=${1:-vllm_osworld}
-cd /root/verl
+
+cd /workspace/codes/verl
 
 # Initialize Ray cluster for multi-node training
 # Make sure Ray is running on all nodes before executing this script
 # On head node: ray start --head --port=6379
 # On worker nodes: ray start --address='head_node_ip:6379'
 # Detect number of GPUs on the current machine
 N_NODES=1
-N_GPUS=$(nvidia-smi --list-gpus | wc -l) 
+# Check if nvidia-smi is available and working
+if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then
+    N_GPUS=$(nvidia-smi --list-gpus | wc -l)
+else
+    N_GPUS=0
+fi
 N_GPUS_PER_NODE=$N_GPUS
 
 # # 生成带时间戳的唯一文件ID，后台运行
@@ -25,7 +31,7 @@ echo "To stop monitoring: kill $!"
 
 echo "Detected $N_GPUS GPUs on this machine"
 
-MODEL_PATH=/capacity/userdata/vcfenxd75jiv/shichenrui/ui_tars/ByteDance-Seed/UI-TARS-1.5
+MODEL_PATH=/workspace/huggingface/dart-gui-7b
 
 #/root/verl/checkpoints/verl_osworld_grpo/vllm_logp_pt_test5_w_KL_trainset15_osworld_reward_script_grpo_k8s_20250906_m3ou6di7/global_step_63/actor/huggingface
 
@@ -53,10 +59,10 @@ export SWAN_FS_GROUP_HOOK=https://open.feishu.cn/open-apis/bot/v2/hook/793155e5-
 # export ROOT_DATA_DIR=rollouter/results/pass16_20250825_train152_pass16_gpu4_env36
 # export RUN_ID=results/pass16_20250825_train152_pass16_gpu4_env36
 
-export ROOT_DATA_DIR=rollouter/results/pass8_20250904_train15_pass8_gpu2_env20_vllm_logp_maxstep15_tesl_vllm_logp_test6
-export RUN_ID=results/pass8_20250904_train15_pass8_gpu2_env20_vllm_logp_maxstep15_tesl_vllm_logp_test6
+export ROOT_DATA_DIR=pass32_uitars_0928
+export RUN_ID=pass32_uitars_0928
 # export EXPERIMENT_NAME=osworld_all_feasible_reward_script_grpo_k8s_20250821_vxer2wco
-export EXPERIMENT_NAME=vllm_logp_pt_test5_w_KL_trainset15_osworld_reward_script_grpo_k8s_$(date +%Y%m%d)_$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
+export EXPERIMENT_NAME=Fixed_$(date +%Y%m%d)_$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
 # export EXPERIMENT_NAME=vllm_logp_pt_test5_w_KL_trainset15_osworld_reward_script_grpo_k8s_20250906_m3ou6di7
 # export EXPERIMENT_NAME=pt_test5_w_KL_trainset15_vllm_logp_osworld_reward_script_grpo_k8s_20250905_91ww0y85
 # export EXPERIMENT_NAME=osworld_all_feasible_reward_script_grpo_k8s_20250827_2txpd14d
@@ -65,7 +71,7 @@ export EXPERIMENT_NAME=vllm_logp_pt_test5_w_KL_trainset15_osworld_reward_script_
 # export RUN_ID=pengxiang_test_0802_max_variance
 # export EXPERIMENT_NAME=osworld_all_feasible_reward_script_grpo_k8s_0802_8_mb64_micro8
 # export ROLLOUT_SERVER_URL=http://172.19.47.166:15959
-export ROLLOUT_SERVER_URL=http://172.19.171.243:15959
+export ROLLOUT_SERVER_URL=h0.0.0.0:8888
 
 # training parameters
 adv_estimator=grpo

pass32_uitars_0928

@@ -0,0 +1 @@
+/data/liuyang/pass32_osworldnew_tmp07/pass32_osworldnew_tmp07

rollouter/requirements.txt

@@ -0,0 +1,266 @@
+accelerate==1.6.0
+agentenv==0.0.1
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.18
+aiohttp-cors==0.8.1
+aiosignal==1.3.2
+airportsdata==20250224
+annotated-types==0.7.0
+anthropic==0.50.0
+antlr4-python3-runtime==4.9.3
+anyio==4.9.0
+astor==0.8.1
+asttokens==3.0.0
+async-timeout==5.0.1
+attrs==25.3.0
+av==15.0.0
+backcall==0.2.0
+backoff==2.2.1
+beautifulsoup4==4.13.4
+blake3==1.0.4
+bleach==6.2.0
+boto3==1.39.10
+botocore==1.39.10
+build==1.2.2.post1
+cachetools==5.5.2
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+cloudpickle==3.1.1
+codetiming==1.4.0
+colorful==0.5.6
+compressed-tensors==0.9.3
+cuda-bindings==12.8.0
+cuda-python==12.8.0
+cupy-cuda12x==13.4.1
+datasets==3.5.0
+decorator==5.2.1
+decord==0.6.0
+defusedxml==0.7.1
+deprecated==1.2.18
+depyf==0.18.0
+dill==0.3.8
+diskcache==5.6.3
+distlib==0.3.9
+distro==1.9.0
+dnspython==2.7.0
+docker-pycreds==0.4.0
+docopt==0.6.2
+einops==0.8.1
+email-validator==2.2.0
+exceptiongroup==1.2.2
+executing==2.2.0
+farama-notifications==0.0.4
+fastapi==0.115.12
+fastapi-cli==0.0.7
+fastjsonschema==2.21.1
+fastrlock==0.8.3
+filelock==3.18.0
+flashinfer-python==0.2.3
+frozenlist==1.6.0
+fsspec==2024.12.0
+gguf==0.16.3
+gitdb==4.0.12
+gitpython==3.1.44
+google-api-core==2.24.2
+google-auth==2.39.0
+googleapis-common-protos==1.70.0
+greenlet==3.2.3
+grpcio==1.71.0
+gymnasium==1.2.0
+h11==0.16.0
+hf-transfer==0.1.9
+hf-xet==1.0.5
+httpcore==1.0.9
+httptools==0.6.4
+httpx==0.28.1
+huggingface-hub==0.30.2
+hydra-core==1.3.2
+idna==3.10
+importlib-metadata==8.0.0
+interegular==0.3.3
+ipython==8.12.3
+jedi==0.19.2
+jinja2==3.1.6
+jiter==0.9.0
+jmespath==1.0.1
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+jupyter-client==8.6.3
+jupyter-core==5.8.1
+jupyterlab-pygments==0.3.0
+lark==1.2.2
+litellm==1.67.2
+llguidance==0.7.19
+llvmlite==0.44.0
+lm-format-enforcer==0.10.11
+markdown-it-py==3.0.0
+markupsafe==3.0.2
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mistral-common==1.5.4
+mistune==3.1.3
+modelscope==1.25.0
+mpmath==1.3.0
+msgpack==1.1.0
+msgspec==0.19.0
+multidict==6.4.3
+multiprocess==0.70.16
+nanobind==2.7.0
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.4
+numba==0.61.2
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cusparselt-cu12==0.6.2
+nvidia-ml-py==12.570.86
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
+nvitop==1.5.0
+omegaconf==2.3.0
+openai==1.76.0
+opencensus==0.11.4
+opencensus-context==0.1.3
+opencv-python-headless==4.11.0.86
+opentelemetry-api==1.26.0
+opentelemetry-exporter-otlp==1.26.0
+opentelemetry-exporter-otlp-proto-common==1.26.0
+opentelemetry-exporter-otlp-proto-grpc==1.26.0
+opentelemetry-exporter-otlp-proto-http==1.26.0
+opentelemetry-proto==1.26.0
+opentelemetry-sdk==1.26.0
+opentelemetry-semantic-conventions==0.47b0
+opentelemetry-semantic-conventions-ai==0.4.6
+orjson==3.10.16
+outlines==0.1.11
+outlines-core==0.1.26
+packaging==25.0
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+partial-json-parser==0.2.1.1.post5
+peft==0.15.2
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==11.2.1
+pip==25.2
+pip-tools==7.4.1
+pipreqs==0.5.0
+platformdirs==4.3.7
+prettytable==3.16.0
+prometheus-client==0.21.1
+prometheus-fastapi-instrumentator==7.1.0
+prompt-toolkit==3.0.51
+propcache==0.3.1
+proto-plus==1.26.1
+protobuf
+psutil==7.0.0
+ptyprocess==0.7.0
+pure-eval==0.2.3
+py-cpuinfo==9.0.0
+py-spy==0.4.0
+pyarrow==19.0.1
+pyasn1==0.6.1
+pyasn1-modules==0.4.2
+pybind11==2.13.6
+pycountry==24.6.1
+pycparser==2.22
+pydantic==2.11.3
+pydantic-core==2.33.1
+pyecharts==2.0.8
+pygments==2.19.1
+pylatexenc==2.10
+pymysql==1.1.1
+pynvml==12.0.0
+pyproject-hooks==1.2.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.0
+python-json-logger==3.3.0
+python-multipart==0.0.20
+pytz==2025.2
+pyyaml==6.0.2
+pyzmq==27.0.0
+qwen-vl-utils==0.0.11
+ray==2.43.0
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+rich==13.9.4
+rich-toolkit==0.14.3
+rpds-py==0.24.0
+rsa==4.9.1
+s3transfer==0.13.1
+safetensors==0.5.3
+scipy==1.15.2
+sentencepiece==0.2.0
+sentry-sdk==2.27.0
+setproctitle==1.3.5
+setuptools==80.9.0
+shellingham==1.5.4
+simplejson==3.20.1
+six
+smart-open==7.1.0
+smmap==5.0.2
+sniffio==1.3.1
+soundfile==0.13.1
+soupsieve==2.7
+sqlalchemy==2.0.41
+stack-data==0.6.3
+starlette==0.46.2
+swankit==0.2.4
+swanlab==0.6.7
+sympy==1.13.1
+tensordict==0.6.2
+tiktoken==0.9.0
+tinycss2==1.4.0
+tokenizers==0.21.1
+tomli==2.2.1
+torch==2.6.0
+torch-memory-saver==0.0.5
+torchao==0.10.0
+torchaudio==2.6.0
+torchdata==0.11.0
+torchvision==0.21.0
+tornado==6.5.1
+tqdm==4.67.1
+traitlets==5.14.3
+transformers==4.51.1
+triton==3.2.0
+typer==0.15.2
+typing-extensions==4.13.2
+typing-inspection==0.4.0
+tzdata==2025.2
+urllib3==2.4.0
+uvicorn==0.34.2
+uvloop==0.21.0
+verl==0.3.0.post1
+virtualenv==20.30.0
+vllm==0.8.5.post1
+wandb==0.19.10
+watchfiles==1.0.5
+wcwidth==0.2.13
+webencodings==0.5.1
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.17.2
+xformers==0.0.29.post2
+xgrammar==0.1.18
+xxhash==3.5.0
+yarg==0.1.9
+yarl==1.20.0
+zipp==3.21.0
+

sim_rollout_pass8.py

@@ -83,7 +83,7 @@ def simulate_rollout(
     limit: Optional[int] = None,
     dry_run: bool = False,
     loops = 10,
-    bootstrap_count = 256,
+    bootstrap_count = 2,
     delete_existing: bool = False,
 ) -> None:
     """Simulate rollout by inserting rows at a controlled, steady rate."""
@@ -109,6 +109,7 @@ def item_at(global_idx: int) -> Dict[str, Any]:
 
     # Initialize DB manager
     db_manager = create_database_manager()
+    print(db_manager.engine.url)  # 看连接到哪个库
 
     # Optionally clear previous rows for this run_id
     if delete_existing:
@@ -155,6 +156,8 @@ def item_at(global_idx: int) -> Dict[str, Any]:
 
             # 插入数据
             model_version = latest_model_version(db_manager, run_id)
+            if model_version == "":
+                model_version = "v0"
             payload = dict(
                 trajectory_id=item["trajectory_id"],
                 run_id=run_id,
@@ -201,15 +204,15 @@ def item_at(global_idx: int) -> Dict[str, Any]:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Simulated rollout producer for trainer unit tests.")
-    parser.add_argument("--json", default="data/train/data_pass@8_train90.json", help="Path to the static JSON data.")
+    parser.add_argument("--json", default="data/train/pass@32_90_trainingser.json", help="Path to the static JSON data.")
     # parser.add_argument("--run-id", default="pengxiang_test_0824_fixed_4_task", help="Run ID to write into DB rows.")
-    parser.add_argument("--run-id", default="pengxiang_test_0829_stepwise_pass8_multinode", help="Run ID to write into DB rows.")
-    parser.add_argument("--rate", type=int, default=26, help="Insert rate per minute.")
+    parser.add_argument("--run-id", default="pass32_uitars_0928", help="Run ID to write into DB rows.")
+    parser.add_argument("--rate", type=int, default=100, help="Insert rate per minute.")
     parser.add_argument("--start-index", type=int, default=0, help="Start from this index in the JSON list.")
     parser.add_argument("--limit", type=int, default=None, help="Only process this many items.")
     parser.add_argument("--dry-run", action="store_true", help="Don't write to DB; just print what would happen.")
     parser.add_argument("--loops", type=int, default=10000, help="Maximum number of full loops over the JSON (default: 10).")
-    parser.add_argument("--bootstrap", type=int, default=512, help="Number of items to insert immediately at startup (default: 256).")
+    parser.add_argument("--bootstrap", type=int, default=200, help="Number of items to insert immediately at startup (default: 256).")
     parser.add_argument(
         "--delete-existing",
         action="store_true",

verl/trainer/main_ppo_async.py

@@ -50,7 +50,7 @@ def run_ppo(config) -> None:
                 runtime_env={"env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN", "VLLM_LOGGING_LEVEL": "WARN", "VLLM_ALLOW_RUNTIME_LORA_UPDATING": "true"}},
                 num_cpus=config.ray_init.num_cpus,
                 dashboard_host="0.0.0.0",
-                object_store_memory=800 * 1024**3,  # 00GB for object store
+                object_store_memory=800 * 1024**3,  # 800GB for object store
             )
 
     # Create a remote instance of the TaskRunner class, and