diff --git a/examples/rllib.ipynb b/examples/rllib.ipynb index 98434dd..fecd12d 100644 --- a/examples/rllib.ipynb +++ b/examples/rllib.ipynb @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "collapsed": false, "jupyter": { @@ -66,109 +66,17 @@ "name": "#%%\n" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: mobile-env in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (2.0.3)\n", - "Requirement already satisfied: gymnasium<1.0.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (0.28.1)\n", - "Requirement already satisfied: matplotlib in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (3.9.2)\n", - "Requirement already satisfied: numpy in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (1.26.4)\n", - "Requirement already satisfied: pandas in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (2.2.3)\n", - "Requirement already satisfied: pygame in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (2.6.1)\n", - "Requirement already satisfied: shapely in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (2.0.6)\n", - "Requirement already satisfied: svgpath2mpl in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from mobile-env) (1.0.0)\n", - "Requirement already satisfied: jax-jumpy>=1.0.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium<1.0.0->mobile-env) (1.0.0)\n", - "Requirement already satisfied: cloudpickle>=1.2.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium<1.0.0->mobile-env) (3.1.0)\n", - "Requirement already satisfied: typing-extensions>=4.3.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium<1.0.0->mobile-env) (4.12.2)\n", - "Requirement already satisfied: farama-notifications>=0.0.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium<1.0.0->mobile-env) (0.0.4)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (1.3.0)\n", - "Requirement already satisfied: cycler>=0.10 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (4.54.1)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (1.4.7)\n", - "Requirement already satisfied: packaging>=20.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (24.1)\n", - "Requirement already satisfied: pillow>=8 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (11.0.0)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (3.2.0)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from matplotlib->mobile-env) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from pandas->mobile-env) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from pandas->mobile-env) (2024.2)\n", - "Requirement already satisfied: six>=1.5 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from python-dateutil>=2.7->matplotlib->mobile-env) (1.16.0)\n", - "Note: you may need to restart the kernel to use updated packages.\n", - "Requirement already satisfied: ray==2.38.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (2.38.0)\n", - "Requirement already satisfied: torch in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (2.5.1)\n", - "Requirement already satisfied: tensorboard in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (2.18.0)\n", - "Requirement already satisfied: click>=7.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (8.1.7)\n", - "Requirement already satisfied: filelock in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (3.16.1)\n", - "Requirement already satisfied: jsonschema in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (4.23.0)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (1.1.0)\n", - "Requirement already satisfied: packaging in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (24.1)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (5.28.3)\n", - "Requirement already satisfied: pyyaml in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (6.0.2)\n", - "Requirement already satisfied: aiosignal in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (1.5.0)\n", - "Requirement already satisfied: requests in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray==2.38.0->ray[rllib]==2.38.0) (2.32.3)\n", - "Requirement already satisfied: pandas in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (2.2.3)\n", - "Requirement already satisfied: tensorboardX>=1.9 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (2.6.2.2)\n", - "Requirement already satisfied: pyarrow>=6.0.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (18.0.0)\n", - "Requirement already satisfied: fsspec in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (2024.10.0)\n", - "Requirement already satisfied: dm-tree in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (0.1.8)\n", - "Requirement already satisfied: gymnasium==0.28.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (0.28.1)\n", - "Requirement already satisfied: lz4 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (4.3.3)\n", - "Requirement already satisfied: scikit-image in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (0.24.0)\n", - "Requirement already satisfied: scipy in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (1.14.1)\n", - "Requirement already satisfied: typer in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (0.12.5)\n", - "Requirement already satisfied: rich in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from ray[rllib]==2.38.0) (13.9.4)\n", - "Requirement already satisfied: numpy>=1.21.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium==0.28.1->ray[rllib]==2.38.0) (1.26.4)\n", - "Requirement already satisfied: jax-jumpy>=1.0.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium==0.28.1->ray[rllib]==2.38.0) (1.0.0)\n", - "Requirement already satisfied: cloudpickle>=1.2.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium==0.28.1->ray[rllib]==2.38.0) (3.1.0)\n", - "Requirement already satisfied: typing-extensions>=4.3.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium==0.28.1->ray[rllib]==2.38.0) (4.12.2)\n", - "Requirement already satisfied: farama-notifications>=0.0.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from gymnasium==0.28.1->ray[rllib]==2.38.0) (0.0.4)\n", - "Requirement already satisfied: networkx in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from torch) (3.4.2)\n", - "Requirement already satisfied: jinja2 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from torch) (3.1.4)\n", - "Requirement already satisfied: setuptools in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from torch) (75.3.0)\n", - "Requirement already satisfied: sympy==1.13.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from torch) (1.13.1)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from sympy==1.13.1->torch) (1.3.0)\n", - "Requirement already satisfied: absl-py>=0.4 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from tensorboard) (2.1.0)\n", - "Requirement already satisfied: grpcio>=1.48.2 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from tensorboard) (1.67.1)\n", - "Requirement already satisfied: markdown>=2.6.8 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from tensorboard) (3.7)\n", - "Requirement already satisfied: six>1.9 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from tensorboard) (1.16.0)\n", - "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from tensorboard) (0.7.2)\n", - "Requirement already satisfied: werkzeug>=1.0.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from tensorboard) (3.1.1)\n", - "Requirement already satisfied: MarkupSafe>=2.1.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard) (3.0.2)\n", - "Requirement already satisfied: attrs>=22.2.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from jsonschema->ray==2.38.0->ray[rllib]==2.38.0) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from jsonschema->ray==2.38.0->ray[rllib]==2.38.0) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from jsonschema->ray==2.38.0->ray[rllib]==2.38.0) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from jsonschema->ray==2.38.0->ray[rllib]==2.38.0) (0.20.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from pandas->ray[rllib]==2.38.0) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from pandas->ray[rllib]==2.38.0) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from pandas->ray[rllib]==2.38.0) (2024.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from requests->ray==2.38.0->ray[rllib]==2.38.0) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from requests->ray==2.38.0->ray[rllib]==2.38.0) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from requests->ray==2.38.0->ray[rllib]==2.38.0) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from requests->ray==2.38.0->ray[rllib]==2.38.0) (2024.8.30)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from rich->ray[rllib]==2.38.0) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from rich->ray[rllib]==2.38.0) (2.18.0)\n", - "Requirement already satisfied: pillow>=9.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from scikit-image->ray[rllib]==2.38.0) (11.0.0)\n", - "Requirement already satisfied: imageio>=2.33 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from scikit-image->ray[rllib]==2.38.0) (2.36.0)\n", - "Requirement already satisfied: tifffile>=2022.8.12 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from scikit-image->ray[rllib]==2.38.0) (2024.9.20)\n", - "Requirement already satisfied: lazy-loader>=0.4 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from scikit-image->ray[rllib]==2.38.0) (0.4)\n", - "Requirement already satisfied: shellingham>=1.3.0 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from typer->ray[rllib]==2.38.0) (1.5.4)\n", - "Requirement already satisfied: mdurl~=0.1 in /Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->ray[rllib]==2.38.0) (0.1.2)\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "outputs": [], "source": [ "# install mobile-env\n", "%pip install -U mobile-env\n", "# install ray RLlib\n", - "%pip install \"ray[rllib]==2.38.0\" torch tensorboard" + "%pip install \"ray[rllib]==2.53.0\" torch tensorboard" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "collapsed": false, "jupyter": { @@ -211,20 +119,24 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2024-11-17 22:09:07,509\tINFO worker.py:1816 -- Started a local Ray instance.\n" + "/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/_private/client_mode_hook.py:104: FutureWarning: `local_mode` is an experimental feature that is no longer maintained and will be removed in the near future. For debugging consider using the Ray distributed debugger.\n", + " return func(*args, **kwargs)\n", + "2026-01-16 11:35:46,197\tINFO worker.py:2007 -- Started a local Ray instance.\n", + "/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/_private/worker.py:2046: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0\n", + " warnings.warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "528c7fce5b4648f99f61495a3af85f3c", + "model_id": "dc6ff312520b45e688b8fbdba1974e42", "version_major": 2, "version_minor": 0 }, @@ -248,11 +160,11 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", "
Python version:3.12.63.11.13
Ray version:2.38.02.53.0
\n", @@ -261,7 +173,7 @@ "\n" ], "text/plain": [ - "RayContext(dashboard_url='', python_version='3.12.6', ray_version='2.38.0', ray_commit='385ee466260ef3cd218d5e372aef5d39338b7b94')" + "RayContext(dashboard_url='', python_version='3.11.13', ray_version='2.53.0', ray_commit='0de211850589aea71f842873bc32574c702ab492')" ] }, "execution_count": 3, @@ -272,7 +184,7 @@ "source": [ "import ray\n", "\n", - "num_cpus: int = 2\n", + "num_cpus: int = 1\n", "\n", "# Init ray with available CPUs (and GPUs) and init ray.\n", "# For local debugging, set num_cpus=0 and local_mode=True. \n", @@ -281,6 +193,7 @@ " include_dashboard=False,\n", " ignore_reinit_error=True,\n", " log_to_driver=False,\n", + " local_mode=True,\n", ")" ] }, @@ -291,72 +204,7 @@ "outputs": [ { "data": { - "text/html": [ - "
\n", - "
\n", - "
\n", - "

Tune Status

\n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
Current time:2024-11-17 22:09:39
Running for: 00:00:29.46
Memory: 18.8/32.0 GiB
\n", - "
\n", - "
\n", - "
\n", - "

System Info

\n", - " Using FIFO scheduling algorithm.
Logical resource usage: 2.0/2 CPUs, 0/0 GPUs\n", - "
\n", - " \n", - "
\n", - "
\n", - "
\n", - "

Trial Status

\n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
Trial name status loc iter total time (s) ts num_healthy_workers num_in_flight_async_\n", - "sample_reqs num_remote_worker_re\n", - "starts
PPO_mobile-small-ma-v0_3029c_00000TERMINATED127.0.0.1:19039 1 20.83824000 100
\n", - "
\n", - "
\n", - "\n" - ], + "text/html": [], "text/plain": [ "" ] @@ -368,8 +216,113 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-11-17 22:09:39,186\tINFO tune.py:1009 -- Wrote the latest version of all result files and experiment state to '/Users/stefanshschneider/ray_results/PPO_2024-11-17_22-09-09' in 0.0187s.\n", - "2024-11-17 22:09:39,733\tINFO tune.py:1041 -- Total run time: 30.05 seconds (29.44 seconds for the tuning loop).\n" + "2026-01-16 11:35:48,934\tWARNING algorithm_config.py:5118 -- You are running PPO on the new API stack! This is the new default behavior for this algorithm. If you don't want to use the new API stack, set `config.api_stack(enable_rl_module_and_learner=False,enable_env_runner_and_connector_v2=False)`. For a detailed migration guide, see here: https://docs.ray.io/en/master/rllib/new-api-stack-migration-guide.html\n", + ":job_id:01000000\n", + ":task_name:bundle_reservation_check_func\n", + ":actor_name:PPO\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":job_id:01000000\n", + ":task_name:bundle_reservation_check_func\n", + ":actor_name:PPO\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/pygame/pkgdata.py:25: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", + " from pkg_resources import resource_stream, resource_exists\n", + "2026-01-16 11:35:49,819\tWARNING rl_module.py:459 -- DeprecationWarning: `RLModule(config=[RLModuleConfig object])` has been deprecated. Use `RLModule(observation_space=.., action_space=.., inference_only=.., model_config=.., catalog_class=..)` instead. This will raise an error in the future!\n", + "2026-01-16 11:35:50,778\tWARNING util.py:61 -- Install gputil for GPU system monitoring.\n", + ":actor_name:PPO(env=mobile-small-ma-v0; env-runners=0; learners=0; multi-agent=True)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":actor_name:PPO(env=mobile-small-ma-v0; env-runners=0; learners=0; multi-agent=True)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-16 11:35:51,037\tERROR tune_controller.py:1331 -- Trial task failed for trial PPO_mobile-small-ma-v0_1f7bc_00000\n", + "Traceback (most recent call last):\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/air/execution/_internal/event_manager.py\", line 110, in resolve_future\n", + " result = ray.get(future)\n", + " ^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/_private/auto_init_hook.py\", line 22, in auto_init_wrapper\n", + " return fn(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/_private/client_mode_hook.py\", line 104, in wrapper\n", + " return func(*args, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/_private/worker.py\", line 2967, in get\n", + " values, debugger_breakpoint = worker.get_objects(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/_private/worker.py\", line 1015, in get_objects\n", + " raise value.as_instanceof_cause()\n", + "ray.exceptions.RayTaskError(MultiAgentEnvError): \u001b[36mray::PPO.train()\u001b[39m (pid=18254, ip=127.0.0.1, actor_id=0833be87dc048bc3bff8671601000000, repr=PPO(env=mobile-small-ma-v0; env-runners=0; learners=0; multi-agent=True))\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/tune/trainable/trainable.py\", line 331, in train\n", + " raise skipped from exception_cause(skipped)\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/tune/trainable/trainable.py\", line 328, in train\n", + " result = self.step()\n", + " ^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py\", line 1242, in step\n", + " train_results, train_iter_ctx = self._run_one_training_iteration()\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py\", line 3666, in _run_one_training_iteration\n", + " training_step_return_value = self.training_step()\n", + " ^^^^^^^^^^^^^^^^^^^^\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/algorithms/ppo/ppo.py\", line 410, in training_step\n", + " episodes, env_runner_results = synchronous_parallel_sample(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/execution/rollout_ops.py\", line 102, in synchronous_parallel_sample\n", + " sampled_data = [worker_set.local_env_runner.sample(**random_action_kwargs)]\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/env/multi_agent_env_runner.py\", line 222, in sample\n", + " samples = self._sample(\n", + " ^^^^^^^^^^^^^\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/env/multi_agent_env_runner.py\", line 405, in _sample\n", + " episodes[env_index].add_env_step(\n", + " File \"/Users/stefanshschneider/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/env/multi_agent_episode.py\", line 544, in add_env_step\n", + " raise MultiAgentEnvError(\n", + "ray.rllib.utils.error.MultiAgentEnvError: Agent 0 acted and then got truncated, but did NOT receive a last (truncation) observation, required for e.g. value function bootstrapping!\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026-01-16 11:35:51,526\tINFO tune.py:1009 -- Wrote the latest version of all result files and experiment state to '/Users/stefanshschneider/ray_results/PPO_2026-01-16_11-35-48' in 0.0401s.\n", + "2026-01-16 11:35:51,529\tERROR tune.py:1037 -- Trials did not complete: [PPO_mobile-small-ma-v0_1f7bc_00000]\n", + "2026-01-16 11:35:51,530\tINFO tune.py:1041 -- Total run time: 2.70 seconds (2.55 seconds for the tuning loop).\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -387,7 +340,7 @@ " # Here, we configure all agents to share the same policy.\n", " .multi_agent(\n", " policies={\"shared_policy\": PolicySpec()},\n", - " policy_mapping_fn=lambda agent_id, episode, worker, **kwargs: \"shared_policy\",\n", + " policy_mapping_fn=lambda agent_id, episode, **kwargs: \"shared_policy\",\n", " )\n", " # RLlib needs 1 CPU for the driver/trainer. The rest can be used for env runners.\n", " .env_runners(num_env_runners=num_cpus - 1, num_cpus_per_env_runner=1)\n", @@ -426,39 +379,19 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "Reusing TensorBoard on port 6006 (pid 10623), started 7:29:55 ago. (Use '!kill 10623' to kill it.)" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stderr", + "output_type": "stream", + "text": [ + "AttributeError: 'NoneType' object has no attribute 'update'\n" + ] }, { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stdout", + "output_type": "stream", + "text": [ + "Launching TensorBoard...\n", + "Please visit http://localhost:6006 in a web browser.\n" + ] } ], "source": [ @@ -485,34 +418,24 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages/ray/rllib/algorithms/algorithm.py:568: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n", - "`UnifiedLogger` will be removed in Ray 2.7.\n", - " return UnifiedLogger(config, logdir, loggers=None)\n", - "/Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n", - "The `JsonLogger interface is deprecated in favor of the `ray.tune.json.JsonLoggerCallback` interface and will be removed in Ray 2.7.\n", - " self._loggers.append(cls(self.config, self.logdir, self.trial))\n", - "/Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n", - "The `CSVLogger interface is deprecated in favor of the `ray.tune.csv.CSVLoggerCallback` interface and will be removed in Ray 2.7.\n", - " self._loggers.append(cls(self.config, self.logdir, self.trial))\n", - "/Users/stefanshschneider/Projects/private/mobile-env/venv/lib/python3.12/site-packages/ray/tune/logger/unified.py:53: RayDeprecationWarning: This API is deprecated and may be removed in future Ray releases. You could suppress this warning by setting env variable PYTHONWARNINGS=\"ignore::DeprecationWarning\"\n", - "The `TBXLogger interface is deprecated in favor of the `ray.tune.tensorboardx.TBXLoggerCallback` interface and will be removed in Ray 2.7.\n", - " self._loggers.append(cls(self.config, self.logdir, self.trial))\n" + "AttributeError: 'NoneType' object has no attribute 'update'\n" ] }, { - "data": { - "text/plain": [ - "Result(\n", - " metrics={'custom_metrics': {}, 'episode_media': {}, 'info': {'learner': {'shared_policy': {'learner_stats': {'allreduce_latency': 0.0, 'grad_gnorm': 1.6213738948992609, 'cur_kl_coeff': 0.2, 'cur_lr': 5.0000000000000016e-05, 'total_loss': 8.009701963493525, 'policy_loss': -0.008299151918210184, 'vf_loss': 8.015849697615185, 'vf_explained_var': 0.04391117921047656, 'kl': 0.01075709559530967, 'entropy': 1.3758156238847477, 'entropy_coeff': 0.0}, 'model': {}, 'custom_metrics': {}, 'num_agent_steps_trained': 127.38853503184713, 'num_grad_updates_lifetime': 2355.5, 'diff_num_grad_updates_vs_sampler_policy': 2354.5}}, 'num_env_steps_sampled': 4000, 'num_env_steps_trained': 4000, 'num_agent_steps_sampled': 20000, 'num_agent_steps_trained': 20000}, 'env_runners': {'episode_reward_max': -2.966881031323917, 'episode_reward_min': -341.7161232310376, 'episode_reward_mean': -187.20572839930978, 'episode_len_mean': 100.0, 'episode_media': {}, 'episodes_timesteps_total': 4000, 'policy_reward_min': {'shared_policy': -90.87986640184575}, 'policy_reward_max': {'shared_policy': 5.369115924522074}, 'policy_reward_mean': {'shared_policy': -37.441145679861954}, 'custom_metrics': {}, 'hist_stats': {'episode_reward': [-341.7161232310376, -210.10479259347105, -280.2208090574522, -172.34659666672397, -296.5657717575985, -166.24725216132202, -199.35735777174065, -223.4849183806289, -68.99070997828451, -203.2493766961578, -161.9034347627089, -193.38030606269552, -156.86632230206808, -171.03194135163199, -197.26961971176652, -161.9014008110669, -226.83900063103283, -258.6117167543199, -245.24433781916883, -212.83897613153187, -156.59672053114303, -2.966881031323917, -163.34609209515062, -139.52202385843694, -243.17107560425902, -219.8687089505345, -131.60110163391084, -91.21862899962417, -141.12357268091293, -180.72687099816667, -205.37555769979545, -290.7562339904011, -265.0984697032583, -78.111949858501, -110.88002408892798, -215.32274560392557, -204.89239800599358, -177.1517175280564, -137.30632788771996, -185.0212705899408], 'episode_lengths': [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100], 'policy_shared_policy_reward': [-75.97376292526073, -55.70188255003805, -60.550893308217304, -77.62041509910816, -71.86916934841331, -61.208364563366565, -43.95764300794533, -26.365322337227756, -17.14514742354721, -61.42831526138406, -81.2820086104112, -56.1438656055417, -28.647571321033976, -60.10606791263098, -54.04129560783462, -28.46998459838941, -58.63935862437817, -38.51265772682591, -31.494795542965775, -15.229800174164643, -57.69939029964555, -56.38478494383098, -53.38873667085906, -74.83473591625346, -54.25812392700937, -44.17057105155368, -19.38297212981529, -22.447590004221805, -26.243521857159024, -54.002597118572346, -49.776872666641, -15.907262620114796, -41.59078339173041, -26.01765372666303, -66.06478536659144, -44.7113374630515, -57.56318914672269, -65.9855046742508, -29.95556847878577, -25.269318617817902, -17.972544549388882, -16.634077926255628, -1.1939891445548616, -33.74458476009921, 0.554486402014063, -33.37317107660478, -25.131440840330622, -73.22265187712725, -39.667383379228035, -31.854729522867117, -49.74153538512278, -21.663601979990823, -42.115258992611174, -12.433170452541424, -35.94986795244273, -51.45034724820258, -21.758297997171862, -70.93040867938883, -39.3858501581646, -9.855401979767663, -50.179085996540806, -28.91682048666958, -29.42490437855196, -35.749361394617665, -12.596150045687988, -48.081331172853446, -46.08858671548431, -22.48500921886698, -27.52214187663568, -26.85487236779168, -40.91696033389797, -32.070355422912904, -35.07694655718345, -60.58767350859549, -28.617683889176686, -23.995020595096115, -21.563521577307224, -29.586049729379667, -58.298961194779054, -28.45784771450478, -49.45721997211554, -32.90268144820824, -62.45383562328173, -16.52633868230682, -65.49892490512049, -71.07145990097465, -49.41098758754849, -34.475792306276745, -51.08985707055336, -52.563619888966315, -36.88369983919451, -47.66704058098666, -68.95923086892174, -50.133234034866604, -41.60113249519934, -39.73737163503491, -28.7213726557908, -45.426866459826705, -70.64948789620209, -28.303877484677265, -32.689526783126375, -52.707779168618785, -36.674089342453655, -12.455842568861625, -22.06948266808256, -0.7104497026480319, 2.949574050941887, 1.7772407699397685, 5.369115924522074, -12.352362074079611, -30.472901016884194, -18.572605951977753, -10.414256792648992, -47.707620800694606, -56.17870753294515, -7.061947121528968, -15.945812150186653, -20.109767173998147, -64.33200632568912, -32.07249108703408, -66.1029676764273, -37.31577402210522, -70.33536439092276, -35.99671422135832, -33.420255293445585, -52.54951853999192, -24.753011060695616, -10.486785892070325, -54.85546945726677, -77.22392400050968, -35.75755055205849, -37.38447699678598, -15.620989061777475, -44.156791494161794, 1.3187064708729692, -32.87415477472307, -11.948652777664567, -17.685064214342777, -27.841745489820152, -0.8690117430736359, -22.588565771531183, -22.924099865812646, -19.587176439232966, -50.3251253311784, -25.6986052731577, -31.166241667648034, -62.51390549794251, -29.036164018646975, -35.82792166732255, -22.182638146606834, -45.595820524068735, -20.901029548934833, -90.87986640184575, -25.831293432341926, -22.167547792604132, -62.689527022778215, -44.448903389348054, -66.41952210274167, -56.00587973896291, -61.192401736570766, -57.74974644339343, -81.7130380097914, -32.95753905338611, -45.14689942116229, -47.53124677552484, -11.745168471397992, -41.909006615648686, -12.19320012745216, -6.543017606664212, -5.721557037337991, -23.736509907690536, -15.182759852910305, -27.148944241951504, -11.647881561340292, -33.16392852503545, -76.10121889360776, -34.61493725183212, -33.74338739756905, -37.24963733184732, -33.61356472906914, -49.54711600665791, -35.252489128799205, -35.7585491182765, -44.80204034403909, -39.532203408220774, -52.687061815037346, -58.19741846381604, -25.281812661360302, -13.2848811722991, -27.70054341554363, -32.78560653001561, -9.58385479427104, -42.3163114226674, -38.11716535084875, -14.503389789917142, -38.1679669833946, -68.81546866730723, -32.79968835471966, -24.544512392451168, -20.693634192068064]}, 'sampler_perf': {'mean_raw_obs_processing_ms': 0.3864632282099763, 'mean_inference_ms': 0.35320356827144533, 'mean_action_processing_ms': 0.11701424161781819, 'mean_env_wait_ms': 1.281081065926603, 'mean_env_render_ms': 0.0}, 'num_faulty_episodes': 0, 'connector_metrics': {'ObsPreprocessorConnector_ms': 0.002974271774291992, 'StateBufferConnector_ms': 0.0027894973754882812, 'ViewRequirementAgentConnector_ms': 0.11682748794555664}, 'num_episodes': 40, 'episode_return_max': -2.966881031323917, 'episode_return_min': -341.7161232310376, 'episode_return_mean': -187.20572839930978, 'episodes_this_iter': 40}, 'num_healthy_workers': 1, 'num_in_flight_async_sample_reqs': 0, 'num_remote_worker_restarts': 0, 'num_agent_steps_sampled': 20000, 'num_agent_steps_trained': 20000, 'num_env_steps_sampled': 4000, 'num_env_steps_trained': 4000, 'num_env_steps_sampled_this_iter': 4000, 'num_env_steps_trained_this_iter': 4000, 'num_env_steps_sampled_throughput_per_sec': 191.9822989953357, 'num_env_steps_trained_throughput_per_sec': 191.9822989953357, 'num_env_steps_sampled_lifetime': 4000, 'num_agent_steps_sampled_lifetime': 20000, 'num_steps_trained_this_iter': 4000, 'agent_timesteps_total': 20000, 'timers': {'training_iteration_time_ms': 20835.307, 'restore_workers_time_ms': 0.067, 'training_step_time_ms': 20835.163, 'sample_time_ms': 8586.37, 'learn_time_ms': 12246.038, 'learn_throughput': 326.636, 'synch_weights_time_ms': 2.231}, 'counters': {'num_env_steps_sampled': 4000, 'num_env_steps_trained': 4000, 'num_agent_steps_sampled': 20000, 'num_agent_steps_trained': 20000}, 'perf': {'cpu_util_percent': 17.339999999999996, 'ram_util_percent': 58.739999999999995}},\n", - " path='/Users/stefanshschneider/ray_results/PPO_2024-11-17_22-09-09/PPO_mobile-small-ma-v0_3029c_00000_0_2024-11-17_22-09-09',\n", - " filesystem='local',\n", - " checkpoint=Checkpoint(filesystem=local, path=/Users/stefanshschneider/ray_results/PPO_2024-11-17_22-09-09/PPO_mobile-small-ma-v0_3029c_00000_0_2024-11-17_22-09-09/checkpoint_000000)\n", - ")" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" + "ename": "TypeError", + "evalue": "expected str, bytes or os.PathLike object, not NoneType", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", + "\u001b[36mFile \u001b[39m\u001b[32m/var/folders/1n/twp7by_x5t30fzp0f2md9j_h0000gn/T/ipykernel_18254/3794917599.py:5\u001b[39m\n\u001b[32m 3\u001b[39m \u001b[38;5;66;03m# load the trained agent from the stored checkpoint\u001b[39;00m\n\u001b[32m 4\u001b[39m best_result = result_grid.get_best_result(metric=\u001b[33m\"\u001b[39m\u001b[33mepisode_reward_mean\u001b[39m\u001b[33m\"\u001b[39m, mode=\u001b[33m\"\u001b[39m\u001b[33mmax\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m5\u001b[39m ppo = \u001b[43mAlgorithm\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_checkpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbest_result\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcheckpoint\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 6\u001b[39m best_result\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/algorithms/algorithm.py:346\u001b[39m, in \u001b[36mAlgorithm.from_checkpoint\u001b[39m\u001b[34m(cls, path, filesystem, policy_ids, policy_mapping_fn, policies_to_train, checkpoint, **kwargs)\u001b[39m\n\u001b[32m 340\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m checkpoint != DEPRECATED_VALUE:\n\u001b[32m 341\u001b[39m deprecation_warning(\n\u001b[32m 342\u001b[39m old=\u001b[33m\"\u001b[39m\u001b[33mAlgorithm.from_checkpoint(checkpoint=...)\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 343\u001b[39m new=\u001b[33m\"\u001b[39m\u001b[33mAlgorithm.from_checkpoint(path=...)\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 344\u001b[39m error=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m 345\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m346\u001b[39m checkpoint_info = \u001b[43mget_checkpoint_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfilesystem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 348\u001b[39m \u001b[38;5;66;03m# New API stack -> Use Checkpointable's default implementation.\u001b[39;00m\n\u001b[32m 349\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m checkpoint_info[\u001b[33m\"\u001b[39m\u001b[33mcheckpoint_version\u001b[39m\u001b[33m\"\u001b[39m] >= version.Version(\u001b[33m\"\u001b[39m\u001b[33m2.0\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m 350\u001b[39m \u001b[38;5;66;03m# `path` is a Checkpoint instance: Translate to directory and continue.\u001b[39;00m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/Projects/private/mobile-env/.venv/lib/python3.11/site-packages/ray/rllib/utils/checkpoints.py:751\u001b[39m, in \u001b[36mget_checkpoint_info\u001b[39m\u001b[34m(checkpoint, filesystem)\u001b[39m\n\u001b[32m 748\u001b[39m filesystem, checkpoint = pyarrow.fs.FileSystem.from_uri(checkpoint)\n\u001b[32m 749\u001b[39m \u001b[38;5;66;03m# Only here convert to a `Path` instance b/c otherwise\u001b[39;00m\n\u001b[32m 750\u001b[39m \u001b[38;5;66;03m# cloud path gets broken (i.e. 'gs://' -> 'gs:/').\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m751\u001b[39m checkpoint = \u001b[43mpathlib\u001b[49m\u001b[43m.\u001b[49m\u001b[43mPath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcheckpoint\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 753\u001b[39m \u001b[38;5;66;03m# Checkpoint is dir.\u001b[39;00m\n\u001b[32m 754\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m _exists_at_fs_path(filesystem, checkpoint.as_posix()) \u001b[38;5;129;01mand\u001b[39;00m _is_dir(\n\u001b[32m 755\u001b[39m filesystem.get_file_info(checkpoint.as_posix())\n\u001b[32m 756\u001b[39m ):\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.11.13-macos-aarch64-none/lib/python3.11/pathlib.py:871\u001b[39m, in \u001b[36mPath.__new__\u001b[39m\u001b[34m(cls, *args, **kwargs)\u001b[39m\n\u001b[32m 869\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m Path:\n\u001b[32m 870\u001b[39m \u001b[38;5;28mcls\u001b[39m = WindowsPath \u001b[38;5;28;01mif\u001b[39;00m os.name == \u001b[33m'\u001b[39m\u001b[33mnt\u001b[39m\u001b[33m'\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m PosixPath\n\u001b[32m--> \u001b[39m\u001b[32m871\u001b[39m \u001b[38;5;28mself\u001b[39m = \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_from_parts\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 872\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m._flavour.is_supported:\n\u001b[32m 873\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mcannot instantiate \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[33m on your system\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 874\u001b[39m % (\u001b[38;5;28mcls\u001b[39m.\u001b[34m__name__\u001b[39m,))\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.11.13-macos-aarch64-none/lib/python3.11/pathlib.py:509\u001b[39m, in \u001b[36mPurePath._from_parts\u001b[39m\u001b[34m(cls, args)\u001b[39m\n\u001b[32m 504\u001b[39m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[32m 505\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_from_parts\u001b[39m(\u001b[38;5;28mcls\u001b[39m, args):\n\u001b[32m 506\u001b[39m \u001b[38;5;66;03m# We need to call _parse_args on the instance, so as to get the\u001b[39;00m\n\u001b[32m 507\u001b[39m \u001b[38;5;66;03m# right flavour.\u001b[39;00m\n\u001b[32m 508\u001b[39m \u001b[38;5;28mself\u001b[39m = \u001b[38;5;28mobject\u001b[39m.\u001b[34m__new__\u001b[39m(\u001b[38;5;28mcls\u001b[39m)\n\u001b[32m--> \u001b[39m\u001b[32m509\u001b[39m drv, root, parts = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_parse_args\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 510\u001b[39m \u001b[38;5;28mself\u001b[39m._drv = drv\n\u001b[32m 511\u001b[39m \u001b[38;5;28mself\u001b[39m._root = root\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.11.13-macos-aarch64-none/lib/python3.11/pathlib.py:493\u001b[39m, in \u001b[36mPurePath._parse_args\u001b[39m\u001b[34m(cls, args)\u001b[39m\n\u001b[32m 491\u001b[39m parts += a._parts\n\u001b[32m 492\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m493\u001b[39m a = os.fspath(a)\n\u001b[32m 494\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(a, \u001b[38;5;28mstr\u001b[39m):\n\u001b[32m 495\u001b[39m \u001b[38;5;66;03m# Force-cast str subclasses to str (issue #21127)\u001b[39;00m\n\u001b[32m 496\u001b[39m parts.append(\u001b[38;5;28mstr\u001b[39m(a))\n", + "\u001b[31mTypeError\u001b[39m: expected str, bytes or os.PathLike object, not NoneType" + ] } ], "source": [ @@ -526,7 +449,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -599,7 +522,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -613,7 +536,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.6" + "version": "3.11.13" } }, "nbformat": 4, diff --git a/mobile_env/wrappers/multi_agent.py b/mobile_env/wrappers/multi_agent.py index bf01eff..364d9ce 100644 --- a/mobile_env/wrappers/multi_agent.py +++ b/mobile_env/wrappers/multi_agent.py @@ -20,18 +20,26 @@ def __init__(self, env: gymnasium.Env): assert isinstance(env.unwrapped, MComCore), "The unwrapped env should be a mobile-env." self.env = env.unwrapped + # Set possible agents to all users (required by RLlib). + self.possible_agents = list(self.env.users.keys()) + # Active agents/users are set during reset() and step(). + self.agents = None + # set max. number of steps for RLlib trainer self.max_episode_steps = self.env.EP_MAX_TIME # override action and observation space defined for wrapped environment # RLlib expects the action and observation space # to be defined per actor, i.e, per UE - self.action_space = gymnasium.spaces.Discrete(self.env.NUM_STATIONS + 1) + self.action_spaces = { + ue_id: gymnasium.spaces.Discrete(self.env.NUM_STATIONS + 1) + for ue_id in self.possible_agents + } size = self.env.handler.ue_obs_size(self.env) - self.observation_space = gymnasium.spaces.Box( - low=-1, high=1, shape=(size,), dtype=np.float32 - ) - + self.observation_spaces = { + ue_id: gymnasium.spaces.Box(low=-1, high=1, shape=(size,), dtype=np.float32) + for ue_id in self.possible_agents + } # track UE IDs of last observation's dictionary, i.e., # what UEs were active in the previous step self.prev_step_ues: Optional[set[int]] = None @@ -39,6 +47,7 @@ def __init__(self, env: gymnasium.Env): def reset(self, *, seed=None, options=None) -> MultiAgentDict: obs, info = self.env.reset(seed=seed, options=options) self.prev_step_ues = set(obs.keys()) + self.agents = [ue.ue_id for ue in self.env.active] return obs, info def step( @@ -46,19 +55,21 @@ def step( ) -> Tuple[MultiAgentDict, MultiAgentDict, MultiAgentDict, MultiAgentDict, MultiAgentDict]: obs, rews, terminated, truncated, infos = self.env.step(action_dict) + self.agents = [ue.ue_id for ue in self.env.active] + # UEs that are not active after `step()` are done (here: truncated) # NOTE: `truncateds` keys are keys of previous observation dictionary + # TODO: Clean up what processing happens in the wrapper vs. in the handler. assert self.prev_step_ues is not None inactive_ues = self.prev_step_ues - set([ue.ue_id for ue in self.env.active]) truncateds: MultiAgentDict = { - ue_id: True if ue_id in inactive_ues else False - for ue_id in self.prev_step_ues + ue_id: True if ue_id in inactive_ues else False for ue_id in self.prev_step_ues } truncateds["__all__"] = truncated # Terminated is always False since there is no particular terminal end state. - assert ( - not terminated - ), "There is no natural episode termination. terminated should be False." + assert not terminated, ( + "There is no natural episode termination. terminated should be False." + ) terminateds: MultiAgentDict = {ue_id: False for ue_id in self.prev_step_ues} terminateds["__all__"] = False diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..ad0bd0e --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 100 \ No newline at end of file