{ "schema_version": "paper_public_manifest_v1", "paper_id": "smoothrl_online_reinforcement_learning_during_asynchronous_execution_2026_09", "slug": "smoothrl_online_reinforcement_learning_during_asynchronous_execution", "title": "SmoothRL: Only Account for Actions Actually Executed", "authors": [], "source": { "arxiv_id": "2608.29768", "pdf_url": "https://arxiv.org/pdf/2608.29768", "project_url": "", "github_url": "", "huggingface_url": "", "original_source": "https://arxiv.org/pdf/2608.29768" }, "site": { "post_url": "/posts/smoothrl_online_reinforcement_learning_during_asynchronous_execution", "canonical_url": "https://haiguangboy.com/posts/smoothrl_online_reinforcement_learning_during_asynchronous_execution", "cover_image": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/cover.webp" }, "taxonomy": { "domain": "embodied_ai", "track": "vla", "tasks": [ "embodied_ai", "vla", "action_generation", "robotics", "state_prediction", "Embodied Intelligence", "Reinforcement Learning" ], "related_topics": [ { "paper_id": "rl_100_performant_robotic_manipulation_with_real_world_reinforcement_learning_2026_08", "title": "RL Should Not Learn from Scratch, It Should Do Post-Training", "url": "https://haiguangboy.com/posts/rl-100", "relation": "contrast", "summary": "On three real-robot tasks, after 250 rounds of online reinforcement learning rollouts, all showed substantial improvement over the frozen base policy: dynamic throwing from 39% to 94%, pen capping from 8% to 83%, box opening from 30% to 90%; the base policy's failures are systematic fixed biases (not random errors), and online RL mainly corrects these systematic biases contradicts real-robot eight-task 1000/1000 trials 100% success; the offline RL stage contributed most of the improvement, while the online stage only cleaned up residual...", "strength": "strong" }, { "paper_id": "beyond_action_residuals_real_world_robot_policy_steering_via_bottleneck_latent_r_2026_08", "title": "At Which Layer Should RL Intervene", "url": "https://haiguangboy.com/posts/zprl", "relation": "contrast", "summary": "The smoothness of the pretrained policy comes from demonstration data; once the value objective starts pushing up Q-values on the execution segment, this smoothness guarantee no longer automatically holds—smoothness must be re-added to the optimization objective as an explicit constraint (boundary penalty terms on per-frame velocity/acceleration/jerk) contradicts mechanism explanation: the guidance is remapping state-action associations, not inventing new actions", "strength": "strong" }, { "paper_id": "n_0_vtla_scaling_vision_tactile_language_action_model_with_latent_tactile_tokens_2026_07", "title": "Tactile as Prediction Target Rather Than Observation Input", "url": "https://haiguangboy.com/posts/n_0_vtla_scaling_vision_tactile_language_action_model_with_latent_tactile_tokens", "relation": "contrast", "summary": "Tactile as Prediction Target Rather Than Observation Input", "strength": "strong" }, { "paper_id": "an_open_foundation_model_towards_2026_07", "title": "An_Open_Foundation_Model_Towards", "url": "https://haiguangboy.com/posts/an_open_foundation_model_towards", "relation": "same_track", "summary": "The asynchronous inference loop actually runs during the rollout process at training time, not only at deployment—this way the replay data records the actions the robot actually executed under timing and execution scheduling fully consistent with the objective function, ensuring the policy is never optimized under dynamics conditions it would not encounter at deployment validates RTC at training time: masking the first d action tokens so the model learns smooth continuation", "strength": "strong" }, { "paper_id": "mathcaln_0_foundation_towards_the_age_of_tactile_intelligence_2026_09", "title": "N0-Foundation: Opening a New Era of Touch", "url": "https://haiguangboy.com/posts/mathcaln_0_foundation_towards_the_age_of_tactile_intelligence", "relation": "same_track", "summary": "N0-Foundation: Opening a New Era of Touch", "strength": "strong" }, { "paper_id": "learning_native_continuation_for_action_chunking_flow_policies_2026_09", "title": "VLA No Longer Stutters When Switching Action Chunks", "url": "https://haiguangboy.com/posts/learning_native_continuation_for_action_chunking_flow_policies", "relation": "same_track", "summary": "VLA No Longer Stutters When Switching Action Chunks", "strength": "strong" } ] }, "analyst_take": { "type": "author_opinion", "text": "This paper and the already-interpreted RL-100 form a divergence worth noting: RL-100's own finding is that the offline RL stage contributed most of the improvement, while the online stage only cleaned up residual failures; this paper, however, treats online RL as the core means of correcting systematic biases (not residual noise). The two sides give different answers to \"how much work online RL should carry,\" which may depend on how close to success the pretraining stage itself brings the policy. Additionally, this paper's principle of \"running the asynchronous loop at training time too, so the optimization objective aligns with the real timing at deployment\" points in the same direction as the recently interpreted Legato—both papers emphasize that the training stage must never turn a blind eye to the timing structure encountered at deployment." }, "ruling": { "importance_score": 3.0, "one_sentence": "SmoothRL: Only Account for Actions Actually Executed" }, "asset_base_url": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution", "assets": [ { "type": "cover_image", "object_key": "papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/cover.webp", "url": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/cover.webp", "content_type": "image/webp", "upload_status": "pending", "role": "post_cover" }, { "type": "pdf_screenshot", "object_key": "papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/page_01.webp", "url": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/page_01.webp", "content_type": "image/webp", "upload_status": "pending", "role": "paper_first_page" }, { "type": "pdf_screenshot", "object_key": "papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/key_figure.webp", "url": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/key_figure.webp", "content_type": "image/webp", "upload_status": "pending", "role": "method_figure" }, { "type": "public_manifest", "object_key": "papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/public_manifest.json", "url": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/public_manifest.json", "content_type": "application/json; charset=utf-8", "upload_status": "pending", "role": "public_manifest" }, { "type": "public_brief", "object_key": "papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/public_brief.md", "url": "https://static.haiguangboy.com/papers/smoothrl_online_reinforcement_learning_during_asynchronous_execution/public_brief.md", "content_type": "text/markdown; charset=utf-8", "upload_status": "pending", "role": "public_brief" } ], "published_at": "2026-09-13T08:06:20+08:00", "created_at": "2026-09-13T08:06:20+08:00", "updated_at": "2026-09-13T08:06:20+08:00" }