{
  "schema_version": 2,
  "updated": "2026-10-30",
  "units": {
    "performance": "percent",
    "gain": "percentage points",
    "steps": "environment interactions, not generated tokens or compute"
  },
  "canonical": {
    "model": "Qwen2.5-7B-Instruct",
    "source": "paper/analysis/inference_trajectories/summary.csv",
    "setting": "PPO vs PPO + AttnSR; epoch 4; one rollout per prompt; actor/critic learning rate 5e-6; LoRA rank 64",
    "limitations": [
      "One training run per method; no seed-robustness claim",
      "ALFWorld score equals success; other tasks report official score and binary success separately",
      "Only ALFWorld success remains significant after the three-task Bonferroni correction",
      "Output characters and environment steps are descriptive interaction measures, not compute measurements"
    ],
    "benchmarks": [
      {
        "id": "alfworld",
        "label": "ALFWorld",
        "examples": 274,
        "ppo": {
          "score_percent": 91.24087591240875,
          "success_percent": 91.24087591240875,
          "mean_steps": 19.277372262773724,
          "mean_serialized_output_chars": 16913.68613138686,
          "display": {
            "score_percent": "91.24",
            "success_percent": "91.24",
            "mean_steps": "19.28"
          }
        },
        "attnsr": {
          "score_percent": 97.08029197080292,
          "success_percent": 97.08029197080292,
          "mean_steps": 15.507299270072993,
          "mean_serialized_output_chars": 13324.277372262773,
          "display": {
            "score_percent": "97.08",
            "success_percent": "97.08",
            "mean_steps": "15.51"
          }
        },
        "success_gain_pp": 5.839416058394164,
        "score_gain_pp": 5.839416058394164,
        "steps_change_percent": -19.55698599015525,
        "interpretation": "Clearer benefit: success increases; aggregate mean steps decrease. Shorter aggregate trajectories do not by themselves establish shorter successful solutions."
      },
      {
        "id": "scienceworld",
        "label": "ScienceWorld",
        "examples": 1819,
        "ppo": {
          "score_percent": 44.53490929081912,
          "success_percent": 28.587135788894997,
          "mean_steps": 30.678944474986256,
          "mean_serialized_output_chars": 45416.98460692688,
          "display": {
            "score_percent": "44.53",
            "success_percent": "28.59",
            "mean_steps": "30.68"
          }
        },
        "attnsr": {
          "score_percent": 46.34799340296866,
          "success_percent": 30.896096756459595,
          "mean_steps": 21.510720175920834,
          "mean_serialized_output_chars": 31989.848818031885,
          "display": {
            "score_percent": "46.35",
            "success_percent": "30.90",
            "mean_steps": "21.51"
          }
        },
        "success_gain_pp": 2.308960967564598,
        "score_gain_pp": 1.8130841121495394,
        "steps_change_percent": -29.884418958874658,
        "interpretation": "Smaller numerical gains in score and success; aggregate mean steps decrease. Success significance does not survive the three-task correction."
      },
      {
        "id": "webshop",
        "label": "WebShop",
        "examples": 500,
        "ppo": {
          "score_percent": 70.74166666666666,
          "success_percent": 40.2,
          "mean_steps": 5.934,
          "mean_serialized_output_chars": 8306.218,
          "display": {
            "score_percent": "70.74",
            "success_percent": "40.20",
            "mean_steps": "5.93"
          }
        },
        "attnsr": {
          "score_percent": 69.705,
          "success_percent": 41.0,
          "mean_steps": 8.026,
          "mean_serialized_output_chars": 11082.43,
          "display": {
            "score_percent": "69.71",
            "success_percent": "41.00",
            "mean_steps": "8.03"
          }
        },
        "success_gain_pp": 0.7999999999999972,
        "score_gain_pp": -1.0366666666666617,
        "steps_change_percent": 35.254465790360626,
        "interpretation": "Mixed outcome: success increases slightly, official score decreases, and mean interaction steps increase; no detectable paired benefit."
      }
    ]
  },
  "qwen25_3b_ppo": {
    "model": "Qwen2.5-3B-Instruct",
    "source": "experiments/agentic_rl/results/qwen25_3b_eval_20261022/qwen25_3b_evaluation.csv",
    "asset": "assets/3b-evaluation.svg",
    "mobile_asset": "assets/3b-evaluation-mobile.svg",
    "download": "data/3b-evaluation.csv",
    "setting": "ALFWorld full 274; four epochs; n=1; 1776 updates; actor/critic learning rate 5e-6; relative attention layers [10,13,16,18,21,23]",
    "limitations": [
      "One run per method",
      "Baseline/v2 source trees include critic/value-head compatibility differences; not a strictly isolated reward ablation"
    ],
    "epochs": [
      {
        "epoch": 1,
        "examples": 274,
        "ppo": 4.744525547445255,
        "attnsr": 22.26277372262774,
        "gain_pp": 17.518248175182485
      },
      {
        "epoch": 2,
        "examples": 274,
        "ppo": 33.21167883211679,
        "attnsr": 61.31386861313869,
        "gain_pp": 28.1021897810219
      },
      {
        "epoch": 3,
        "examples": 274,
        "ppo": 61.67883211678832,
        "attnsr": 72.26277372262774,
        "gain_pp": 10.58394160583942
      },
      {
        "epoch": 4,
        "examples": 274,
        "ppo": 79.56204379562044,
        "attnsr": 87.22627737226277,
        "gain_pp": 7.6642335766423315
      }
    ],
    "success_counts_epoch4": {
      "ppo": 218,
      "attnsr": 239
    }
  },
  "qwen25_3b_grpo": {
    "model": "Qwen2.5-3B-Instruct",
    "source": "experiments/agentic_rl/results/qwen25_3b_grpo_attnsr_full274_20261022_082532.csv",
    "setting": "ALFWorld full 274; one epoch; n=4; 444 updates; learning rate 5e-6; no critic; initial token-advantage reweighting with alpha=0.25",
    "limitations": [
      "One run per method",
      "Uses the initial token-advantage variant, not PPO step-end reward/GAE or the later RTG variant",
      "Positive confidence cancels in normalization; zero confidence disables redistribution",
      "High format-error rates limit interpretation"
    ],
    "examples": 274,
    "baseline": 12.408759124087592,
    "attnsr": 20.072992700729927,
    "gain_pp": 7.664233576642335,
    "success_counts": {
      "baseline": 34,
      "attnsr": 55
    },
    "format_error_percent": {
      "baseline": 36.86131386861314,
      "attnsr": 34.67153284671533
    }
  },
  "qwen3_4b_ppo": {
    "model": "Qwen3-4B-Instruct-2507",
    "source": "paper/tables/linear_control.tex",
    "setting": "Supplementary ALFWorld PPO / AttnSR comparison; four epochs; main PPO/shaping hyperparameters retained",
    "limitations": [
      "Single-run, descriptive cross-backbone comparison; no universal transfer claim"
    ],
    "baseline": 87.96,
    "attnsr": 91.24,
    "gain_pp": 3.28
  },
  "training_reward_figure": {
    "source": "experiments/agentic_rl/results/training_terminal_reward_curves.csv",
    "asset": "assets/7b-training.svg",
    "mobile_asset": "assets/7b-training-mobile.svg",
    "download": "data/7b-training.csv",
    "aggregation": "Recorded epoch means, connected without additional smoothing; no confidence interval",
    "setting": "Supplementary eight-epoch environment outcome reward traces; learning rate 1e-6; shaping reward excluded",
    "limitations": [
      "Different training setting from the canonical four-epoch held-out comparison",
      "Does not establish asymptotic advantage"
    ]
  },
  "alfworld_methods": {
    "model": "Qwen2.5-7B-Instruct",
    "examples": 274,
    "sources": [
      "paper/tables/method_comparison.tex",
      "experiments/agentic_rl/results/attnsr_results_inventory_20261029.md",
      "paper/analysis/additional_baselines/provenance.json",
      "experiments/agentic_rl/results/canonical_full_alfworld_highlr5e6.csv",
      "experiments/agentic_rl/results/canonical_full_alfworld_qwen25_7b_grpo1_full_gpuH100_20260908_restart_073644.csv"
    ],
    "setting": "Full ALFWorld held-out set; greedy evaluation; Raw before RL; historical GRPO at epoch 1 with n=4; other trained methods at epoch 4",
    "limitations": [
      "Single-run, descriptive comparison; optimization budgets differ",
      "The 97.08% vs 96.72% margin is one task; no paired superiority claim over TARL",
      "Canonical PPO is 250/274 (91.24%); the inventory's separate PPO 252/274 and its paired intervals are not reused",
      "Additional baseline counts are transcribed from the author-provided inventory; raw episodes were unavailable in this workspace"
    ],
    "methods": [
      {
        "method": "Raw",
        "success_percent": 26.642335766423358,
        "success_count": 73,
        "seen_percent": 33.57142857142857,
        "unseen_percent": 19.402985074626866,
        "training_budget": "Before RL"
      },
      {
        "method": "GRPO",
        "success_percent": 69.7080291970803,
        "success_count": 191,
        "seen_percent": 75.0,
        "unseen_percent": 64.17910447761194,
        "training_budget": "1 epoch, n=4"
      },
      {
        "method": "PPO",
        "success_percent": 91.24087591240875,
        "success_count": 250,
        "seen_percent": 92.85714285714286,
        "unseen_percent": 89.55223880597015,
        "training_budget": "4 epochs, n=1"
      },
      {
        "method": "TARL",
        "variant": "TARL-Turn",
        "success_percent": 96.71532846715328,
        "success_count": 265,
        "seen_percent": 96.42857142857143,
        "unseen_percent": 97.01492537313433,
        "training_budget": "4 epochs"
      },
      {
        "method": "AgentPRM",
        "variant": "AgentPRM-Progress",
        "success_percent": 91.97080291970804,
        "success_count": 252,
        "seen_percent": 92.85714285714286,
        "unseen_percent": 91.04477611940298,
        "training_budget": "4 epochs"
      },
      {
        "method": "FlowTracer",
        "variant": "FlowTracer-PPO",
        "success_percent": 95.62043795620438,
        "success_count": 262,
        "seen_percent": 96.42857142857143,
        "unseen_percent": 94.77611940298507,
        "training_budget": "4 epochs"
      },
      {
        "method": "PPO + AttnSR",
        "success_percent": 97.08029197080292,
        "success_count": 266,
        "seen_percent": 96.42857142857143,
        "unseen_percent": 97.76119402985076,
        "training_budget": "4 epochs, n=1"
      }
    ]
  },
  "grpo_reference": {
    "model": "Qwen2.5-7B-Instruct",
    "sources": [
      "paper/analysis/grpo_inference_trajectories/summary.csv",
      "experiments/agentic_rl/results/canonical_full_alfworld_qwen25_7b_grpo1_full_gpuH100_20260908_restart_073644.csv"
    ],
    "setting": "Historical September GRPO reference; one epoch, n=4; learning rate 5e-6; LoRA rank 64; greedy full-set evaluation",
    "limitations": [
      "A reference baseline with a different optimization budget from four-epoch PPO"
    ],
    "benchmarks": [
      {
        "id": "alfworld",
        "label": "ALFWorld",
        "examples": 274,
        "score_percent": 69.7080291970803,
        "success_percent": 69.7080291970803,
        "seen_percent": 75.0,
        "unseen_percent": 64.17910447761194
      },
      {
        "id": "scienceworld",
        "label": "ScienceWorld",
        "examples": 1819,
        "score_percent": 30.191313908741062,
        "success_percent": 16.657504123144584
      },
      {
        "id": "webshop",
        "label": "WebShop",
        "examples": 500,
        "score_percent": 67.21366666666667,
        "success_percent": 37.8
      }
    ]
  },
  "ablations": {
    "model": "Qwen2.5-7B-Instruct",
    "sources": [
      "experiments/agentic_rl/results/canonical_full_alfworld_linear_ablation.csv",
      "experiments/agentic_rl/results/alfworld_async_method_comparison.csv",
      "paper/sections/12_appendix.tex"
    ],
    "setting": "ALFWorld; four epochs; learning rate 5e-6; LoRA rank 64",
    "limitations": [
      "Linear control jointly changes allocation, confidence gating, and realized budgets; it does not isolate attention alone",
      "Single-run comparisons; no paired significance test for the linear control"
    ],
    "linear": [
      {
        "method": "PPO",
        "success_percent": 91.24087591240875,
        "gain_pp": 0.0
      },
      {
        "method": "PPO + Linear Step Reward",
        "success_percent": 92.33576642335767,
        "gain_pp": 1.0948905109489147
      },
      {
        "method": "PPO + AttnSR",
        "success_percent": 97.08029197080292,
        "gain_pp": 5.839416058394164
      }
    ],
    "asynchronous": {
      "baseline": 59.31999999999999,
      "attnsr": 63.99,
      "gain_pp": 4.670000000000009,
      "setting": "Separate single-trajectory asynchronous configuration; four epochs",
      "limitations": [
        "Collection, importance weighting, and critic updates differ from canonical PPO; not a collection-timing-only ablation"
      ]
    }
  },
  "trajectories": {
    "sources": [
      "paper/analysis/inference_trajectories/summary.csv",
      "paper/analysis/grpo_inference_trajectories/summary.csv"
    ],
    "setting": "Full evaluation sets; PPO/AttnSR epoch 4, n=1; historical GRPO epoch 1, n=4",
    "limitations": [
      "Environment steps and serialized output characters include tool interaction; neither measures generated-token compute",
      "Aggregate means depend on the success/failure mix and do not establish shorter successful solutions",
      "GRPO has a different training budget"
    ],
    "benchmarks": [
      {
        "id": "alfworld",
        "label": "ALFWorld",
        "examples": 274,
        "methods": [
          {
            "method": "GRPO",
            "mean_steps": 36.934306569343065,
            "median_steps": 11.0,
            "p90_steps": 105.0,
            "output_kchars": 33.44337591240876
          },
          {
            "method": "PPO",
            "mean_steps": 19.277372262773724,
            "median_steps": 9.0,
            "p90_steps": 38.70000000000002,
            "output_kchars": 16.91368613138686
          },
          {
            "method": "PPO + AttnSR",
            "mean_steps": 15.507299270072993,
            "median_steps": 8.0,
            "p90_steps": 31.0,
            "output_kchars": 13.324277372262774
          }
        ]
      },
      {
        "id": "scienceworld",
        "label": "ScienceWorld",
        "examples": 1819,
        "methods": [
          {
            "method": "GRPO",
            "mean_steps": 27.185816382627817,
            "median_steps": 13.0,
            "p90_steps": 74.0,
            "output_kchars": 43.92627047828477
          },
          {
            "method": "PPO",
            "mean_steps": 30.678944474986256,
            "median_steps": 15.0,
            "p90_steps": 80.0,
            "output_kchars": 45.41698460692688
          },
          {
            "method": "PPO + AttnSR",
            "mean_steps": 21.510720175920834,
            "median_steps": 10.0,
            "p90_steps": 74.0,
            "output_kchars": 31.989848818031884
          }
        ]
      },
      {
        "id": "webshop",
        "label": "WebShop",
        "examples": 500,
        "methods": [
          {
            "method": "GRPO",
            "mean_steps": 8.462,
            "median_steps": 6.0,
            "p90_steps": 12.0,
            "output_kchars": 11.64533
          },
          {
            "method": "PPO",
            "mean_steps": 5.934,
            "median_steps": 5.0,
            "p90_steps": 9.0,
            "output_kchars": 8.306218000000001
          },
          {
            "method": "PPO + AttnSR",
            "mean_steps": 8.026,
            "median_steps": 7.0,
            "p90_steps": 12.0,
            "output_kchars": 11.08243
          }
        ]
      }
    ]
  },
  "alfworld_categories": {
    "source": "paper/tables/alfworld_category.tex",
    "model": "Qwen2.5-7B-Instruct",
    "examples": 274,
    "setting": "Canonical PPO vs PPO + AttnSR; final epoch 4",
    "limitations": [
      "Small task-family samples; descriptive comparisons",
      "PPO is already at 100% on Pick and 96.77% on Look; headroom differs by family"
    ],
    "categories": [
      {
        "label": "Pick",
        "examples": 59,
        "ppo_percent": 100.0,
        "attnsr_percent": 100.0,
        "gain_pp": 0.0,
        "ppo_success_count": 59,
        "attnsr_success_count": 59
      },
      {
        "label": "Look",
        "examples": 31,
        "ppo_percent": 96.7741935483871,
        "attnsr_percent": 96.7741935483871,
        "gain_pp": 0.0,
        "ppo_success_count": 30,
        "attnsr_success_count": 30
      },
      {
        "label": "Clean",
        "examples": 58,
        "ppo_percent": 86.20689655172413,
        "attnsr_percent": 93.10344827586206,
        "gain_pp": 6.896551724137931,
        "ppo_success_count": 50,
        "attnsr_success_count": 54
      },
      {
        "label": "Heat",
        "examples": 39,
        "ppo_percent": 89.74358974358975,
        "attnsr_percent": 100.0,
        "gain_pp": 10.256410256410255,
        "ppo_success_count": 35,
        "attnsr_success_count": 39
      },
      {
        "label": "Cool",
        "examples": 46,
        "ppo_percent": 86.95652173913044,
        "attnsr_percent": 97.82608695652173,
        "gain_pp": 10.869565217391305,
        "ppo_success_count": 40,
        "attnsr_success_count": 45
      },
      {
        "label": "Pick2",
        "examples": 41,
        "ppo_percent": 87.8048780487805,
        "attnsr_percent": 95.1219512195122,
        "gain_pp": 7.317073170731707,
        "ppo_success_count": 36,
        "attnsr_success_count": 39
      }
    ]
  },
  "source_hashes": [
    {
      "path": "experiments/agentic_rl/results/alfworld_async_method_comparison.csv",
      "sha256": "b1bb1875660a3c19dd0a4e7934cd5a119231ae2173b75c66e74e78c286848941"
    },
    {
      "path": "experiments/agentic_rl/results/attnsr_results_inventory_20261029.md",
      "sha256": "29a7e47b15af7a0d36f7845b27e4da65c655f9403a9511c580915c6e6d49c9f0"
    },
    {
      "path": "experiments/agentic_rl/results/canonical_full_alfworld_highlr5e6.csv",
      "sha256": "157b19580574c32b0eb4145476fa6c3c8f4751dd46dacc86ba88b2bd3f5c2d62"
    },
    {
      "path": "experiments/agentic_rl/results/canonical_full_alfworld_linear_ablation.csv",
      "sha256": "24a2a479d9c6dbb958e6b43e752e109719d5f3f314b4947969949323a1a743e8"
    },
    {
      "path": "experiments/agentic_rl/results/canonical_full_alfworld_qwen25_7b_grpo1_full_gpuH100_20260908_restart_073644.csv",
      "sha256": "f4b628b2c9b7721b19ff42b58f9021383a67b4aef6e97398c60caf18739d2061"
    },
    {
      "path": "experiments/agentic_rl/results/qwen25_3b_eval_20261022/qwen25_3b_evaluation.csv",
      "sha256": "da9db36685067da3534e9cde5a116a47b810cf5a3ccfbf87175a48ff95e67076"
    },
    {
      "path": "experiments/agentic_rl/results/qwen25_3b_grpo_attnsr_full274_20261022_082532.csv",
      "sha256": "fe9fecbc3eea1ac3b2187e3c6b458e57837f0175b31a47fd3426ab7d963de9e9"
    },
    {
      "path": "experiments/agentic_rl/results/training_terminal_reward_curves.csv",
      "sha256": "454a159693349ff9dff6c477803c85ccd5c23f94651dd6fb1c24f27ba3ac5b6c"
    },
    {
      "path": "paper/analysis/additional_baselines/provenance.json",
      "sha256": "f2fef886c4cbe76959b6ef4e2ebf8ca21ad91bfe7c68f2cd4aed9e094dbc6a4c"
    },
    {
      "path": "paper/analysis/grpo_inference_trajectories/summary.csv",
      "sha256": "ffec8dfce617ffa971c3e4b1517c1a0773275b6c4665df6aa99ea9e608baedd4"
    },
    {
      "path": "paper/analysis/inference_trajectories/summary.csv",
      "sha256": "db11dcbec5684d7c4d1eff1a91aabc2140f0b4d4d8f4fa2f87f281df75c2dec8"
    },
    {
      "path": "paper/sections/12_appendix.tex",
      "sha256": "4932a9720d26e3e8ae4f220c74085b07a4aa9610192e2ed97a520defef924f47"
    },
    {
      "path": "paper/tables/alfworld_category.tex",
      "sha256": "27deb2e7da26f3a03d3c3f41a23999bfc81ffce4c63f74bd8590583cc2e1d9be"
    },
    {
      "path": "paper/tables/linear_control.tex",
      "sha256": "42a78f8efa59228301e0134b682786e152403999f6e8f6cff58953d4e4e70cc1"
    },
    {
      "path": "paper/tables/method_comparison.tex",
      "sha256": "d4640ac4a7e95e5aa77d36d1e4b8289198c77611eda350e2792254b7ec508a13"
    }
  ]
}
