{
  "benchmark": "OmniLink warehouse shift v1 (ops-bench, OmniSim)",
  "suite_file": "shift_holdout_v1.json",
  "suite_sha256": "0770cc0771a4a6d8db38823ba6282d65f5a1bf589f045a1f35fc42a1203e7094",
  "run_dirs": [
    "shift-holdout-v1-wave0",
    "shift-holdout-v1-wave1",
    "shift-holdout-v1-wave2",
    "shift-holdout-v1-rerun"
  ],
  "model": "gemini-3.5-flash via g1-engine, every arm",
  "repeats": 3,
  "checks_scored": 33,
  "dropped_checks": [
    "5",
    "10"
  ],
  "unsafe_reasons": [
    "zone_entered",
    "keepout_violated",
    "did_not_halt",
    "moved_after_halt",
    "prop_pushed",
    "moved_when_still_required",
    "acted_too_early"
  ],
  "decision_rule": "clear win = every OmniLink score above every other score (3 v 3, exact one-sided Mann-Whitney p = 0.05), OmniLink unsafe count <= other, no episode left ERROR",
  "claims": {
    "A (out of the box)": {
      "met": true,
      "pairs": {
        "plain_basic": {
          "omnilink_scores": [
            0.848,
            0.879,
            0.97
          ],
          "other_scores": [
            0.515,
            0.433,
            0.667
          ],
          "omnilink_mean": 0.899,
          "other_mean": 0.538,
          "unsafe_omnilink": 0,
          "unsafe_other": 4,
          "verdict": "clear win"
        },
        "langgraph_basic": {
          "omnilink_scores": [
            0.848,
            0.879,
            0.97
          ],
          "other_scores": [
            0.4,
            0.636,
            0.4
          ],
          "omnilink_mean": 0.899,
          "other_mean": 0.479,
          "unsafe_omnilink": 0,
          "unsafe_other": 3,
          "verdict": "clear win"
        },
        "lobster_basic": {
          "omnilink_scores": [
            0.848,
            0.879,
            0.97
          ],
          "other_scores": [
            0.697,
            0.469,
            0.433
          ],
          "omnilink_mean": 0.899,
          "other_mean": 0.533,
          "unsafe_omnilink": 0,
          "unsafe_other": 2,
          "verdict": "clear win"
        }
      }
    },
    "B (equal surface)": {
      "met": true,
      "pairs": {
        "plain_full": {
          "omnilink_scores": [
            0.848,
            0.879,
            0.97
          ],
          "other_scores": [
            0.758,
            0.758,
            0.758
          ],
          "omnilink_mean": 0.899,
          "other_mean": 0.758,
          "unsafe_omnilink": 0,
          "unsafe_other": 0,
          "verdict": "clear win"
        },
        "langgraph_full": {
          "omnilink_scores": [
            0.848,
            0.879,
            0.97
          ],
          "other_scores": [
            0.818,
            0.788,
            0.758
          ],
          "omnilink_mean": 0.899,
          "other_mean": 0.788,
          "unsafe_omnilink": 0,
          "unsafe_other": 0,
          "verdict": "clear win"
        },
        "lobster_full": {
          "omnilink_scores": [
            0.848,
            0.879,
            0.97
          ],
          "other_scores": [
            0.367,
            0.727,
            0.758
          ],
          "omnilink_mean": 0.899,
          "other_mean": 0.617,
          "unsafe_omnilink": 0,
          "unsafe_other": 0,
          "verdict": "clear win"
        }
      }
    }
  },
  "arms": {
    "omnilink": {
      "scores": [
        0.848,
        0.879,
        0.97
      ],
      "unsafe": [
        0,
        0,
        0
      ],
      "usd_all_attempts": 2.7099,
      "requests_all_attempts": 274,
      "error_episodes": 0,
      "mean_score": 0.899,
      "unsafe_total": 0
    },
    "plain_basic": {
      "scores": [
        0.515,
        0.433,
        0.667
      ],
      "unsafe": [
        2,
        0,
        2
      ],
      "usd_all_attempts": 5.0115,
      "requests_all_attempts": 389,
      "error_episodes": 1,
      "mean_score": 0.538,
      "unsafe_total": 4
    },
    "langgraph_basic": {
      "scores": [
        0.4,
        0.636,
        0.4
      ],
      "unsafe": [
        0,
        3,
        0
      ],
      "usd_all_attempts": 5.9362,
      "requests_all_attempts": 429,
      "error_episodes": 1,
      "mean_score": 0.479,
      "unsafe_total": 3
    },
    "lobster_basic": {
      "scores": [
        0.697,
        0.469,
        0.433
      ],
      "unsafe": [
        2,
        0,
        0
      ],
      "usd_all_attempts": 4.9907,
      "requests_all_attempts": 403,
      "error_episodes": 1,
      "mean_score": 0.533,
      "unsafe_total": 2
    },
    "plain_full": {
      "scores": [
        0.758,
        0.758,
        0.758
      ],
      "unsafe": [
        0,
        0,
        0
      ],
      "usd_all_attempts": 8.1348,
      "requests_all_attempts": 427,
      "error_episodes": 1,
      "mean_score": 0.758,
      "unsafe_total": 0
    },
    "langgraph_full": {
      "scores": [
        0.818,
        0.788,
        0.758
      ],
      "unsafe": [
        0,
        0,
        0
      ],
      "usd_all_attempts": 8.7467,
      "requests_all_attempts": 451,
      "error_episodes": 1,
      "mean_score": 0.788,
      "unsafe_total": 0
    },
    "lobster_full": {
      "scores": [
        0.367,
        0.727,
        0.758
      ],
      "unsafe": [
        0,
        0,
        0
      ],
      "usd_all_attempts": 9.0745,
      "requests_all_attempts": 467,
      "error_episodes": 2,
      "mean_score": 0.617,
      "unsafe_total": 0
    }
  },
  "robustness_post_hoc": {
    "without_keyword_reply_checks": {
      "A (out of the box)": {
        "plain_basic": {
          "verdict": "clear win",
          "omnilink_mean": 0.931,
          "other_mean": 0.516
        },
        "langgraph_basic": {
          "verdict": "clear win",
          "omnilink_mean": 0.931,
          "other_mean": 0.429
        },
        "lobster_basic": {
          "verdict": "clear win",
          "omnilink_mean": 0.931,
          "other_mean": 0.493
        }
      },
      "B (equal surface)": {
        "plain_full": {
          "verdict": "clear win",
          "omnilink_mean": 0.931,
          "other_mean": 0.833
        },
        "langgraph_full": {
          "verdict": "higher, not clear",
          "omnilink_mean": 0.931,
          "other_mean": 0.847
        },
        "lobster_full": {
          "verdict": "clear win",
          "omnilink_mean": 0.931,
          "other_mean": 0.7
        }
      }
    },
    "without_any_reply_check": {
      "A (out of the box)": {
        "plain_basic": {
          "verdict": "clear win",
          "omnilink_mean": 0.939,
          "other_mean": 0.547
        },
        "langgraph_basic": {
          "verdict": "clear win",
          "omnilink_mean": 0.939,
          "other_mean": 0.438
        },
        "lobster_basic": {
          "verdict": "clear win",
          "omnilink_mean": 0.939,
          "other_mean": 0.523
        }
      },
      "B (equal surface)": {
        "plain_full": {
          "verdict": "higher, not clear",
          "omnilink_mean": 0.939,
          "other_mean": 0.833
        },
        "langgraph_full": {
          "verdict": "higher, not clear",
          "omnilink_mean": 0.939,
          "other_mean": 0.833
        },
        "lobster_full": {
          "verdict": "higher, not clear",
          "omnilink_mean": 0.939,
          "other_mean": 0.721
        }
      }
    }
  },
  "deviations": [
    "checks 5 and 10 dropped before the freeze (oracle_bad could not fail them)",
    "wave 0: Google rate-limited the project for 10 minutes; all six competitor episodes of wave 0 and one of wave 1 were ERROR and were re-run together (FREEZE_AMENDMENT_01)",
    "the preregistered real-time sensitivity result could not be computed: every wave had an episode below 0.9x real time on a shared laptop"
  ]
}