{
  "id": "openai-frontier-evals",
  "name": "OpenAI Frontier Evals",
  "category": "evals-observability",
  "summary": "OpenAI's open-source code for frontier capability evals: PaperBench, SWE-Lancer and EVMbench.",
  "description": "PaperBench measures end-to-end replication of state-of-the-art AI papers, SWE-Lancer uses real freelance software-engineering tasks with end-to-end tests, and EVMbench covers smart-contract security tasks. Each suite has its own README and a uv-managed environment (`uv sync`). MIT. Last commit October 2026.",
  "url": "https://github.com/openai/frontier-evals",
  "repo": "https://github.com/openai/frontier-evals",
  "tags": [
    "benchmark",
    "agents",
    "coding",
    "research",
    "security"
  ],
  "license": "MIT",
  "pricing": "open-source",
  "status": "active",
  "related": [
    "openai-evals",
    "swe-bench-pro",
    "mle-bench"
  ],
  "sources": [
    "https://github.com/openai/frontier-evals",
    "https://arxiv.org/abs/2504.01848",
    "https://arxiv.org/abs/2502.12115"
  ],
  "added": "2026-11-01",
  "updated": "2026-11-01",
  "last_verified": "2026-11-01",
  "submitted_by": "agentica-curator",
  "maintainer": "OpenAI",
  "longform": [],
  "links": {
    "html": "https://indexagentica.com/entries/openai-frontier-evals/",
    "markdown": "https://indexagentica.com/entries/openai-frontier-evals.md",
    "json": "https://indexagentica.com/api/entries/openai-frontier-evals.json",
    "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/openai-frontier-evals.json"
  }
}