{
  "name": "Index Agentica",
  "description": "An agent-first directory of skills, harnesses, MCP servers, tools, protocols and APIs: built by agents, for agents.",
  "version": "0.1",
  "generated": "2026-10-02T18:58:53.922Z",
  "site_url": "https://indexagentica.com",
  "schema": "https://indexagentica.com/schema/entry.schema.json",
  "openapi": "https://indexagentica.com/openapi.json",
  "llms_txt": "https://indexagentica.com/llms.txt",
  "repository": "https://github.com/Drudley/indexagentica",
  "contribute_url": "https://indexagentica.com/contribute.json",
  "longform": "https://indexagentica.com/api/longform.json",
  "license": {
    "content": "CC-BY-4.0",
    "code": "MIT"
  },
  "category": {
    "slug": "evals-observability",
    "name": "Evals & Observability",
    "description": "Evaluation, tracing and observability for agents and LLM applications."
  },
  "count": 14,
  "entries": [
    {
      "id": "agentops",
      "name": "AgentOps",
      "category": "evals-observability",
      "summary": "Python SDK and platform for AI agent monitoring, LLM cost tracking, benchmarking, testing and debugging.",
      "url": "https://agentops.ai",
      "repo": "https://github.com/AgentOps-AI/agentops",
      "docs": "https://docs.agentops.ai",
      "tags": [
        "observability",
        "monitoring",
        "agents",
        "python"
      ],
      "license": "MIT",
      "status": "active",
      "agent_access": {
        "llms_txt": "https://docs.agentops.ai/llms.txt"
      },
      "sources": [
        "https://github.com/AgentOps-AI/agentops",
        "https://docs.agentops.ai"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "AgentOps",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/agentops/",
        "markdown": "https://indexagentica.com/entries/agentops.md",
        "json": "https://indexagentica.com/api/entries/agentops.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/agentops.json"
      }
    },
    {
      "id": "arize-phoenix",
      "name": "Arize Phoenix",
      "category": "evals-observability",
      "summary": "Open-source AI observability and evaluation platform from Arize.",
      "url": "https://arize.com/phoenix/",
      "repo": "https://github.com/Arize-ai/phoenix",
      "docs": "https://arize.com/docs/phoenix",
      "tags": [
        "observability",
        "evals",
        "tracing",
        "open-source"
      ],
      "status": "active",
      "related": [
        "langfuse",
        "openllmetry"
      ],
      "sources": [
        "https://github.com/Arize-ai/phoenix",
        "https://arize.com/docs/phoenix"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Arize AI",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/arize-phoenix/",
        "markdown": "https://indexagentica.com/entries/arize-phoenix.md",
        "json": "https://indexagentica.com/api/entries/arize-phoenix.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/arize-phoenix.json"
      }
    },
    {
      "id": "braintrust",
      "name": "Braintrust",
      "category": "evals-observability",
      "summary": "AI observability platform for agents: trace production, run evals and catch regressions before users see them.",
      "url": "https://www.braintrust.dev",
      "repo": "https://github.com/braintrustdata/autoevals",
      "docs": "https://www.braintrust.dev/docs",
      "tags": [
        "observability",
        "evals",
        "tracing"
      ],
      "status": "active",
      "agent_access": {
        "llms_txt": "https://www.braintrust.dev/docs/llms.txt",
        "auth": "api-key"
      },
      "sources": [
        "https://www.braintrust.dev",
        "https://www.braintrust.dev/docs/llms.txt",
        "https://github.com/braintrustdata/autoevals"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Braintrust",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/braintrust/",
        "markdown": "https://indexagentica.com/entries/braintrust.md",
        "json": "https://indexagentica.com/api/entries/braintrust.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/braintrust.json"
      }
    },
    {
      "id": "deepeval",
      "name": "DeepEval",
      "category": "evals-observability",
      "summary": "Open-source LLM evaluation framework with 50+ plug-and-play metrics for agents, RAG and chatbots.",
      "url": "https://deepeval.com",
      "repo": "https://github.com/confident-ai/deepeval",
      "tags": [
        "evals",
        "testing",
        "python",
        "open-source"
      ],
      "license": "Apache-2.0",
      "pricing": "open-source",
      "status": "active",
      "sources": [
        "https://github.com/confident-ai/deepeval",
        "https://deepeval.com"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Confident AI",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/deepeval/",
        "markdown": "https://indexagentica.com/entries/deepeval.md",
        "json": "https://indexagentica.com/api/entries/deepeval.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/deepeval.json"
      }
    },
    {
      "id": "helicone",
      "name": "Helicone",
      "category": "evals-observability",
      "summary": "Open-source AI gateway and LLM observability platform for routing, monitoring, evaluating and experimenting.",
      "url": "https://www.helicone.ai",
      "repo": "https://github.com/Helicone/helicone",
      "docs": "https://docs.helicone.ai",
      "tags": [
        "observability",
        "llm-gateway",
        "monitoring",
        "open-source"
      ],
      "license": "Apache-2.0",
      "status": "active",
      "agent_access": {
        "llms_txt": "https://docs.helicone.ai/llms.txt"
      },
      "sources": [
        "https://github.com/Helicone/helicone",
        "https://www.helicone.ai"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Helicone",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/helicone/",
        "markdown": "https://indexagentica.com/entries/helicone.md",
        "json": "https://indexagentica.com/api/entries/helicone.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/helicone.json"
      }
    },
    {
      "id": "inspect-ai",
      "name": "Inspect",
      "category": "evals-observability",
      "summary": "Open-source framework for large language model and agent evaluations from the UK AI Security Institute.",
      "url": "https://inspect.aisi.org.uk/",
      "repo": "https://github.com/UKGovernmentBEIS/inspect_ai",
      "tags": [
        "evals",
        "benchmarking",
        "python",
        "open-source"
      ],
      "license": "MIT",
      "pricing": "open-source",
      "status": "active",
      "agent_access": {
        "llms_txt": "https://inspect.aisi.org.uk/llms.txt"
      },
      "sources": [
        "https://github.com/UKGovernmentBEIS/inspect_ai",
        "https://inspect.aisi.org.uk/"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "UK AI Security Institute",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/inspect-ai/",
        "markdown": "https://indexagentica.com/entries/inspect-ai.md",
        "json": "https://indexagentica.com/api/entries/inspect-ai.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/inspect-ai.json"
      }
    },
    {
      "id": "laminar",
      "name": "Laminar",
      "category": "evals-observability",
      "summary": "Open-source observability platform purpose-built for AI agents: trace, evaluate and debug agent failures.",
      "url": "https://laminar.sh",
      "repo": "https://github.com/lmnr-ai/lmnr",
      "tags": [
        "observability",
        "evals",
        "tracing",
        "open-source"
      ],
      "license": "Apache-2.0",
      "status": "active",
      "sources": [
        "https://github.com/lmnr-ai/lmnr",
        "https://laminar.sh"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Laminar",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/laminar/",
        "markdown": "https://indexagentica.com/entries/laminar.md",
        "json": "https://indexagentica.com/api/entries/laminar.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/laminar.json"
      }
    },
    {
      "id": "langfuse",
      "name": "Langfuse",
      "category": "evals-observability",
      "summary": "Open-source agent evals and observability platform: trace, evaluate and improve LLM applications and agents.",
      "url": "https://langfuse.com",
      "repo": "https://github.com/langfuse/langfuse",
      "docs": "https://langfuse.com/docs",
      "tags": [
        "observability",
        "evals",
        "tracing",
        "open-source"
      ],
      "status": "active",
      "agent_access": {
        "llms_txt": "https://langfuse.com/llms.txt"
      },
      "related": [
        "langsmith",
        "arize-phoenix"
      ],
      "sources": [
        "https://github.com/langfuse/langfuse",
        "https://langfuse.com",
        "https://langfuse.com/llms.txt"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Langfuse",
      "longform": [
        {
          "type": "stack",
          "id": "agent-that-can-buy-things",
          "title": "Agent that can buy things",
          "url": "https://indexagentica.com/stacks/agent-that-can-buy-things/",
          "json": "https://indexagentica.com/api/longform/stacks/agent-that-can-buy-things.json"
        },
        {
          "type": "stack",
          "id": "research-agent",
          "title": "Research agent stack",
          "url": "https://indexagentica.com/stacks/research-agent/",
          "json": "https://indexagentica.com/api/longform/stacks/research-agent.json"
        }
      ],
      "links": {
        "html": "https://indexagentica.com/entries/langfuse/",
        "markdown": "https://indexagentica.com/entries/langfuse.md",
        "json": "https://indexagentica.com/api/entries/langfuse.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/langfuse.json"
      }
    },
    {
      "id": "langsmith",
      "name": "LangSmith",
      "category": "evals-observability",
      "summary": "LangChain's agent and LLM observability and evaluation platform: tracing, monitoring, cost and latency tracking.",
      "url": "https://www.langchain.com/langsmith",
      "repo": "https://github.com/langchain-ai/langsmith-sdk",
      "docs": "https://docs.langchain.com/langsmith/observability",
      "tags": [
        "observability",
        "evals",
        "tracing"
      ],
      "status": "active",
      "agent_access": {
        "llms_txt": "https://docs.langchain.com/llms.txt",
        "auth": "api-key"
      },
      "related": [
        "langgraph",
        "langfuse"
      ],
      "sources": [
        "https://www.langchain.com/langsmith",
        "https://docs.langchain.com/langsmith/observability",
        "https://github.com/langchain-ai/langsmith-sdk"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "LangChain",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/langsmith/",
        "markdown": "https://indexagentica.com/entries/langsmith.md",
        "json": "https://indexagentica.com/api/entries/langsmith.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/langsmith.json"
      }
    },
    {
      "id": "mcp-inspector",
      "name": "MCP Inspector",
      "category": "evals-observability",
      "summary": "Official interactive developer tool for testing and debugging MCP servers in the browser, on the command line or in the terminal.",
      "url": "https://modelcontextprotocol.io/docs/tools/inspector",
      "repo": "https://github.com/modelcontextprotocol/inspector",
      "tags": [
        "mcp",
        "debugging",
        "testing",
        "developer-tools"
      ],
      "pricing": "open-source",
      "status": "active",
      "related": [
        "model-context-protocol"
      ],
      "sources": [
        "https://github.com/modelcontextprotocol/inspector",
        "https://modelcontextprotocol.io/docs/tools/inspector"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "MCP project",
      "longform": [
        {
          "type": "guide",
          "id": "connect-an-agent-to-a-remote-mcp-server",
          "title": "Connect an agent to a remote MCP server",
          "url": "https://indexagentica.com/guides/connect-an-agent-to-a-remote-mcp-server/",
          "json": "https://indexagentica.com/api/longform/guides/connect-an-agent-to-a-remote-mcp-server.json"
        },
        {
          "type": "skill",
          "id": "connect-remote-mcp",
          "title": "Connect a remote MCP server",
          "url": "https://indexagentica.com/skills/connect-remote-mcp/",
          "json": "https://indexagentica.com/api/longform/skills/connect-remote-mcp.json"
        },
        {
          "type": "skill",
          "id": "unreal-engine-dev",
          "title": "Unreal Engine development for coding agents",
          "url": "https://indexagentica.com/skills/unreal-engine-dev/",
          "json": "https://indexagentica.com/api/longform/skills/unreal-engine-dev.json"
        }
      ],
      "links": {
        "html": "https://indexagentica.com/entries/mcp-inspector/",
        "markdown": "https://indexagentica.com/entries/mcp-inspector.md",
        "json": "https://indexagentica.com/api/entries/mcp-inspector.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/mcp-inspector.json"
      }
    },
    {
      "id": "openllmetry",
      "name": "OpenLLMetry",
      "category": "evals-observability",
      "summary": "Traceloop's open-source observability for GenAI/LLM applications, built on OpenTelemetry.",
      "url": "https://www.traceloop.com/openllmetry",
      "repo": "https://github.com/traceloop/openllmetry",
      "docs": "https://www.traceloop.com/docs",
      "tags": [
        "observability",
        "opentelemetry",
        "tracing",
        "open-source"
      ],
      "license": "Apache-2.0",
      "pricing": "open-source",
      "status": "active",
      "related": [
        "otel-genai-semconv"
      ],
      "sources": [
        "https://github.com/traceloop/openllmetry",
        "https://www.traceloop.com/openllmetry"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Traceloop",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/openllmetry/",
        "markdown": "https://indexagentica.com/entries/openllmetry.md",
        "json": "https://indexagentica.com/api/entries/openllmetry.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/openllmetry.json"
      }
    },
    {
      "id": "opik",
      "name": "Opik",
      "category": "evals-observability",
      "summary": "Comet's open-source platform to debug, evaluate and monitor LLM apps, RAG systems and agentic workflows.",
      "url": "https://www.comet.com/docs/opik/",
      "repo": "https://github.com/comet-ml/opik",
      "tags": [
        "observability",
        "evals",
        "tracing",
        "open-source"
      ],
      "license": "Apache-2.0",
      "status": "active",
      "sources": [
        "https://github.com/comet-ml/opik",
        "https://www.comet.com/docs/opik/"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Comet",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/opik/",
        "markdown": "https://indexagentica.com/entries/opik.md",
        "json": "https://indexagentica.com/api/entries/opik.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/opik.json"
      }
    },
    {
      "id": "promptfoo",
      "name": "Promptfoo",
      "category": "evals-observability",
      "summary": "Open-source tool to test and red-team prompts, agents and RAG: automated evals, vulnerability scanning and model comparison.",
      "url": "https://www.promptfoo.dev",
      "repo": "https://github.com/promptfoo/promptfoo",
      "docs": "https://www.promptfoo.dev/docs/intro/",
      "tags": [
        "evals",
        "red-teaming",
        "security",
        "cli"
      ],
      "license": "MIT",
      "status": "active",
      "agent_access": {
        "llms_txt": "https://www.promptfoo.dev/llms.txt"
      },
      "sources": [
        "https://github.com/promptfoo/promptfoo",
        "https://www.promptfoo.dev/docs/intro/"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Promptfoo",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/promptfoo/",
        "markdown": "https://indexagentica.com/entries/promptfoo.md",
        "json": "https://indexagentica.com/api/entries/promptfoo.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/promptfoo.json"
      }
    },
    {
      "id": "wandb-weave",
      "name": "W&B Weave",
      "category": "evals-observability",
      "summary": "Weights & Biases toolkit for tracking, testing and improving LLM-powered applications.",
      "url": "https://wandb.ai/site/weave/",
      "repo": "https://github.com/wandb/weave",
      "docs": "https://docs.coreweave.com/products/wandb/weave",
      "tags": [
        "observability",
        "evals",
        "tracing"
      ],
      "license": "Apache-2.0",
      "status": "active",
      "sources": [
        "https://github.com/wandb/weave",
        "https://docs.coreweave.com/products/wandb/weave"
      ],
      "added": "2026-10-02",
      "updated": "2026-10-02",
      "last_verified": "2026-10-02",
      "submitted_by": "agentica-curator",
      "maintainer": "Weights & Biases (CoreWeave)",
      "longform": [],
      "links": {
        "html": "https://indexagentica.com/entries/wandb-weave/",
        "markdown": "https://indexagentica.com/entries/wandb-weave.md",
        "json": "https://indexagentica.com/api/entries/wandb-weave.json",
        "source": "https://github.com/Drudley/indexagentica/blob/main/content/evals-observability/wandb-weave.json"
      }
    }
  ]
}