.github/workflows/evals.yml

name: evals

# Structural, routing, and behavioral testing for every skill in this catalog,
# plus the references our markdown makes, run by skillscope
# (https://github.com/amd/skillscope). The harness used to live here, under
# eval/ and .github/scripts/select_evals.py, with the reference half in
# external-reference-check.yml and .github/lychee.toml; it moved out so other
# repos could run the same graders against their own skills, and what is left
# is the half that was always specific to amd/skills -- which skills are
# published, which runners we own, and whose key pays for a run.
#
# The `with:` block below is the whole configuration. Everything skillscope
# needs to know about this repo is an input here, so there is no second file to
# keep in step:
#
#   * routing_room  -- the published bundle, installed side by side.
#   * behavior_*    -- the everyday Strix Halo pool, on both platforms.
#   * scoped_*      -- the Instinct pool: its own labels, its own gate label,
#                      its own credentials. A skill asks for that hardware in
#                      its evals/machine.yml; this file says what it costs.
#   * api_*         -- AMD's internal LLM gateway, named by secret rather than
#                      value so the called workflow never learns either.
#
# Selection is still by change: routing runs only when a change can move a
# routing decision, and a behavioral leg runs only for the skills a change
# touches. `skillscope select` makes that call, so it can be re-run by hand.

on:
  pull_request:
    # `labeled` is here so adding a gate label (e.g. enable_mi_ci) to an open PR
    # starts that leg. Re-running an existing run would not work: the replayed
    # event payload is the one from before the label was applied.
    types: [opened, synchronize, reopened, labeled]
    paths:
      - "skills/**"
      # This file holds the routing room, the runner topology, and the harness
      # pin, so a change to it can move any result. It is passed as an infra
      # path below for the same reason.
      - ".github/workflows/evals.yml"
      # The structural checks resolve every reference our prose makes, so the
      # prose is an input too. Free to check, and cheap to be wrong about.
      - "**/*.md"
  # Catches external link rot in markdown nobody is editing. Runs the free
  # checks only -- see `mode` below -- so a quiet week costs one ubuntu runner.
  schedule:
    # Mondays at 12:00 UTC.
    - cron: "0 12 * * 1"
  workflow_dispatch:
    inputs:
      mode:
        description: "Which grader to run."
        type: choice
        options: [both, routing, behavioral, references]
        default: both
      skills:
        description: "Comma-separated skill names (blank = every skill with a dataset)."
        required: false
        default: ""
      only:
        description: "Comma-separated case ids to run (routing only)."
        required: false
        default: ""
      min_accuracy:
        description: "Fail the routing run below this accuracy (0-1). 0 reports without gating."
        required: false
        default: "0"

concurrency:
  group: evals-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  # Lets a job ask GitHub for an OIDC token, which is traded for a
  # short-lived Anthropic one instead of reading a key.
  id-token: write

jobs:
  # Mark the `results` check this produces as required in branch protection.
  evals:
    # This pin is the harness. Each job in the called workflow checks skillscope
    # out at the commit this ref names and runs it from there, so bumping the
    # line below is the whole of upgrading -- there is no second version to keep
    # in step with it.
    uses: amd/skillscope/.github/workflows/skill-evals.yml@v0.1.3
    # By name, not by value: the Instinct legs read a secret scoped to the
    # `behavioral-instinct` environment, and an environment's secrets are
    # readable only inside the job that declares it.
    secrets: inherit
    with:
      # --- what this repo looks like ---------------------------------------
      skill_globs: skills/*

      # The published bundle, in the order .claude-plugin/marketplace.json
      # lists it. Listed rather than read from the manifest, because who a
      # skill competes against is what its score means: adding a name here
      # moves every other skill's number, and that deserves a visible diff.
      # Publishing a skill means adding it in both places.
      routing_room: "local-ai-use,local-ai-app-integration,serving-llms-on-instinct,tracelens-analysis-orchestrator,hyperloom-workload-optimizer"

      # A change to this file is a change to the harness as far as this repo is
      # concerned, so it re-runs every skill rather than whatever the diff
      # happened to touch.
      infra_paths: .github/workflows/evals.yml

      # Prose outside the skills whose references are checked too. The skills'
      # own markdown is always checked; this adds the catalog's documentation,
      # its walkthroughs, and the skills still in staging.
      doc_globs: "*.md,docs/**/*.md,walkthroughs/*.md,staging/**/*.md"

      # What this catalog asks of a skill on top of the format: a governance
      # card, with those three sections filled in. See docs/skill-requirements.md.
      skill_files: skill-card.md
      skill_sections: Description,Owner,License

      # --- where the graded runs happen ------------------------------------
      #
      # A routing decision is made from skill descriptions alone, so it needs
      # no GPU and no second platform. It stays on a self-hosted Linux runner
      # only because that is what can reach the internal LLM gateway.
      routing_runner: '["self-hosted", "strix_halo", "Linux"]'
      behavior_runner: '["self-hosted", "strix_halo"]'
      behavior_os: Linux,Windows

      # The Instinct pool. A skill lands here by asking for these labels in its
      # evals/machine.yml; nothing in this file names a skill, because a
      # hardcoded skill-to-runner table drifts from reality the first time a
      # skill is added. The pool is scarce, so touching such a skill is
      # necessary but not sufficient: a maintainer opts the PR in with the gate
      # label. It also sits outside the AMD network and cannot reach the
      # internal gateway, so it calls api.anthropic.com with its own budgeted
      # key from its own environment.
      scoped_runner: '["self-hosted", "Linux", "X64"]'
      scoped_gate: enable_mi_ci
      scoped_environment: behavioral-instinct

      # --- this run --------------------------------------------------------
      #
      # `references` runs the structural and external checks and skips the
      # graded ones, which is all the weekly schedule is here for: a cron that
      # quietly re-ran every skill against a real agent would be a standing
      # bill nobody asked for.
      mode: ${{ github.event_name == 'schedule' && 'references' || inputs.mode || 'both' }}
      skills: ${{ inputs.skills }}
      only: ${{ inputs.only }}

      # Reports the routing score without gating on it, which is what this
      # catalog has always done. Raise it once the prompts have a number worth
      # holding a bar against.
      min_accuracy: ${{ inputs.min_accuracy || '0' }}

      # A skill's optional evals/extended_evals.json is that skill's product
      # repo's bill to pay; this catalog grades the required dataset only.
      # Extended datasets are still checked structurally.
      extended: false

      # Fetch every external URL our markdown links to, in a job of its own
      # that the aggregate `results` check ignores. Leave it out of branch
      # protection: a rate limit or a runner IP some host blocks is not a
      # reason to hold a merge. The weekly schedule above is what catches rot
      # in markdown nobody is editing.
      external_references: true
      excluded_urls: '^https://openai\.com/codex/?$,^https://www\.amd\.com/en/resources/product-security\.html$'

      # AMD's internal gateway. The key is named rather than passed, and
      # `$API_KEY` in the headers is replaced with it, so a gateway that wants
      # the key in a header needs no second secret.
      api_key_secret: ORCHESTR_API_KEY
      api_base_url: https://llm-api.amd.com/Anthropic
      api_custom_headers: |
        Ocp-Apim-Subscription-Key: $API_KEY
        user: a1_ucicd

      # Some runners talk straight to api.anthropic.com, so they can use GitHub OIDC tokens to get temporary Anthropic access instead of storing a key. 
      # (The gateway runners above can't, since those tokens only work with api.anthropic.com.)
      # The four fields below are just IDs—no secrets—and only work if GitHub sends a signed token from this repo.
      scoped_api_key_secret: ""
      scoped_federation_rule_id: fdrl_014u39zsJmsex5SkL1gWg7qo
      scoped_federation_organization_id: 9bdff596-1595-4140-8301-817e43ef8412
      scoped_federation_service_account_id: svac_016zcTqZ4XkFdjAJCag9jQC9
      scoped_federation_workspace_id: wrkspc_01EUPtU7mqFtoQowMAREPvoq