Dynamic Discovery

Automatically find and evaluate all skills that have eval cases - no need to hardcode skill names.

name: Skill Eval
on:
  pull_request:
    paths:
      - 'skills/**'

permissions:
  contents: read
  pull-requests: write

jobs:
  discover:
    runs-on: ubuntu-latest
    outputs:
      skills: ${{ steps.find.outputs.skills }}
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
          sparse-checkout: skills

      - name: Find skills with evals
        id: find
        run: |
          skills=$(find skills -name "*.yaml" -path "*/evals/*" \
            -exec dirname {} \; | xargs -I{} dirname {} | \
            xargs -I{} basename {} | sort -u | \
            jq -R -s -c 'split("\n") | map(select(. != ""))')
          echo "skills=$skills" >> "$GITHUB_OUTPUT"
          echo "Found: $skills"

  eval:
    needs: discover
    if: needs.discover.outputs.skills != '[]'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        skill: ${{ fromJSON(needs.discover.outputs.skills) }}
    steps:
      - uses: actions/checkout@v6

      - uses: skill-bench/skill-eval-action@v1
        with:
          skill-name: ${{ matrix.skill }}
          skill-path: skills/${{ matrix.skill }}
          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}

This uses sparse-checkout in the discover step to only fetch the skills/ directory, keeping it fast.