Evals

# agents/eval_questions.yml
eval_questions:
  - name: revenue_by_quarter
    question: What was our revenue by quarter over the last two years?
    certifiedQuery: revenue_by_quarter        # reference an existing certified query by name

  - name: arr_last_4_years
    question: What was our ARR over the last 4 years?
    sql: |                                    # ...or inline SQL ground truth
      SELECT date_trunc('year', created_at) AS year, SUM(arr) AS arr
      FROM subscriptions GROUP BY 1 ORDER BY 1

env:
  CUBE_API_URL: ${{ vars.CUBE_API_URL }}
  CUBE_API_KEY: ${{ secrets.CUBE_API_KEY }}
  DEPLOYMENT_ID: ${{ vars.CUBE_DEPLOYMENT_ID }}
  CUBE_BRANCH: ${{ github.head_ref || github.ref_name }}

steps:
  - name: Install Cube CLI
    env:
      CUBE_VERSION: ${{ vars.CUBE_CLI_VERSION }}
    shell: bash
    run: |
      set -euo pipefail
      test -n "$CUBE_VERSION"
      curl -fsSL https://raw.githubusercontent.com/cube-js/cube/master/install-cli.sh | sh

  - name: Run Cube agent evals
    shell: bash
    timeout-minutes: 35
    run: |
      set -euo pipefail
      trap 'if [[ -f eval.json && ! -s eval.json ]]; then rm -f eval.json; fi' EXIT
      cube evals run "$DEPLOYMENT_ID" \
        --branch "$CUBE_BRANCH" \
        --wait \
        --timeout 30m \
        --json > eval.json

  - name: Upload eval result
    if: always()
    uses: actions/upload-artifact@v4
    with:
      name: cube-eval
      path: eval.json
      if-no-files-found: ignore

env:
  CUBE_API_URL: ${{ vars.CUBE_API_URL }}
  CUBE_API_KEY: ${{ secrets.CUBE_API_KEY }}
  DEPLOYMENT_ID: ${{ vars.CUBE_DEPLOYMENT_ID }}
  CUBE_BRANCH: ${{ github.head_ref || github.ref_name }}

steps:
  - name: Run Cube agent evals through the Platform API
    shell: bash
    timeout-minutes: 35
    run: |
      set -euo pipefail

      api="${CUBE_API_URL%/}/api/v1/deployments/$DEPLOYMENT_ID/evaluations"
      auth=(-H "Authorization: Api-Key $CUBE_API_KEY")
      reads=(
        curl --fail --silent --show-error
        --connect-timeout 10 --max-time 30
        --retry 4 --retry-delay 2 --retry-all-errors
        "${auth[@]}"
      )

      started=$(
        jq -cn --arg branch "$CUBE_BRANCH" '{branchName: $branch}' |
          curl --fail-with-body --silent --show-error \
            --connect-timeout 10 --max-time 30 \
            "${auth[@]}" -H 'Content-Type: application/json' \
            --data-binary @- "$api"
      )
      evaluation_id=$(jq -er '.id | select(type == "number")' <<<"$started")

      deadline=$((SECONDS + 1800))
      last_status=
      while :; do
        if (( SECONDS >= deadline )); then
          echo "Timed out waiting for eval run $evaluation_id" >&2
          exit 1
        fi

        "${reads[@]}" --output run.json "$api/$evaluation_id"
        status=$(jq -r '.status | if type == "string" then ascii_downcase else "" end' run.json)
        if [[ "$status" != "$last_status" ]]; then
          echo "Eval run $evaluation_id: $status"
          last_status=$status
        fi

        case "$status" in
          completed|failed) break ;;
          *) sleep 5 ;;
        esac
      done

      "${reads[@]}" --output results.json "$api/$evaluation_id/results"
      jq -n --slurpfile evalRun run.json --slurpfile results results.json \
        '{evalRun: $evalRun[0], results: $results[0]}' > eval.json
      jq . eval.json

      if ! jq -e '
        (.evalRun.status | ascii_downcase) == "completed" and
        (.results.items | type) == "array" and
        (.results.items | length) > 0 and
        .results.pageInfo.hasNextPage == false and
        all(.results.items[]; (.verdict // "" | ascii_downcase) == "pass")
      ' eval.json > /dev/null; then
        echo "The eval run failed, was incomplete, or did not pass every question" >&2
        exit 1
      fi

  - name: Upload eval result
    if: always()
    uses: actions/upload-artifact@v4
    with:
      name: cube-eval
      path: eval.json
      if-no-files-found: ignore