#!/usr/bin/env bash
#ddev-generated
## Description: Hand one dataset task to an agent adapter and record the attempt; grading stays a separate verb (ddev ai-eval-grade)
## Usage: ai-eval-run <task-id> [--dataset=<ref>] [--agent=<name>] [--model=<id>] [--print-prompt] [--list-agents]
## Example: "ddev ai-eval-run dv01_view_build --agent=manual"
set -euo pipefail

# Run and grade are separate verbs, on purpose: a failing ATTEMPT and a
# failing GRADE are different facts and a harness must never confuse them.
# This command never grades. Its outcome answers one question only: did an
# attempt run to completion?
#
#   0  an attempt ran (whether it built the right thing is grade's business)
#   2  usage error                      APG_RUN_USAGE
#   3  task not in the dataset          APG_RUN_TASK_NOT_FOUND
#   4  no adapter by that name          APG_RUN_NO_ADAPTER
#   5  credential preflight refused     APG_RUN_CRED_REFUSED
#   1  the adapter ran and failed       APG_RUN_ATTEMPT_FAILED
#
# Those codes are what this script returns, but a caller invoking it as
# `ddev ai-eval-run` will NOT see them: ddev collapses every non-zero exit
# from a custom command to 1 (verified on ddev v1.25.1, host and web
# commands alike; the real code survives only as text on stderr,
# "error=exit status 5"). By contrast `ddev exec` passes codes through
# faithfully. So each outcome above also carries a stable stderr token that
# survives the collapse, and the authoritative machine-readable record of a
# run is run.json in the run directory, which does not depend on ddev at
# all: read `attempted` and `adapter_exit_code` there.
#
# Two properties are non-negotiable for an eval bed and both are enforced
# here rather than left as documentation:
#
#   Vanilla by default. The default adapter is inert: it ships no agent, no
#   skills and no rule set. A tuned harness is something this bed MEASURES,
#   never something it ships. Selecting a real runtime is always an explicit
#   act (--agent, or AI_PG_AGENT).
#
#   Bed-scoped credentials. An eval bed runs untrusted attempts by
#   definition, so an adapter that hands credentials to an agent must not be
#   handed the host's. The preflight below looks for host-wide credential
#   sharing in the bed's own DDEV config and refuses (exit 5) when the
#   selected adapter needs credentials. Override deliberately with
#   AI_PG_ALLOW_HOST_CREDS=1 if you know the bed is disposable.

APPROOT="${DDEV_APPROOT:?ai-eval-run must run as a ddev command from a project root}"
if ! command -v python3 >/dev/null 2>&1; then
  echo "APG_RUN_USAGE python3 is required on the host (this command reads JSON and writes the run manifest with it)." >&2
  exit 2
fi
STATE_DIR="${AI_PG_BASELINE_DIR:-${APPROOT}/.ai-proving-ground}"
# Container-side path to the recipe; the drush call runs inside web.
AI_PG_ROOT="${AI_PG_ROOT:-/var/www/html/recipes/ai_proving_ground}"

TASK_ID=""
DATASET="droost_build_tasks"
AGENT="${AI_PG_AGENT:-manual}"
MODEL=""
PRINT_PROMPT=0
LIST_AGENTS=0

for arg in "$@"; do
  case "${arg}" in
    --dataset=*) DATASET="${arg#*=}" ;;
    --agent=*)   AGENT="${arg#*=}" ;;
    --model=*)   MODEL="${arg#*=}" ;;
    --print-prompt) PRINT_PROMPT=1 ;;
    --list-agents)  LIST_AGENTS=1 ;;
    -*) echo "APG_RUN_USAGE unknown option '${arg}'. See 'ddev ai-eval-run --help'." >&2; exit 2 ;;
    *)
      if [ -n "${TASK_ID}" ]; then
        echo "APG_RUN_USAGE more than one task id given ('${TASK_ID}' and '${arg}')." >&2
        exit 2
      fi
      TASK_ID="${arg}"
      ;;
  esac
done

# Adapter search order: the bed's installed adapters first, so a bed can drop
# in its own runtime without editing the recipe, then the recipe checkout, so
# a bed that copied the commands by hand still finds the shipped ones.
AGENT_DIRS=("${APPROOT}/.ddev/agents" "${AI_PG_AGENT_DIR:-${APPROOT}/recipes/ai_proving_ground/ddev/agents}")

find_agent() {
  local name="$1" dir
  for dir in "${AGENT_DIRS[@]}"; do
    if [ -f "${dir}/${name}" ]; then
      printf '%s\n' "${dir}/${name}"
      return 0
    fi
  done
  return 1
}

# An adapter declares what it does with credentials in a header line, the same
# way a ddev command declares its description:
#   ## Credentials: none | bed-scoped | host-shared
# Anything else, or a missing line, reads as "unknown" and is treated as
# needing credentials. Fail closed: an eval bed that guesses in the
# permissive direction is exactly the failure this project exists to catch.
agent_credentials() {
  local file="$1" declared
  declared="$(sed -n 's/^## Credentials:[[:space:]]*//p' "${file}" | head -1 | tr -d '[:space:]' || true)"
  case "${declared}" in
    none|bed-scoped|host-shared) printf '%s\n' "${declared}" ;;
    *) printf 'unknown\n' ;;
  esac
}

agent_description() {
  sed -n 's/^## Description:[[:space:]]*//p' "$1" | head -1 || true
}

if [ "${LIST_AGENTS}" -eq 1 ]; then
  echo "Adapters visible to this bed (search order):"
  found=0
  for dir in "${AGENT_DIRS[@]}"; do
    [ -d "${dir}" ] || continue
    for f in "${dir}"/*; do
      [ -f "${f}" ] || continue
      found=1
      printf '  %-14s credentials:%-12s %s\n' \
        "$(basename "${f}")" "$(agent_credentials "${f}")" "$(agent_description "${f}")"
      printf '  %-14s %s\n' "" "${f}"
    done
  done
  [ "${found}" -eq 1 ] || echo "  (none found; looked in ${AGENT_DIRS[*]})"
  exit 0
fi

if [ -z "${TASK_ID}" ]; then
  echo "APG_RUN_USAGE usage: ddev ai-eval-run <task-id> [--dataset=<ref>] [--agent=<name>] [--model=<id>]" >&2
  echo "       ddev ai-eval-run --list-agents" >&2
  exit 2
fi

# --- Resolve the task through the same loader ai-eval-grade uses ------------
STDERR_FILE="$(mktemp)"
trap 'rm -f "${STDERR_FILE}"' EXIT
set +e
RAW="$(ddev exec drush php:script --script-path="${AI_PG_ROOT}/scripts" task.php -- "${TASK_ID}" "${DATASET}" 2>"${STDERR_FILE}")"
TASK_RC=$?
set -e
if [ "${TASK_RC}" -ne 0 ]; then
  cat "${STDERR_FILE}" >&2
  if grep -q 'APG_TASK_NOT_FOUND' "${STDERR_FILE}"; then
    echo "APG_RUN_TASK_NOT_FOUND ${TASK_ID} is not in ${DATASET}" >&2
    exit 3
  fi
  echo "ERROR: could not resolve task '${TASK_ID}' in the bed." >&2
  exit 1
fi
# drush may print its own lines around ours; the run request is the single
# JSON line, so take the last line that opens an object.
REQUEST="$(printf '%s\n' "${RAW}" | grep -E '^\{' | tail -1 || true)"
if [ -z "${REQUEST}" ]; then
  cat "${STDERR_FILE}" >&2
  echo "ERROR: the bed returned no run request for '${TASK_ID}'." >&2
  exit 1
fi

json_field() {
  printf '%s' "${REQUEST}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get(sys.argv[1], ""))' "$1"
}
PROMPT="$(json_field prompt)"
GRADEABLE="$(json_field gradeable)"
DATASET_REF="$(json_field dataset)"

if [ "${PRINT_PROMPT}" -eq 1 ]; then
  printf '%s\n' "${PROMPT}"
  exit 0
fi

if [ "${GRADEABLE}" != "True" ] && [ "${GRADEABLE}" != "true" ]; then
  echo "NOTE: '${TASK_ID}' carries no metadata.droost spec, so ai-eval-grade will skip it" >&2
  echo "      (APG_GRADE_SKIP on stderr; no envelope). The attempt can still run; nothing will grade it." >&2
fi

# --- Select the adapter -----------------------------------------------------
if ! AGENT_FILE="$(find_agent "${AGENT}")"; then
  echo "APG_RUN_NO_ADAPTER no adapter named '${AGENT}'. Looked in: ${AGENT_DIRS[*]}" >&2
  echo "       See 'ddev ai-eval-run --list-agents'." >&2
  exit 4
fi
if [ ! -x "${AGENT_FILE}" ]; then
  echo "APG_RUN_NO_ADAPTER adapter '${AGENT_FILE}' is not executable (chmod +x it)." >&2
  exit 4
fi
CRED_MODE="$(agent_credentials "${AGENT_FILE}")"

# --- Credential preflight ---------------------------------------------------
# Reports only the NAMES of what it finds, never a value: this prints to a
# terminal and lands in a manifest that may be attached to a run.
# Installed recipe first, then the checkout containing this command, matching
# the adapter lookup's installed/checkout search pattern.
PREFLIGHT_DIRS=("${APPROOT}/recipes/ai_proving_ground/scripts"
  "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)/scripts")
PREFLIGHT=""
for dir in "${PREFLIGHT_DIRS[@]}"; do
  if [ -f "${dir}/cred-preflight.sh" ]; then
    PREFLIGHT="${dir}/cred-preflight.sh"
    break
  fi
done
if [ -z "${PREFLIGHT}" ]; then
  echo "APG_RUN_USAGE cannot locate scripts/cred-preflight.sh." >&2
  exit 2
fi
# Capture first so a detector failure cannot silently bypass the preflight,
# and fail closed with the stable token a harness already watches for.
if ! FINDINGS="$(bash "${PREFLIGHT}" "${APPROOT}")"; then
  echo "APG_RUN_CRED_REFUSED credential preflight could not inspect ${APPROOT}/.ddev; refusing to run." >&2
  exit 5
fi
cred_findings=()
while IFS= read -r finding; do
  [ -z "${finding}" ] || cred_findings+=("${finding}")
done <<< "${FINDINGS}"

SCOPE="bed-scoped"
if [ "${#cred_findings[@]}" -gt 0 ]; then
  SCOPE="host-shared"
fi

if [ "${CRED_MODE}" != "none" ] && [ "${SCOPE}" = "host-shared" ]; then
  if [ "${AI_PG_ALLOW_HOST_CREDS:-0}" != "1" ]; then
    {
      echo "APG_RUN_CRED_REFUSED adapter '${AGENT}' (credentials: ${CRED_MODE}) would run against a bed that shares host credentials."
      echo "An eval bed runs untrusted attempts by definition, so the agent's credentials must be scoped to the bed and the run."
      echo "What this bed exposes:"
      for f in "${cred_findings[@]}"; do echo "  - ${f}"; done
      echo
      echo "Fix one of these, then re-run:"
      echo "  - give the bed its own credentials and drop the host mount, or"
      echo "  - use an adapter that declares '## Credentials: none' (see --list-agents), or"
      echo "  - set AI_PG_ALLOW_HOST_CREDS=1 if this bed is disposable and you accept the exposure."
    } >&2
    exit 5
  fi
  echo "WARNING: running with host-shared credentials because AI_PG_ALLOW_HOST_CREDS=1." >&2
fi

# --- Run directory and manifest --------------------------------------------
# Lives beside the baselines, which ai-eval-reset does NOT restore, so the
# record of an attempt survives the reset that follows it.
RUN_ID="${TASK_ID}-$(date -u +%Y%m%dT%H%M%SZ)-$(od -An -N2 -tx1 /dev/urandom | tr -d ' \n')"
RUN_DIR="${STATE_DIR}/runs/${RUN_ID}"
mkdir -p "${RUN_DIR}"
printf '%s\n' "${PROMPT}" > "${RUN_DIR}/prompt.txt"
printf '%s\n' "${REQUEST}" > "${RUN_DIR}/request.json"
PROMPT_SHA="$(sha256sum "${RUN_DIR}/prompt.txt" | cut -d' ' -f1)"
STARTED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

write_manifest() {
  # New file then rename: the bind mount can serve a stale view of an
  # in-place rewrite across the host/container boundary on eCryptfs hosts.
  RUN_ID="${RUN_ID}" TASK_ID="${TASK_ID}" DATASET_REF="${DATASET_REF}" \
  AGENT="${AGENT}" AGENT_FILE="${AGENT_FILE}" CRED_MODE="${CRED_MODE}" \
  SCOPE="${SCOPE}" MODEL="${MODEL}" STARTED="${STARTED}" FINISHED="${1}" \
  RC="${2}" ATTEMPTED="${3}" PROMPT_SHA="${PROMPT_SHA}" APPROOT="${APPROOT}" \
  FINDINGS="$(printf '%s\n' "${cred_findings[@]+"${cred_findings[@]}"}")" \
  python3 - "${RUN_DIR}/run.json" <<'PY'
import json, os, sys
findings = [f for f in os.environ.get("FINDINGS", "").split("\n") if f]
manifest = {
    "apg_run_manifest": "1",
    "run_id": os.environ["RUN_ID"],
    "task_id": os.environ["TASK_ID"],
    "dataset": os.environ["DATASET_REF"],
    "agent": os.environ["AGENT"],
    "agent_path": os.environ["AGENT_FILE"],
    "agent_credentials": os.environ["CRED_MODE"],
    "bed_credential_scope": os.environ["SCOPE"],
    "bed_credential_findings": findings,
    "model": os.environ["MODEL"],
    "started": os.environ["STARTED"],
    "finished": os.environ["FINISHED"],
    "adapter_exit_code": int(os.environ["RC"]),
    "attempted": os.environ["ATTEMPTED"] == "1",
    "prompt_sha256": os.environ["PROMPT_SHA"],
    "approot": os.environ["APPROOT"],
}
target = sys.argv[1]
with open(target + ".tmp", "w") as fh:
    json.dump(manifest, fh, indent=2)
    fh.write("\n")
os.replace(target + ".tmp", target)
PY
}

echo "==> task ${TASK_ID} (${DATASET_REF})"
echo "==> adapter ${AGENT} (credentials: ${CRED_MODE}; bed: ${SCOPE})"
echo "==> run ${RUN_DIR}"
write_manifest "" 0 0

# The adapter inherits this terminal on purpose: a real agent CLI is often
# interactive, and the run request reaches it by file and environment rather
# than on stdin so it can still read from the operator.
set +e
APG_RUN_DIR="${RUN_DIR}" \
APG_TASK_ID="${TASK_ID}" \
APG_DATASET="${DATASET_REF}" \
APG_PROMPT_FILE="${RUN_DIR}/prompt.txt" \
APG_REQUEST_FILE="${RUN_DIR}/request.json" \
APG_MODEL="${MODEL}" \
APG_APPROOT="${APPROOT}" \
APG_DDEV_PROJECT="${DDEV_PROJECT:-}" \
  "${AGENT_FILE}"
ADAPTER_RC=$?
set -e

FINISHED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
if [ "${ADAPTER_RC}" -eq 0 ]; then
  write_manifest "${FINISHED}" "${ADAPTER_RC}" 1
  echo
  echo "OK: attempt recorded at ${RUN_DIR}"
  echo "Grade it (separate verb; this command never grades):"
  if [ -n "${MODEL}" ]; then
    echo "  ddev ai-eval-grade ${TASK_ID} ${DATASET} ${MODEL} --run=${RUN_ID}"
  else
    echo "  ddev ai-eval-grade ${TASK_ID} ${DATASET} --run=${RUN_ID}"
  fi
  exit 0
fi

write_manifest "${FINISHED}" "${ADAPTER_RC}" 0
echo "APG_RUN_ATTEMPT_FAILED adapter '${AGENT}' exited ${ADAPTER_RC}; no attempt was recorded as complete." >&2
echo "       Manifest: ${RUN_DIR}/run.json" >&2
exit 1
