#!/usr/bin/env bash # Agente de diagnóstico/corrección del lab GitOps de cruzcloud. # # Diseño de seguridad (importante, ver README.md #Diseño de seguridad): # `claude -p` se invoca con --allowedTools "" (CERO herramientas) y # --json-schema, así que Claude NUNCA ejecuta un comando directamente — # solo devuelve una decisión estructurada. Este script (bash, confiable, # revisado por humano) es el único que ejecuta kubectl/docker, y solo # tras revalidar la decisión de Claude contra su propio cálculo de # elegibilidad. Esto evita depender de --allowedTools como sandbox de # comandos: se probó en vivo que "Bash(echo pintest)" también permite # "echo pintest && echo INJECTED" (coincidencia por prefijo, no exacta), # así que un Bash tool con prefijo nunca habría sido un límite seguro # para comandos root-capaces como kubectl/docker. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="${LAB_AGENT_ENV_FILE:-${SCRIPT_DIR}/lab-agent.env}" if [[ ! -f "$ENV_FILE" ]]; then echo "Falta ${ENV_FILE} — copia lab-agent.env.example y ajústalo." >&2 exit 1 fi # shellcheck source=/dev/null source "$ENV_FILE" : "${NAMESPACE:=ecommerce}" : "${KUBECONFIG_PATH:=/DATA/.kube/config}" : "${UNHEALTHY_THRESHOLD_MIN:=10}" : "${RATE_LIMIT_MIN:=30}" : "${COMMERCE_BACKEND_DEPLOYMENT:=medusa-deploy}" : "${COMMERCE_BACKEND_WINDOW_START:=00:00}" : "${COMMERCE_BACKEND_WINDOW_END:=06:00}" : "${DOCKER_RESTART_ALLOWLIST:=gitea-runner}" : "${GITEA_ACTIONS_REPOS:=devops/apps-registry}" : "${GITEA_API_BASE:=https://gitea.cruzcloud.net/api/v1}" : "${PUBLIC_DOMAINS:=shop.cruzcloud.net commerce.cruzcloud.net media.cruzcloud.net gitea.cruzcloud.net docs.cruzcloud.net}" : "${STATE_FILE:=/DATA/AppData/k3d-lab/lab-agent/state.json}" : "${LOG_DIR:=/DATA/AppData/k3d-lab/lab-agent/logs}" : "${ANTHROPIC_API_KEY_FILE:=/etc/claude-lab-agent/secrets/anthropic_api_key}" : "${GITEA_TOKEN_FILE:=/etc/claude-lab-agent/secrets/gitea_token}" : "${CLAUDE_MODEL:=claude-sonnet-5}" : "${CLAUDE_MAX_BUDGET_USD:=0.50}" : "${NTFY_URL:=}" : "${DRY_RUN:=1}" for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=1 ;; --live) DRY_RUN=0 ;; *) echo "Argumento desconocido: $arg" >&2; exit 1 ;; esac done export KUBECONFIG="$KUBECONFIG_PATH" mkdir -p "$LOG_DIR" "$(dirname "$STATE_FILE")" RUN_TS="$(date -u +%Y%m%dT%H%M%SZ)" RUN_LOG="${LOG_DIR}/run-${RUN_TS}.log" ACTIONS_LOG="${LOG_DIR}/actions.jsonl" exec > >(tee -a "$RUN_LOG") 2>&1 iso_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } log() { echo "[$(iso_now)] $*"; } trap 'log "ERROR inesperado en línea $LINENO (comando: $BASH_COMMAND)"' ERR log "=== lab-agent inicio (DRY_RUN=${DRY_RUN}) ===" for c in kubectl docker curl jq flock claude date; do command -v "$c" >/dev/null 2>&1 || { log "FATAL: falta comando requerido '$c'"; exit 1; } done [[ -f "$STATE_FILE" ]] || echo '{"resources":{}}' > "$STATE_FILE" STATE_LOCK="${STATE_FILE}.lock" # --------------------------------------------------------------------------- # Estado persistente (rate limit + "un solo intento" tras fallo) # --------------------------------------------------------------------------- state_read() { # key -> objeto JSON (o {} si no existe) local key="$1" flock "$STATE_LOCK" jq -c --arg k "$key" '.resources[$k] // {}' "$STATE_FILE" } state_write() { # key, json-string local key="$1" value_json="$2" local tmp valfile tmp="$(mktemp "${STATE_FILE}.XXXXXX")" valfile="$(mktemp)" printf '%s' "$value_json" > "$valfile" ( flock -x 9 jq --arg k "$key" --slurpfile v "$valfile" '.resources[$k] = $v[0]' "$STATE_FILE" > "$tmp" mv "$tmp" "$STATE_FILE" ) 9>"$STATE_LOCK" rm -f "$valfile" } state_clear() { # key local key="$1" local tmp tmp="$(mktemp "${STATE_FILE}.XXXXXX")" ( flock -x 9 jq --arg k "$key" 'del(.resources[$k])' "$STATE_FILE" > "$tmp" mv "$tmp" "$STATE_FILE" ) 9>"$STATE_LOCK" } mark_action_taken() { # resource -> preserva first_unhealthy_at, marca last_action_at=ahora y blocked=true local resource="$1" prev now new_state prev="$(state_read "$resource")" now="$(iso_now)" new_state="$(jq -nc --argjson prev "$prev" --arg t "$now" \ '{first_unhealthy_at: ($prev.first_unhealthy_at // $t), last_action_at: $t, blocked: true}')" state_write "$resource" "$new_state" } log_action() { # resource decision result reason local resource="$1" decision="$2" result="$3" reason="$4" jq -nc --arg ts "$(iso_now)" --arg resource "$resource" --arg decision "$decision" \ --arg result "$result" --arg reason "$reason" --arg dry_run "$DRY_RUN" \ '{ts:$ts, resource:$resource, decision:$decision, result:$result, reason:$reason, dry_run:($dry_run=="1")}' \ >> "$ACTIONS_LOG" } # --------------------------------------------------------------------------- # Notificación (Paso 3 — no-op documentado hasta que NTFY_URL esté configurado) # --------------------------------------------------------------------------- notify_send() { # title message local title="$1" message="$2" if [[ -z "$NTFY_URL" ]]; then log "NOTIFY (sin canal configurado aún — Paso 3) [$title]" log "$message" return 0 fi if ! curl -s -m 10 -H "Title: ${title}" -d "$message" "$NTFY_URL" >/dev/null 2>&1; then log "WARN: fallo al enviar notificación a NTFY_URL" fi } # --------------------------------------------------------------------------- # Diagnóstico: deployments de ecommerce # --------------------------------------------------------------------------- DEPLOY_JSON="$(kubectl -n "$NAMESPACE" get deployment -o json 2>/dev/null)" \ || { log "FATAL: no se pudo consultar deployments en namespace ${NAMESPACE}"; exit 1; } mapfile -t DETECTED_DEPLOYMENTS < <(jq -r '.items[].metadata.name' <<<"$DEPLOY_JSON") in_commerce_backend_window() { local now_hm start end now_hm="$(date +%H:%M)" start="$COMMERCE_BACKEND_WINDOW_START" end="$COMMERCE_BACKEND_WINDOW_END" if [[ "$start" < "$end" ]]; then [[ ( "$now_hm" > "$start" || "$now_hm" == "$start" ) && "$now_hm" < "$end" ]] else [[ "$now_hm" > "$start" || "$now_hm" == "$start" || "$now_hm" < "$end" ]] fi } # devuelve: "eligible" | "not_eligible:" compute_eligibility() { # resource_key is_unhealthy window_ok local resource_key="$1" is_unhealthy="$2" window_ok="$3" local state first_unhealthy_at last_action_at blocked state="$(state_read "$resource_key")" first_unhealthy_at="$(jq -r '.first_unhealthy_at // empty' <<<"$state")" last_action_at="$(jq -r '.last_action_at // empty' <<<"$state")" blocked="$(jq -r '.blocked // false' <<<"$state")" if [[ "$is_unhealthy" != "1" ]]; then [[ -n "$first_unhealthy_at" ]] && state_clear "$resource_key" echo "not_eligible:healthy" return fi if [[ -z "$first_unhealthy_at" ]]; then first_unhealthy_at="$(iso_now)" state_write "$resource_key" "$(jq -nc --arg t "$first_unhealthy_at" '{first_unhealthy_at:$t, blocked:false}')" fi if [[ "$blocked" == "true" ]]; then echo "not_eligible:blocked_tras_intento_previo_sin_recuperacion" return fi if [[ "$window_ok" != "1" ]]; then echo "not_eligible:fuera_de_ventana_horaria_permitida" return fi local since_epoch now_e minutes_unhealthy since_epoch="$(date -u -d "$first_unhealthy_at" +%s)" now_e="$(date -u +%s)" minutes_unhealthy=$(( (now_e - since_epoch) / 60 )) if (( minutes_unhealthy < UNHEALTHY_THRESHOLD_MIN )); then echo "not_eligible:bajo_umbral(${minutes_unhealthy}m<${UNHEALTHY_THRESHOLD_MIN}m)" return fi if [[ -n "$last_action_at" ]]; then local last_e minutes_since_action last_e="$(date -u -d "$last_action_at" +%s)" minutes_since_action=$(( (now_e - last_e) / 60 )) if (( minutes_since_action < RATE_LIMIT_MIN )); then echo "not_eligible:rate_limited(${minutes_since_action}m<${RATE_LIMIT_MIN}m)" return fi fi echo "eligible" } declare -A RESOURCE_ELIGIBILITY # resource_key -> eligible | not_eligible:motivo declare -A RESOURCE_SUMMARY # resource_key -> línea de texto para el prompt DEPLOY_SUMMARY_LINES=() while IFS= read -r dep; do desired="$(jq -r --arg n "$dep" '.items[] | select(.metadata.name==$n) | (.spec.replicas // 0)' <<<"$DEPLOY_JSON")" ready="$(jq -r --arg n "$dep" '.items[] | select(.metadata.name==$n) | (.status.readyReplicas // 0)' <<<"$DEPLOY_JSON")" available="$(jq -r --arg n "$dep" '.items[] | select(.metadata.name==$n) | ([.status.conditions[]? | select(.type=="Available") | .status] | first // "Unknown")' <<<"$DEPLOY_JSON")" unhealthy=0 [[ "$ready" -lt "$desired" || "$available" != "True" ]] && unhealthy=1 window_ok=1 if [[ "$dep" == "$COMMERCE_BACKEND_DEPLOYMENT" ]]; then in_commerce_backend_window && window_ok=1 || window_ok=0 fi key="deploy:${dep}" elig="$(compute_eligibility "$key" "$unhealthy" "$window_ok")" RESOURCE_ELIGIBILITY["$key"]="$elig" line="deployment/${dep} (ns=${NAMESPACE}): ready=${ready}/${desired}, Available=${available}, unhealthy=${unhealthy}, elegibilidad=${elig}" RESOURCE_SUMMARY["$key"]="$line" DEPLOY_SUMMARY_LINES+=("$line") log "$line" done < <(printf '%s\n' "${DETECTED_DEPLOYMENTS[@]}") # --------------------------------------------------------------------------- # Diagnóstico: contenedor gitea-runner (docker) # --------------------------------------------------------------------------- check_gitea_runner_stall() { # Señal vía API de Gitea Actions: runs con status=running que llevan # más de UNHEALTHY_THRESHOLD_MIN minutos sin terminar. Fail-safe: si el # token no existe o la API falla/tiene forma inesperada, se omite esta # señal (nunca dispara una acción sobre una lectura no verificada). if [[ ! -r "$GITEA_TOKEN_FILE" ]]; then echo "omitida:sin_token_en_${GITEA_TOKEN_FILE}" return fi local token repo resp http_code body stuck_total=0 detail="" token="$(<"$GITEA_TOKEN_FILE")" for repo in $GITEA_ACTIONS_REPOS; do resp="$(curl -s -w '\n%{http_code}' --max-time 10 \ -H "Authorization: token ${token}" \ "${GITEA_API_BASE}/repos/${repo}/actions/runs?limit=10" 2>/dev/null)" || resp=$'\n000' http_code="$(tail -n1 <<<"$resp")" body="$(sed '$d' <<<"$resp")" if [[ "$http_code" != "200" ]]; then log "WARN: API de Gitea Actions para ${repo} devolvió HTTP ${http_code}; se omite señal de stall para ese repo" continue fi local stuck stuck="$(jq -r --argjson th "$UNHEALTHY_THRESHOLD_MIN" ' [ (.workflow_runs // .runs // [])[] | select(.status=="running") | select( ( (now - ((.updated_at // .run_started_at // .created_at) | fromdateiso8601)) / 60 ) > $th ) ] | length' <<<"$body" 2>/dev/null)" || stuck="" if [[ -z "$stuck" ]]; then log "WARN: no se pudo interpretar la respuesta de Gitea Actions para ${repo}; se omite señal de stall" continue fi if (( stuck > 0 )); then stuck_total=$(( stuck_total + stuck )) detail="${detail}${repo}:${stuck} run(s) atascado(s) >${UNHEALTHY_THRESHOLD_MIN}m; " fi done token="" if (( stuck_total > 0 )); then echo "stalled:${detail}" else echo "ok" fi } DOCKER_UNHEALTHY=0 DOCKER_DETAIL="docker:gitea-runner no está en la lista blanca (DOCKER_RESTART_ALLOWLIST)" if [[ " $DOCKER_RESTART_ALLOWLIST " == *" gitea-runner "* ]]; then runner_status="$(docker inspect gitea-runner --format '{{.State.Status}}' 2>/dev/null || echo "no_encontrado")" stall_signal="$(check_gitea_runner_stall)" crash_signal=0 [[ "$runner_status" != "running" ]] && crash_signal=1 DOCKER_UNHEALTHY=0 reason_bits=() if [[ "$crash_signal" == "1" ]]; then DOCKER_UNHEALTHY=1 reason_bits+=("estado_contenedor=${runner_status}") fi if [[ "$stall_signal" == stalled:* ]]; then DOCKER_UNHEALTHY=1 reason_bits+=("$stall_signal") fi DOCKER_DETAIL="docker/gitea-runner: estado=${runner_status}, señal_gitea_actions=${stall_signal}, unhealthy=${DOCKER_UNHEALTHY}" [[ "${#reason_bits[@]}" -gt 0 ]] && DOCKER_DETAIL="${DOCKER_DETAIL}, motivo=[${reason_bits[*]}]" fi log "$DOCKER_DETAIL" GITEA_KEY="docker:gitea-runner" GITEA_ELIG="not_eligible:no_esta_en_lista_blanca" if [[ " $DOCKER_RESTART_ALLOWLIST " == *" gitea-runner "* ]]; then GITEA_ELIG="$(compute_eligibility "$GITEA_KEY" "$DOCKER_UNHEALTHY" "1")" fi RESOURCE_ELIGIBILITY["$GITEA_KEY"]="$GITEA_ELIG" RESOURCE_SUMMARY["$GITEA_KEY"]="$DOCKER_DETAIL, elegibilidad=${GITEA_ELIG}" log "elegibilidad ${GITEA_KEY}: ${GITEA_ELIG}" # --------------------------------------------------------------------------- # Diagnóstico: contexto adicional (no gatilla acciones, solo informativo) # --------------------------------------------------------------------------- CLUSTER_WIDE_SNAPSHOT="$(kubectl get pods -A -o custom-columns='NS:.metadata.namespace,POD:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,READY:.status.containerStatuses[0].ready' 2>/dev/null || echo "no disponible")" PUBLIC_CHECKS="" for d in $PUBLIC_DOMAINS; do code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "https://${d}/" 2>/dev/null || echo "000")" PUBLIC_CHECKS="${PUBLIC_CHECKS}${d} -> HTTP ${code}\n" log "curl https://${d}/ -> HTTP ${code}" done # --------------------------------------------------------------------------- # Construir prompt para claude -p (Claude NO recibe herramientas — ver # cabecera del archivo) # --------------------------------------------------------------------------- ELIGIBLE_LIST="" NOT_ELIGIBLE_LIST="" for key in "${!RESOURCE_ELIGIBILITY[@]}"; do val="${RESOURCE_ELIGIBILITY[$key]}" if [[ "$val" == "eligible" ]]; then ELIGIBLE_LIST="${ELIGIBLE_LIST}- ${key}\n" else NOT_ELIGIBLE_LIST="${NOT_ELIGIBLE_LIST}- ${key}: ${val}\n" fi done [[ -z "$ELIGIBLE_LIST" ]] && ELIGIBLE_LIST="(ninguno este ciclo)\n" JSON_SCHEMA='{ "type": "object", "properties": { "actions": { "type": "array", "items": { "type": "object", "properties": { "resource": {"type": "string"}, "decision": {"type": "string", "enum": ["restart", "notify_only"]}, "reason": {"type": "string"} }, "required": ["resource", "decision", "reason"] } }, "summary": {"type": "string"} }, "required": ["actions", "summary"] }' PROMPT="$(cat <>"$RUN_LOG" )" || { log "FATAL: la invocación a claude -p falló" notify_send "lab-agent: error" "claude -p terminó con error, ver ${RUN_LOG}" exit 1 } log "--- respuesta cruda de claude -p ---" log "$CLAUDE_JSON" IS_ERROR="$(jq -r '.is_error // false' <<<"$CLAUDE_JSON")" PERMISSION_DENIALS="$(jq -c '.permission_denials // []' <<<"$CLAUDE_JSON")" if [[ "$PERMISSION_DENIALS" != "[]" ]]; then log "ALERTA: claude intentó usar herramientas pese a --allowedTools \"\": ${PERMISSION_DENIALS}" notify_send "lab-agent: ALERTA de seguridad" "El modelo intentó invocar herramientas denegadas: ${PERMISSION_DENIALS}" fi if [[ "$IS_ERROR" == "true" ]]; then log "FATAL: claude -p devolvió is_error=true" notify_send "lab-agent: error" "claude -p devolvió un error, ver ${RUN_LOG}" exit 1 fi DECISION_JSON="$(jq -r '.structured_output // empty' <<<"$CLAUDE_JSON")" if [[ -z "$DECISION_JSON" ]]; then log "FATAL: no se recibió structured_output válido" notify_send "lab-agent: error" "Respuesta sin structured_output, ver ${RUN_LOG}" exit 1 fi CLAUDE_SUMMARY="$(jq -r '.summary // ""' <<<"$DECISION_JSON")" log "resumen de claude: ${CLAUDE_SUMMARY}" # --------------------------------------------------------------------------- # Ejecutar (o simular, en DRY_RUN) solo lo que el wrapper valida como # elegible — nunca confiar en la palabra de Claude por sí sola. # --------------------------------------------------------------------------- run_and_capture() { # cmd... -> setea LAST_ACTION_STDOUT/STDERR, devuelve el exit code real del comando local errfile rc errfile="$(mktemp)" if LAST_ACTION_STDOUT="$("$@" 2>"$errfile")"; then rc=0 else rc=$? fi LAST_ACTION_STDERR="$(cat "$errfile" 2>/dev/null || true)" rm -f "$errfile" return "$rc" } execute_corrective_action() { # resource -> imprime "ok"|"fail" y deja stdout/stderr en variables globales local resource="$1" LAST_ACTION_STDOUT="" LAST_ACTION_STDERR="" LAST_ACTION_CMD="" case "$resource" in docker:gitea-runner) LAST_ACTION_CMD="docker restart gitea-runner" if [[ "$DRY_RUN" == "1" ]]; then LAST_ACTION_STDOUT="[DRY_RUN] no se ejecutó de verdad" echo "ok"; return fi if run_and_capture docker restart gitea-runner; then echo "ok"; else echo "fail"; fi ;; deploy:*) local dep="${resource#deploy:}" local found=0 for d in "${DETECTED_DEPLOYMENTS[@]}"; do [[ "$d" == "$dep" ]] && found=1; done if [[ "$found" != "1" ]]; then LAST_ACTION_STDERR="deployment '${dep}' no está entre los detectados este ciclo — rechazado" echo "fail"; return fi LAST_ACTION_CMD="kubectl -n ${NAMESPACE} rollout restart deployment/${dep}" if [[ "$DRY_RUN" == "1" ]]; then LAST_ACTION_STDOUT="[DRY_RUN] no se ejecutó de verdad" echo "ok"; return fi if run_and_capture kubectl -n "$NAMESPACE" rollout restart "deployment/${dep}"; then echo "ok"; else echo "fail"; fi ;; *) LAST_ACTION_STDERR="recurso desconocido: ${resource}" echo "fail" ;; esac } NOTIFY_LINES=() ACTIONS_TAKEN=0 while IFS= read -r action_json; do [[ -z "$action_json" ]] && continue resource="$(jq -r '.resource' <<<"$action_json")" decision="$(jq -r '.decision' <<<"$action_json")" claude_reason="$(jq -r '.reason' <<<"$action_json")" eligibility="${RESOURCE_ELIGIBILITY[$resource]:-not_eligible:recurso_no_reconocido_por_el_wrapper}" if [[ "$decision" == "restart" && "$eligibility" != "eligible" ]]; then log "ANOMALÍA: claude propuso restart en '${resource}' pero el wrapper lo marca '${eligibility}' — RECHAZADO, no se ejecuta" log_action "$resource" "restart_rejected" "rejected_not_eligible" "claude_reason=${claude_reason} | wrapper_eligibility=${eligibility}" NOTIFY_LINES+=("[RECHAZADO] ${resource}: claude propuso restart pero no es elegible (${eligibility}). Motivo de claude: ${claude_reason}") continue fi if [[ "$decision" == "restart" ]]; then result="$(execute_corrective_action "$resource")" ACTIONS_TAKEN=1 # Tras cualquier intento (éxito o fallo), marcar blocked=true: un solo # intento automático por incidente, sin reintentos silenciosos. mark_action_taken "$resource" if [[ "$result" == "ok" ]]; then log "ACCIÓN EJECUTADA: ${LAST_ACTION_CMD} -> OK" log_action "$resource" "restart" "success" "$claude_reason" NOTIFY_LINES+=("[ACCIÓN OK] ${resource}: ${LAST_ACTION_CMD}. Motivo: ${claude_reason}") else log "ACCIÓN FALLIDA: ${LAST_ACTION_CMD:-N/A} -> ${LAST_ACTION_STDERR}" log_action "$resource" "restart" "failure" "$claude_reason | stderr=${LAST_ACTION_STDERR}" NOTIFY_LINES+=("[ACCIÓN FALLÓ] ${resource}: ${LAST_ACTION_STDERR}. Queda bloqueado hasta que un humano lo revise (no se reintentará solo).") fi else log "notify_only: ${resource} — ${claude_reason}" log_action "$resource" "notify_only" "n/a" "$claude_reason" if [[ "$eligibility" != "not_eligible:healthy" ]]; then NOTIFY_LINES+=("[SOLO DIAGNÓSTICO] ${resource} (${eligibility}): ${claude_reason}") fi fi done < <(jq -c '.actions[]?' <<<"$DECISION_JSON") # --------------------------------------------------------------------------- # Notificación final # --------------------------------------------------------------------------- if [[ "${#NOTIFY_LINES[@]}" -gt 0 || "$ACTIONS_TAKEN" == "1" ]]; then MSG="Resumen: ${CLAUDE_SUMMARY}"$'\n\n' for l in "${NOTIFY_LINES[@]}"; do MSG="${MSG}${l}"$'\n'; done MSG="${MSG}"$'\n'"log completo: ${RUN_LOG}" title="lab-agent" [[ "$ACTIONS_TAKEN" == "1" ]] && title="lab-agent: acción correctiva ejecutada" [[ "$DRY_RUN" == "1" ]] && title="[DRY_RUN] ${title}" notify_send "$title" "$MSG" else log "Sin anomalías ni acciones este ciclo — no se notifica." fi log "=== lab-agent fin ==="