#!/usr/bin/env bash
# Laboratorio estable v4.4.3: ZimaOS + Docker + k3d + Argo CD + Multi-Repo + Governance + Monitoring Recovery + Runner
#
# Modos:
#   bootstrap          Crea el clúster si no existe; si existe, lo inicia y reconcilia.
#   recover            Recupera únicamente un clúster existente.
#   ensure             Health check no destructivo para systemd; no instala Argo CD.
#   status             Muestra diagnóstico del laboratorio.
#   repo-sync          Clona/actualiza apps-registry y platform-infra.
#   gitops             Actualiza el repo y aplica root-apps-registry.
#   monitoring-diagnose Diagnostica kube-prometheus-stack/Grafana.
#   monitoring-recover  Finaliza una operación obsoleta y resincroniza Grafana.
#   runner             Adopta, crea o recupera el Gitea Actions runner.
#   install-autostart  Instala servicio/timer systemd, orden de montajes y autorreparación.
#   reset              Reconstrucción limpia; requiere confirmación explícita.
#
# Ejemplo:
#   LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh bootstrap

set -Eeuo pipefail
IFS=$'\n\t'

MODE="${1:-bootstrap}"

if [[ -z "${BASH_VERSION:-}" ]]; then
  printf 'ERROR: este script requiere Bash. Ejecútalo con: bash deploy-lab.sh %s\n' "$MODE" >&2
  exit 1
fi

if [[ $(id -u) -eq 0 && "$MODE" != "install-autostart" ]]; then
  printf 'ERROR: ejecuta el modo %s como usuario normal, sin sudo.\n' "$MODE" >&2
  printf 'Ejemplo: LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh %s\n' "$MODE" >&2
  exit 1
fi

CLUSTER_NAME="${CLUSTER_NAME:-lab-cluster}"
APP_DATA_PATH="${APP_DATA_PATH:-/DATA/AppData/k3d-lab}"
KUBECONFIG_PATH="${KUBECONFIG_PATH:-/DATA/.kube/config}"
CONFIG_FILE="${CONFIG_FILE:-${APP_DATA_PATH}/config/k3d-lab.yaml}"
BOOTSTRAP_DIR="${BOOTSTRAP_DIR:-${APP_DATA_PATH}/bootstrap}"
BACKUP_DIR="${BACKUP_DIR:-${APP_DATA_PATH}/backups}"

API_PORT="${API_PORT:-46421}"
HTTP_PORT="${HTTP_PORT:-90}"
HTTPS_PORT="${HTTPS_PORT:-9443}"
AGENT_COUNT="${AGENT_COUNT:-2}"
K3D_SUBNET="${K3D_SUBNET:-auto}"
K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.35.5-k3s1}"
ARGOCD_VERSION="${ARGOCD_VERSION:-v3.2.0}"
ARGOCD_HOST="${ARGOCD_HOST:-argocd.cruzcloud.net}"
EXPECTED_HOST_IP="${EXPECTED_HOST_IP:-192.168.68.61}"
ALLOW_IP_CHANGE="${ALLOW_IP_CHANGE:-false}"
INSTALL_HEADLAMP_RBAC="${INSTALL_HEADLAMP_RBAC:-true}"
APPLY_POSTDEPLOY_RBAC="${APPLY_POSTDEPLOY_RBAC:-true}"
WAIT_SECONDS="${WAIT_SECONDS:-240}"
ARGOCD_ROLLOUT_TIMEOUT="${ARGOCD_ROLLOUT_TIMEOUT:-420}"
AGENT_RECOVERY_WAIT="${AGENT_RECOVERY_WAIT:-75}"
AUTO_REREGISTER_STALE_AGENTS="${AUTO_REREGISTER_STALE_AGENTS:-true}"
CHECK_INGRESS_AFTER_BOOT="${CHECK_INGRESS_AFTER_BOOT:-true}"

# Bootstrap GitOps / App-of-Apps.
BOOTSTRAP_ROOT_APP="${BOOTSTRAP_ROOT_APP:-true}"
ROOT_APP_NAME="${ROOT_APP_NAME:-root-apps-registry}"
ROOT_APP_REPO_URL="${ROOT_APP_REPO_URL:-https://gitea.cruzcloud.net/devops/apps-registry.git}"
ROOT_APP_REPO_PATH="${ROOT_APP_REPO_PATH:-apps}"
ROOT_APP_REVISION="${ROOT_APP_REVISION:-main}"
ROOT_APP_PROJECT="${ROOT_APP_PROJECT:-default}"
ROOT_APP_MANIFEST="${ROOT_APP_MANIFEST:-${BOOTSTRAP_DIR}/00-root-apps-registry.yaml}"
ROOT_APP_WAIT_SECONDS="${ROOT_APP_WAIT_SECONDS:-600}"
GITOPS_APPS_WAIT_SECONDS="${GITOPS_APPS_WAIT_SECONDS:-900}"
GITOPS_STRICT_HEALTH="${GITOPS_STRICT_HEALTH:-false}"
EXPECTED_GITOPS_APPS="${EXPECTED_GITOPS_APPS:-}"
ARGOCD_REPO_SECRET_NAME="${ARGOCD_REPO_SECRET_NAME:-apps-registry-repository}"
ARGOCD_REPO_USERNAME_FILE="${ARGOCD_REPO_USERNAME_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-username}"
ARGOCD_REPO_PASSWORD_FILE="${ARGOCD_REPO_PASSWORD_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-password}"

# Copia local persistente del repositorio GitOps.
APP_REGISTRY_CACHE_DIR="${APP_REGISTRY_CACHE_DIR:-${APP_DATA_PATH}/git/apps-registry}"
APP_REGISTRY_ROOT_MANIFEST="${APP_REGISTRY_ROOT_MANIFEST:-application.yaml}"
APP_REGISTRY_COMMIT_FILE="${APP_REGISTRY_COMMIT_FILE:-${APP_DATA_PATH}/git/apps-registry.commit}"
APP_REGISTRY_ALLOW_CACHE_FALLBACK="${APP_REGISTRY_ALLOW_CACHE_FALLBACK:-true}"
APP_REGISTRY_REFRESH_ON_ENSURE="${APP_REGISTRY_REFRESH_ON_ENSURE:-false}"
APP_REGISTRY_CLONE_DEPTH="${APP_REGISTRY_CLONE_DEPTH:-1}"

# Repositorio de governance/Kustomize.
PLATFORM_INFRA_REPO_URL="${PLATFORM_INFRA_REPO_URL:-https://gitea.cruzcloud.net/devops/platform-infra.git}"
PLATFORM_INFRA_REVISION="${PLATFORM_INFRA_REVISION:-main}"
PLATFORM_INFRA_CACHE_DIR="${PLATFORM_INFRA_CACHE_DIR:-${APP_DATA_PATH}/git/platform-infra}"
PLATFORM_INFRA_COMMIT_FILE="${PLATFORM_INFRA_COMMIT_FILE:-${APP_DATA_PATH}/git/platform-infra.commit}"
PLATFORM_INFRA_ALLOW_CACHE_FALLBACK="${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK:-true}"
PLATFORM_INFRA_REFRESH_ON_ENSURE="${PLATFORM_INFRA_REFRESH_ON_ENSURE:-false}"
PLATFORM_INFRA_CLONE_DEPTH="${PLATFORM_INFRA_CLONE_DEPTH:-1}"
PLATFORM_INFRA_RENDER_DIR="${PLATFORM_INFRA_RENDER_DIR:-${APP_DATA_PATH}/rendered/platform-infra}"
APPLY_GOVERNANCE_SNAPSHOTS="${APPLY_GOVERNANCE_SNAPSHOTS:-true}"
ENFORCE_MONITORING_LIMITRANGE="${ENFORCE_MONITORING_LIMITRANGE:-true}"
MONITORING_GOVERNANCE_PATH="${MONITORING_GOVERNANCE_PATH:-monitoring-governance}"
ARGOCD_PLATFORM_REPO_SECRET_NAME="${ARGOCD_PLATFORM_REPO_SECRET_NAME:-platform-infra-repository}"

# Diagnóstico de monitoring/Grafana.
MONITORING_APP_NAME="${MONITORING_APP_NAME:-monitoring-app}"
MONITORING_NAMESPACE="${MONITORING_NAMESPACE:-monitoring}"
MONITORING_FIXED_CHART_VERSION="${MONITORING_FIXED_CHART_VERSION:-58.3.3}"
MONITORING_INGRESS_HOST="${MONITORING_INGRESS_HOST:-grafana.cruzcloud.net}"
MONITORING_RECOVERY_WAIT_SECONDS="${MONITORING_RECOVERY_WAIT_SECONDS:-900}"
MONITORING_OPERATION_STOP_WAIT="${MONITORING_OPERATION_STOP_WAIT:-180}"
AUTO_RECOVER_STALE_MONITORING_OPERATION="${AUTO_RECOVER_STALE_MONITORING_OPERATION:-true}"
MONITORING_FORCE_RECOVERY="${MONITORING_FORCE_RECOVERY:-false}"
ARGOCD_CLI_IMAGE="${ARGOCD_CLI_IMAGE:-quay.io/argoproj/argocd:${ARGOCD_VERSION}}"

# Gitea Actions runner externo al clúster.
INSTALL_GITEA_RUNNER="${INSTALL_GITEA_RUNNER:-true}"
GITEA_RUNNER_REQUIRED="${GITEA_RUNNER_REQUIRED:-true}"
GITEA_RUNNER_CONTAINER="${GITEA_RUNNER_CONTAINER:-gitea-runner}"
GITEA_RUNNER_IMAGE="${GITEA_RUNNER_IMAGE:-docker.io/gitea/act_runner:latest}"
GITEA_INSTANCE_URL="${GITEA_INSTANCE_URL:-https://gitea.cruzcloud.net}"
GITEA_RUNNER_NAME="${GITEA_RUNNER_NAME:-zimaos-lab-runner}"
GITEA_RUNNER_LABELS="${GITEA_RUNNER_LABELS:-ubuntu-latest:docker://node:20-bookworm}"
GITEA_RUNNER_ROOT="${GITEA_RUNNER_ROOT:-${APP_DATA_PATH}/gitea-runner}"
GITEA_RUNNER_DATA_DIR="${GITEA_RUNNER_DATA_DIR:-${GITEA_RUNNER_ROOT}/data}"
GITEA_RUNNER_IMAGE_FILE="${GITEA_RUNNER_IMAGE_FILE:-${GITEA_RUNNER_ROOT}/image.txt}"
GITEA_RUNNER_TOKEN_FILE="${GITEA_RUNNER_TOKEN_FILE:-${APP_DATA_PATH}/secrets/gitea-runner-registration-token}"
MIGRATE_EXISTING_RUNNER="${MIGRATE_EXISTING_RUNNER:-false}"

K3S_SERVER_PATH="${APP_DATA_PATH}/k3s/server"
K3S_SERVER_STORAGE="${APP_DATA_PATH}/storage/server-0"
K3S_AGENT0_STORAGE="${APP_DATA_PATH}/storage/agent-0"
K3S_AGENT1_STORAGE="${APP_DATA_PATH}/storage/agent-1"
TOKEN_FILE="${APP_DATA_PATH}/secrets/k3s-cluster-token"
POSTDEPLOY_RBAC_FILE="${APP_DATA_PATH}/gitea-postdeploy-validator-rbac.yaml"

if [[ -t 1 ]]; then
  C_RESET=$'\033[0m'
  C_BLUE=$'\033[1;34m'
  C_GREEN=$'\033[1;32m'
  C_YELLOW=$'\033[1;33m'
  C_RED=$'\033[1;31m'
else
  C_RESET=''
  C_BLUE=''
  C_GREEN=''
  C_YELLOW=''
  C_RED=''
fi

log()  { printf '\n%s[%s] %s%s\n' "$C_BLUE" "$(date '+%F %T')" "$*" "$C_RESET"; }
ok()   { printf '%sOK:%s %s\n' "$C_GREEN" "$C_RESET" "$*"; }
warn() { printf '%sAVISO:%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; }
die()  { printf '%sERROR:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; exit 1; }

on_error() {
  local exit_code=$?
  printf '%sERROR:%s fallo en la línea %s: %s\n' \
    "$C_RED" "$C_RESET" "${BASH_LINENO[0]:-desconocida}" "${BASH_COMMAND:-desconocido}" >&2
  exit "$exit_code"
}
trap on_error ERR

have() { command -v "$1" >/dev/null 2>&1; }

sudo_cmd() {
  if [[ $(id -u) -eq 0 ]]; then
    "$@"
  else
    sudo "$@"
  fi
}

require_commands() {
  local command_name
  for command_name in docker k3d kubectl curl git awk sed grep findmnt ss tar base64 tr; do
    have "$command_name" || die "No se encontró el comando requerido: ${command_name}"
  done
}

detect_host_ip() {
  ip -4 route get 1.1.1.1 2>/dev/null | awk '{
    for (i = 1; i <= NF; i++) {
      if ($i == "src") {
        print $(i + 1)
        exit
      }
    }
  }'
}

LAB_HOST_IP="${LAB_HOST_IP:-$(detect_host_ip)}"
export LAB_HOST_IP CLUSTER_NAME APP_DATA_PATH K3D_SUBNET K3S_IMAGE AGENT_COUNT API_PORT HTTP_PORT HTTPS_PORT

validate_host_ip() {
  [[ -n "$LAB_HOST_IP" ]] || die "No fue posible detectar LAB_HOST_IP. Defínela manualmente."

  if [[ -n "$EXPECTED_HOST_IP" && "$LAB_HOST_IP" != "$EXPECTED_HOST_IP" ]]; then
    if [[ "$ALLOW_IP_CHANGE" == "true" ]]; then
      warn "La IP detectada ${LAB_HOST_IP} no coincide con ${EXPECTED_HOST_IP}; se continuará por ALLOW_IP_CHANGE=true."
    else
      die "La IP detectada es ${LAB_HOST_IP}, pero NPM/Gitea esperan ${EXPECTED_HOST_IP}. Reserva la IP o usa ALLOW_IP_CHANGE=true conscientemente."
    fi
  fi
}

ensure_docker() {
  if have systemctl && ! systemctl is-active --quiet docker.service; then
    if [[ $(id -u) -eq 0 || -t 0 ]]; then
      log "Iniciando Docker"
      sudo_cmd systemctl start docker.service
    else
      die "Docker no está activo. systemd debe iniciarlo antes de ejecutar el watchdog."
    fi
  fi

  docker info >/dev/null 2>&1 ||
    die "Docker no responde para el usuario $(id -un). Valida docker.service y el grupo docker."
}

validate_data_mount() {
  findmnt -T /DATA >/dev/null 2>&1 ||
    die "/DATA no está montado. No se iniciará k3d para evitar escribir en el filesystem raíz."

  [[ -d "$APP_DATA_PATH" ]] ||
    die "No existe la ruta persistente ${APP_DATA_PATH}."

  # En ZimaOS el usuario devops no puede escribir directamente en /DATA,
  # pero sí debe poder escribir en su ruta del laboratorio.
  local test_file="${APP_DATA_PATH}/.k3d-write-test.$$"

  touch "$test_file" 2>/dev/null ||
    die "${APP_DATA_PATH} no permite escritura para $(id -un)."

  rm -f "$test_file"
}

prepare_directories() {
  log "Preparando almacenamiento persistente"

  sudo_cmd install -d -m 0750 -o "$(id -un)" \
    "$APP_DATA_PATH" \
    "${APP_DATA_PATH}/config" \
    "${APP_DATA_PATH}/secrets" \
    "$BOOTSTRAP_DIR" \
    "$BACKUP_DIR" \
    "$GITEA_RUNNER_ROOT" \
    "$GITEA_RUNNER_DATA_DIR" \
    "$(dirname "$APP_REGISTRY_CACHE_DIR")" \
    "$(dirname "$PLATFORM_INFRA_CACHE_DIR")" \
    "$PLATFORM_INFRA_RENDER_DIR" \
    "$(dirname "$KUBECONFIG_PATH")"

  # K3s y los workloads escriben como root dentro de los nodos.
  sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH"
  sudo_cmd install -d -m 0755 \
    "$K3S_SERVER_STORAGE" \
    "$K3S_AGENT0_STORAGE" \
    "$K3S_AGENT1_STORAGE"

  sudo_cmd chown "$(id -un)" "$(dirname "$KUBECONFIG_PATH")"
}

generate_cluster_token() {
  local token=""

  if have openssl; then
    token="$(openssl rand -hex 32)"
  elif have od; then
    token="$(
      od -An -N32 -tx1 /dev/urandom |
        awk '{
          for (i = 1; i <= NF; i++) {
            printf "%s", $i
          }
        }
        END {
          printf "\n"
        }'
    )"
  elif have hexdump; then
    token="$(hexdump -vn32 -e '1/1 "%02x"' /dev/urandom)"
  else
    die "No hay un generador aleatorio disponible. Se requiere openssl, od o hexdump."
  fi

  [[ "$token" =~ ^[0-9a-fA-F]{64}$ ]] ||
    die "No fue posible generar un token hexadecimal seguro."

  printf '%s\n' "$token"
}

ensure_cluster_token() {
  if [[ ! -s "$TOKEN_FILE" ]]; then
    log "Generando token persistente del clúster"
    umask 077
    generate_cluster_token > "$TOKEN_FILE"
    chmod 0600 "$TOKEN_FILE"
  fi

  K3S_CLUSTER_TOKEN="$(<"$TOKEN_FILE")"

  [[ "$K3S_CLUSTER_TOKEN" =~ ^[0-9a-fA-F]{64}$ ]] ||
    die "El token persistente de K3s no es válido."

  export K3S_CLUSTER_TOKEN
}

write_k3d_config() {
  log "Generando configuración declarativa de k3d"

  cat > "$CONFIG_FILE" <<'YAML'
apiVersion: k3d.io/v1alpha5
kind: Simple

metadata:
  name: ${CLUSTER_NAME}

servers: 1
agents: ${AGENT_COUNT}
image: ${K3S_IMAGE}
subnet: "${K3D_SUBNET}"
token: "${K3S_CLUSTER_TOKEN}"

kubeAPI:
  host: "${LAB_HOST_IP}"
  hostIP: "0.0.0.0"
  hostPort: "${API_PORT}"

ports:
  - port: "${HTTP_PORT}:80"
    nodeFilters:
      - loadbalancer
  - port: "${HTTPS_PORT}:443"
    nodeFilters:
      - loadbalancer

volumes:
  # Conserva SQLite, certificados y token de K3s fuera de los contenedores.
  - volume: "${APP_DATA_PATH}/k3s/server:/var/lib/rancher/k3s/server"
    nodeFilters:
      - server:0

  # Conserva datos de PVC creados por local-path-provisioner.
  - volume: "${APP_DATA_PATH}/storage/server-0:/var/lib/rancher/k3s/storage"
    nodeFilters:
      - server:0
  - volume: "${APP_DATA_PATH}/storage/agent-0:/var/lib/rancher/k3s/storage"
    nodeFilters:
      - agent:0
  - volume: "${APP_DATA_PATH}/storage/agent-1:/var/lib/rancher/k3s/storage"
    nodeFilters:
      - agent:1

options:
  k3d:
    wait: true
    timeout: "240s"
    disableRollback: false
  k3s:
    extraArgs:
      - arg: "--tls-san=${LAB_HOST_IP}"
        nodeFilters:
          - server:*
  kubeconfig:
    updateDefaultKubeconfig: false
    switchCurrentContext: false
YAML

  chmod 0600 "$CONFIG_FILE"
}

cluster_exists() {
  k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | grep -Fxq "$CLUSTER_NAME"
}

cluster_containers_exist() {
  docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" --format '{{.ID}}' | grep -q .
}

ports_in_use_by_non_cluster_process() {
  local port
  for port in "$API_PORT" "$HTTP_PORT" "$HTTPS_PORT"; do
    if ss -lntH "sport = :${port}" 2>/dev/null | grep -q .; then
      return 0
    fi
  done
  return 1
}

create_cluster() {
  if cluster_containers_exist; then
    die "Existen contenedores etiquetados para ${CLUSTER_NAME}, pero k3d no reconoce el clúster. No se crearán duplicados."
  fi

  if ports_in_use_by_non_cluster_process; then
    sudo_cmd ss -lntp | grep -E ":(${API_PORT}|${HTTP_PORT}|${HTTPS_PORT})\\b" || true
    die "Uno o más puertos del laboratorio están ocupados."
  fi

  log "Creando clúster ${CLUSTER_NAME} con IPAM estático"
  docker pull "$K3S_IMAGE"
  k3d cluster create --config "$CONFIG_FILE"
}

start_cluster() {
  log "Iniciando clúster existente ${CLUSTER_NAME}"
  k3d cluster start "$CLUSTER_NAME" --wait --timeout "${WAIT_SECONDS}s" || true
}

set_restart_policy() {
  mapfile -t cluster_containers < <(
    docker ps -aq --filter "label=k3d.cluster=${CLUSTER_NAME}"
  )

  if (( ${#cluster_containers[@]} > 0 )); then
    docker update --restart=unless-stopped "${cluster_containers[@]}" >/dev/null
    ok "Política Docker unless-stopped aplicada a ${#cluster_containers[@]} contenedores del clúster."
  fi
}

write_kubeconfig() {
  local temp_file context_name configured_server
  temp_file="$(mktemp)"

  k3d kubeconfig get "$CLUSTER_NAME" > "$temp_file"
  [[ -s "$temp_file" ]] || die "k3d no generó el kubeconfig."

  context_name="$(
    KUBECONFIG="$temp_file" \
      kubectl config get-contexts -o name 2>/dev/null |
      head -1
  )"

  [[ -n "$context_name" ]] ||
    die "El kubeconfig generado no contiene ningún contexto."

  # Algunas combinaciones de k3d/kubectl pueden producir un archivo válido
  # sin current-context. Lo fijamos explícitamente antes de instalarlo.
  KUBECONFIG="$temp_file" \
    kubectl config use-context "$context_name" >/dev/null

  configured_server="$(
    KUBECONFIG="$temp_file" \
      kubectl config view \
      --minify \
      -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null
  )"

  [[ -n "$configured_server" ]] ||
    die "El kubeconfig generado no contiene un API Server válido."

  # El watchdog se ejecuta sin TTY y no debe invocar sudo.
  # install-autostart garantiza previamente el propietario de esta ruta.
  install -d \
    -m 0700 \
    "$(dirname "$KUBECONFIG_PATH")" ||
    die "No se pudo preparar $(dirname "$KUBECONFIG_PATH") como $(id -un)."

  install \
    -m 0600 \
    "$temp_file" \
    "$KUBECONFIG_PATH" ||
    die "No se pudo actualizar ${KUBECONFIG_PATH} como $(id -un)."

  rm -f "$temp_file"
  export KUBECONFIG="$KUBECONFIG_PATH"

  [[ "$(kubectl config current-context 2>/dev/null || true)" == "$context_name" ]] ||
    die "No fue posible activar el contexto ${context_name}."

  ok "Kubeconfig actualizado: ${KUBECONFIG_PATH}"
  ok "Contexto activo: ${context_name}"
  ok "API Server: ${configured_server}"
}

wait_for_api() {
  log "Esperando API de Kubernetes"
  local deadline=$((SECONDS + WAIT_SECONDS))

  while (( SECONDS < deadline )); do
    if kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1; then
      ok "API de Kubernetes disponible."
      return 0
    fi
    sleep 5
  done

  docker logs --tail 200 "k3d-${CLUSTER_NAME}-server-0" || true
  die "La API no quedó disponible dentro de ${WAIT_SECONDS}s."
}

node_ready() {
  local node_name="$1"
  [[ "$(kubectl get node "$node_name" \
    -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' \
    2>/dev/null || true)" == "True" ]]
}

wait_agent_ready() {
  local node_name="$1"
  local timeout_seconds="$2"
  local deadline=$((SECONDS + timeout_seconds))

  while (( SECONDS < deadline )); do
    if node_ready "$node_name"; then
      return 0
    fi
    sleep 5
  done

  return 1
}

recover_not_ready_agents() {
  local index node_name container_name docker_ip kubernetes_ip

  # El servidor y el API deben estar sanos antes de reparar agentes.
  kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1 ||
    die "No se repararán agentes porque el API Server no está disponible."

  for (( index = 0; index < AGENT_COUNT; index++ )); do
    node_name="k3d-${CLUSTER_NAME}-agent-${index}"
    container_name="$node_name"

    if node_ready "$node_name"; then
      continue
    fi

    warn "${node_name} no existe o no está Ready; reiniciando el contenedor."
    docker restart "$container_name" >/dev/null || true

    if wait_agent_ready "$node_name" "$AGENT_RECOVERY_WAIT"; then
      ok "${node_name} se recuperó después del reinicio."
      continue
    fi

    docker_ip="$(
      docker inspect \
        -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \
        "$container_name" 2>/dev/null || true
    )"

    kubernetes_ip="$(
      kubectl get node "$node_name" \
        -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' \
        2>/dev/null || true
    )"

    warn "${node_name} continúa sin estar Ready. DockerIP=${docker_ip:-ausente}, KubernetesIP=${kubernetes_ip:-ausente}."

    if [[ "$AUTO_REREGISTER_STALE_AGENTS" != "true" ]]; then
      continue
    fi

    # En un lab, si el agente lleva el periodo de gracia completo sin responder,
    # se elimina únicamente su objeto Node y password secret para permitir
    # un registro limpio con la IP estática actual.
    warn "Forzando re-registro seguro de ${node_name}."
    kubectl delete node "$node_name" --ignore-not-found=true >/dev/null
    kubectl delete secret \
      -n kube-system \
      "${node_name}.node-password.k3s" \
      --ignore-not-found=true >/dev/null

    docker restart "$container_name" >/dev/null

    if wait_agent_ready "$node_name" "$((AGENT_RECOVERY_WAIT * 2))"; then
      ok "${node_name} volvió a registrarse y está Ready."
    else
      docker logs --tail 180 "$container_name" || true
      die "${node_name} no se recuperó después del re-registro."
    fi
  done

  if ! docker ps --format '{{.Names}}' |
    grep -Fxq "k3d-${CLUSTER_NAME}-serverlb"; then
    warn "El load balancer no está activo; intentando iniciarlo."
    docker start "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true
  fi
}

nodes_are_ready() {
  local expected_nodes=$((AGENT_COUNT + 1))
  local actual_nodes ready_nodes

  actual_nodes="$(kubectl get nodes --no-headers 2>/dev/null | awk 'NF {count++} END {print count+0}')"
  ready_nodes="$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep -c '^True$' || true)"

  [[ "$actual_nodes" -eq "$expected_nodes" && "$ready_nodes" -eq "$expected_nodes" ]]
}

wait_for_nodes_once() {
  local deadline=$((SECONDS + WAIT_SECONDS))

  while (( SECONDS < deadline )); do
    if nodes_are_ready; then
      return 0
    fi
    sleep 5
  done

  return 1
}

wait_for_nodes() {
  log "Esperando $((AGENT_COUNT + 1)) nodos registrados y Ready"

  if wait_for_nodes_once; then
    kubectl get nodes -o wide
    return 0
  fi

  recover_not_ready_agents

  if wait_for_nodes_once; then
    kubectl get nodes -o wide
    return 0
  fi

  kubectl get nodes -o wide || true
  docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \
    --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' || true
  die "No se registraron y quedaron Ready los $((AGENT_COUNT + 1)) nodos esperados."
}


validate_runtime_after_boot() {
  [[ "$CHECK_INGRESS_AFTER_BOOT" == "true" ]] || return 0

  log "Validando Traefik, Argo CD y el puerto publicado"

  if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then
    if ! kubectl rollout status deployment/traefik \
      -n kube-system \
      --timeout="${AGENT_RECOVERY_WAIT}s"; then
      warn "Traefik no quedó disponible; reiniciando el Deployment."
      kubectl rollout restart deployment/traefik -n kube-system
      kubectl rollout status deployment/traefik \
        -n kube-system \
        --timeout="${WAIT_SECONDS}s"
    fi
  fi

  if kubectl get deployment/argocd-server -n argocd >/dev/null 2>&1; then
    if ! kubectl rollout status deployment/argocd-server \
      -n argocd \
      --timeout="${AGENT_RECOVERY_WAIT}s"; then
      warn "argocd-server no quedó disponible; reiniciando el Deployment."
      kubectl rollout restart deployment/argocd-server -n argocd
      kubectl rollout status deployment/argocd-server \
        -n argocd \
        --timeout="${WAIT_SECONDS}s"
    fi
  fi

  if ! curl -sS \
    --max-time 8 \
    -o /dev/null \
    -H "Host: ${ARGOCD_HOST}" \
    "http://127.0.0.1:${HTTP_PORT}/"; then

    warn "El Ingress no respondió en 127.0.0.1:${HTTP_PORT}; reiniciando serverlb y Traefik."
    docker restart "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true

    if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then
      kubectl rollout restart deployment/traefik -n kube-system
      kubectl rollout status deployment/traefik \
        -n kube-system \
        --timeout="${WAIT_SECONDS}s"
    fi

    sleep 10
    curl -sS \
      --max-time 8 \
      -o /dev/null \
      -H "Host: ${ARGOCD_HOST}" \
      "http://127.0.0.1:${HTTP_PORT}/" ||
      die "El Ingress sigue sin responder en el puerto ${HTTP_PORT}."
  fi

  ok "Runtime, Traefik e Ingress disponibles."
}

argocd_rollout_diagnostics() {
  local resource="$1"
  local namespace="$2"

  warn "Diagnóstico del rollout ${resource} en ${namespace}"
  kubectl get "$resource" -n "$namespace" -o wide || true
  kubectl get pods -n "$namespace" -o wide || true
  kubectl get events -n "$namespace" \
    --sort-by='.lastTimestamp' 2>/dev/null |
    tail -60 || true
}

cleanup_terminating_argocd_server_pods() {
  local desired available terminating_pods pod

  desired="$(
    kubectl get deployment/argocd-server \
      -n argocd \
      -o jsonpath='{.spec.replicas}' 2>/dev/null || true
  )"

  available="$(
    kubectl get deployment/argocd-server \
      -n argocd \
      -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true
  )"

  desired="${desired:-1}"
  available="${available:-0}"

  # Solo se fuerza la eliminación cuando la nueva réplica ya está disponible.
  if (( available < desired )); then
    return 1
  fi

  terminating_pods="$(
    kubectl get pods \
      -n argocd \
      -l app.kubernetes.io/name=argocd-server \
      -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' \
      2>/dev/null || true
  )"

  [[ -n "$terminating_pods" ]] || return 1

  while IFS= read -r pod; do
    [[ -n "$pod" ]] || continue
    warn "Eliminando Pod antiguo de argocd-server atascado en Terminating: ${pod}"
    kubectl delete pod "$pod" \
      -n argocd \
      --grace-period=0 \
      --force || true
  done <<< "$terminating_pods"

  return 0
}

wait_argocd_rollout() {
  local kind="$1"
  local name="$2"
  local namespace="${3:-argocd}"
  local resource="${kind}/${name}"

  if kubectl rollout status "$resource" \
    -n "$namespace" \
    --timeout="${ARGOCD_ROLLOUT_TIMEOUT}s"; then
    return 0
  fi

  argocd_rollout_diagnostics "$resource" "$namespace"

  if [[ "$resource" == "deployment/argocd-server" ]] &&
    cleanup_terminating_argocd_server_pods; then

    log "Reintentando rollout de argocd-server después de limpiar la réplica antigua"

    kubectl rollout status "$resource" \
      -n "$namespace" \
      --timeout="180s"
    return
  fi

  die "El rollout ${resource} no quedó disponible."
}

install_argocd() {
  log "Instalando/reconciliando Argo CD ${ARGOCD_VERSION}"

  kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f -

  kubectl apply \
    --server-side \
    --force-conflicts \
    -n argocd \
    -f "https://raw.githubusercontent.com/argoproj/argo-cd/${ARGOCD_VERSION}/manifests/install.yaml"

  local current_insecure
  current_insecure="$(
    kubectl get configmap argocd-cmd-params-cm \
      -n argocd \
      -o jsonpath='{.data.server\.insecure}' \
      2>/dev/null || true
  )"

  if [[ "$current_insecure" != "true" ]]; then
    kubectl patch configmap argocd-cmd-params-cm \
      -n argocd \
      --type merge \
      -p '{"data":{"server.insecure":"true"}}'

    # Reiniciar únicamente cuando la configuración cambió.
    kubectl rollout restart deployment/argocd-server -n argocd
  else
    ok "argocd-server ya está configurado con server.insecure=true; no se reinicia innecesariamente."
  fi

  local deployments=(
    argocd-server
    argocd-repo-server
    argocd-redis
    argocd-dex-server
    argocd-applicationset-controller
    argocd-notifications-controller
  )
  local deployment

  for deployment in "${deployments[@]}"; do
    if kubectl get deployment "$deployment" -n argocd >/dev/null 2>&1; then
      wait_argocd_rollout deployment "$deployment" argocd
    fi
  done

  if kubectl get statefulset argocd-application-controller -n argocd >/dev/null 2>&1; then
    wait_argocd_rollout statefulset argocd-application-controller argocd
  fi
}

apply_argocd_ingress() {
  log "Aplicando Ingress de Argo CD"

  cat <<YAML | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: argocd-server-ingress
  namespace: argocd
  annotations:
    traefik.ingress.kubernetes.io/router.entrypoints: web
spec:
  ingressClassName: traefik
  rules:
    - host: ${ARGOCD_HOST}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: argocd-server
                port:
                  number: 80
YAML
}

install_headlamp_rbac() {
  [[ "$INSTALL_HEADLAMP_RBAC" == "true" ]] || return 0

  log "Creando RBAC de Headlamp para laboratorio"

  kubectl create namespace headlamp --dry-run=client -o yaml | kubectl apply -f -
  kubectl create serviceaccount headlamp-admin -n headlamp --dry-run=client -o yaml | kubectl apply -f -

  cat <<'YAML' | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: headlamp-admin-binding
subjects:
  - kind: ServiceAccount
    name: headlamp-admin
    namespace: headlamp
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
YAML
}

apply_postdeploy_rbac() {
  [[ "$APPLY_POSTDEPLOY_RBAC" == "true" ]] || return 0

  log "Aplicando RBAC de solo lectura para validación postdeploy"

  kubectl create namespace ecommerce --dry-run=client -o yaml | kubectl apply -f -

  cat <<'YAML' | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitea-postdeploy-validator
  namespace: ecommerce
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: gitea-postdeploy-validator
  namespace: ecommerce
rules:
  - apiGroups: ["apps"]
    resources: ["deployments", "replicasets"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: gitea-postdeploy-validator
  namespace: ecommerce
subjects:
  - kind: ServiceAccount
    name: gitea-postdeploy-validator
    namespace: ecommerce
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: gitea-postdeploy-validator
YAML
}

generate_postdeploy_credentials() {
  [[ "$APPLY_POSTDEPLOY_RBAC" == "true" ]] || return 0

  local secret_dir="${APP_DATA_PATH}/gitea-actions-secrets"
  local owner tmp_dir token ca_b64 server kubeconfig_file output_file

  owner="$(id -un)"
  server="https://${LAB_HOST_IP}:${API_PORT}"
  tmp_dir="$(mktemp -d)"
  kubeconfig_file="${tmp_dir}/postdeploy-validator.kubeconfig"

  # No aplicamos chmod a un glob con archivos históricos que podrían pertenecer
  # a root. Generamos todo en un directorio temporal y luego instalamos cada
  # archivo con propietario y permisos explícitos.
  sudo_cmd install -d -m 0700 -o "$owner" "$secret_dir"

  token="$(
    kubectl create token gitea-postdeploy-validator \
      -n ecommerce \
      --duration=8760h
  )"

  ca_b64="$(
    kubectl config view \
      --raw \
      --minify \
      --flatten \
      -o jsonpath='{.clusters[0].cluster.certificate-authority-data}'
  )"

  [[ -n "$token" ]] || die "No fue posible generar el token postdeploy."
  [[ -n "$ca_b64" ]] || die "No fue posible leer la CA del clúster."

  printf '%s\n' "$token" > "${tmp_dir}/K8S_TOKEN.txt"
  printf '%s\n' "$ca_b64" > "${tmp_dir}/K8S_CA_B64.txt"
  printf '%s\n' "$server" > "${tmp_dir}/K8S_SERVER.txt"
  printf '%s\n' "$LAB_HOST_IP" > "${tmp_dir}/K8S_TLS_SERVER_NAME.txt"

  cat > "$kubeconfig_file" <<EOF
apiVersion: v1
kind: Config
clusters:
  - name: ${CLUSTER_NAME}
    cluster:
      certificate-authority-data: ${ca_b64}
      server: ${server}
      tls-server-name: ${LAB_HOST_IP}
contexts:
  - name: gitea-postdeploy-validator@${CLUSTER_NAME}
    context:
      cluster: ${CLUSTER_NAME}
      namespace: ecommerce
      user: gitea-postdeploy-validator
current-context: gitea-postdeploy-validator@${CLUSTER_NAME}
users:
  - name: gitea-postdeploy-validator
    user:
      token: ${token}
EOF

  # Compatibilidad con el workflow histórico que usa un único secret
  # KUBE_CONFIG_DATA en Base64, pero ahora con privilegios mínimos.
  base64 "$kubeconfig_file" |
    tr -d '\n' \
    > "${tmp_dir}/KUBE_CONFIG_DATA.txt"
  printf '\n' >> "${tmp_dir}/KUBE_CONFIG_DATA.txt"

  for output_file in \
    K8S_TOKEN.txt \
    K8S_CA_B64.txt \
    K8S_SERVER.txt \
    K8S_TLS_SERVER_NAME.txt \
    KUBE_CONFIG_DATA.txt
  do
    sudo_cmd install \
      -m 0600 \
      -o "$owner" \
      "${tmp_dir}/${output_file}" \
      "${secret_dir}/${output_file}"
  done

  # Los kubeconfig históricos también contienen credenciales. Ajustamos
  # únicamente propietario y permisos; no asumimos ningún grupo llamado devops.
  for output_file in kubeconfig-gitea.yaml lab-cluster.kubeconfig; do
    if [[ -f "${secret_dir}/${output_file}" ]]; then
      sudo_cmd chown "$owner" "${secret_dir}/${output_file}"
      sudo_cmd chmod 0600 "${secret_dir}/${output_file}"
    fi
  done

  rm -rf "$tmp_dir"

  warn "Credenciales postdeploy regeneradas en ${secret_dir}."
  warn "Actualiza en Gitea K8S_TOKEN/K8S_CA_B64/K8S_SERVER/K8S_TLS_SERVER_NAME o el secret histórico KUBE_CONFIG_DATA."
}


read_optional_secret() {
  local variable_name="$1"
  local file_path="$2"
  local value="${!variable_name:-}"

  if [[ -n "$value" ]]; then
    printf '%s' "$value"
    return 0
  fi

  if [[ -s "$file_path" ]]; then
    tr -d '\r\n' < "$file_path"
    return 0
  fi

  return 1
}


git_with_optional_credentials() {
  local username="" password="" askpass_dir="" askpass_script="" result=0

  username="$(
    read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true
  )"
  password="$(
    read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true
  )"

  if [[ -z "$username" && -z "$password" ]]; then
    GIT_TERMINAL_PROMPT=0 git "$@"
    return
  fi

  [[ -n "$username" && -n "$password" ]] ||
    die "El clon privado requiere usuario y token/password completos."

  askpass_dir="$(mktemp -d)"
  askpass_script="${askpass_dir}/askpass.sh"

  cat > "$askpass_script" <<'ASKPASS'
#!/bin/sh
case "$1" in
  *sername*) printf '%s\n' "$GIT_BOOTSTRAP_USERNAME" ;;
  *assword*) printf '%s\n' "$GIT_BOOTSTRAP_PASSWORD" ;;
  *) printf '\n' ;;
esac
ASKPASS
  chmod 0700 "$askpass_script"

  GIT_TERMINAL_PROMPT=0 \
  GIT_ASKPASS="$askpass_script" \
  GIT_BOOTSTRAP_USERNAME="$username" \
  GIT_BOOTSTRAP_PASSWORD="$password" \
  git "$@" || result=$?

  rm -rf "$askpass_dir"
  return "$result"
}

apps_registry_cache_valid() {
  [[ -d "${APP_REGISTRY_CACHE_DIR}/.git" ]] &&
    git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1
}

record_apps_registry_commit() {
  local commit
  commit="$(
    git -C "$APP_REGISTRY_CACHE_DIR" rev-parse HEAD 2>/dev/null || true
  )"

  [[ -n "$commit" ]] || return 0

  printf '%s\n' "$commit" > "$APP_REGISTRY_COMMIT_FILE"
  chmod 0600 "$APP_REGISTRY_COMMIT_FILE"
  ok "apps-registry cacheado en commit ${commit:0:12}."
}


find_application_manifest_by_name() {
  local app_name="$1"
  local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}"
  local manifest="" manifest_name=""

  while IFS= read -r manifest; do
    manifest_name="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.metadata.name}' 2>/dev/null || true
    )"

    if [[ "$manifest_name" == "$app_name" ]]; then
      printf '%s\n' "$manifest"
      return 0
    fi
  done < <(
    find "$apps_dir" -maxdepth 1 -type f \
      \( -name '*.yaml' -o -name '*.yml' \) | sort
  )

  return 1
}

validate_monitoring_app_contract() {
  local manifest="" revision="" ttl="" ingress_enabled="" ingress_host=""
  local request_cpu="" request_memory="" limit_cpu="" limit_memory=""
  local parameters_dump=""

  manifest="$(find_application_manifest_by_name "$MONITORING_APP_NAME" || true)"
  [[ -n "$manifest" ]] ||
    die "No se encontró Application/${MONITORING_APP_NAME} en ${ROOT_APP_REPO_PATH}/."

  revision="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
  )"

  [[ -n "$revision" ]] ||
    die "${manifest} no contiene spec.source.targetRevision."

  if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]]; then
    die "${MONITORING_APP_NAME} usa ${revision}. Usa ${MONITORING_FIXED_CHART_VERSION} o superior."
  fi

  ttl="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished}' \
      2>/dev/null || true
  )"
  request_cpu="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.cpu}' \
      2>/dev/null || true
  )"
  request_memory="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.memory}' \
      2>/dev/null || true
  )"
  limit_cpu="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.cpu}' \
      2>/dev/null || true
  )"
  limit_memory="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.memory}' \
      2>/dev/null || true
  )"
  parameters_dump="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{range .spec.source.helm.parameters[*]}{.name}={.value}{"\n"}{end}' \
      2>/dev/null || true
  )"

  if [[ -z "$ttl" ]] &&
     ! grep -q '^prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60$' \
       <<< "$parameters_dump"; then
    die "${MONITORING_APP_NAME} debe configurar ttlSecondsAfterFinished=60."
  fi

  [[ -n "$request_cpu" && -n "$request_memory" &&
     -n "$limit_cpu" && -n "$limit_memory" ]] ||
    die "${MONITORING_APP_NAME} debe definir requests/limits del Job admission."

  ingress_enabled="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.enabled}' \
      2>/dev/null || true
  )"
  ingress_host="$(
    kubectl create --dry-run=client -f "$manifest" \
      -o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.hosts[0]}' \
      2>/dev/null || true
  )"

  [[ "$ingress_enabled" == "true" ]] ||
    die "${MONITORING_APP_NAME} debe habilitar grafana.ingress.enabled=true."

  [[ "$ingress_host" == "$MONITORING_INGRESS_HOST" ]] ||
    die "Grafana debe publicarse en ${MONITORING_INGRESS_HOST}; actual=${ingress_host:-vacío}."

  ok "Contrato monitoring válido: chart=${revision}, TTL, recursos admission e Ingress Grafana."
}

validate_apps_registry_cache() {
  apps_registry_cache_valid ||
    die "El caché ${APP_REGISTRY_CACHE_DIR} no es un repositorio Git válido."

  [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]] ||
    die "No existe ${APP_REGISTRY_ROOT_MANIFEST} en el repositorio apps-registry."

  [[ -d "${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" ]] ||
    die "No existe la ruta ${ROOT_APP_REPO_PATH}/ en el repositorio apps-registry."

  kubectl apply \
    --dry-run=client \
    -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \
    >/dev/null

  local manifest_name manifest_namespace
  manifest_name="$(
    kubectl create \
      --dry-run=client \
      -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \
      -o jsonpath='{.metadata.name}' 2>/dev/null || true
  )"
  manifest_namespace="$(
    kubectl create \
      --dry-run=client \
      -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \
      -o jsonpath='{.metadata.namespace}' 2>/dev/null || true
  )"

  [[ "$manifest_name" == "$ROOT_APP_NAME" ]] ||
    die "${APP_REGISTRY_ROOT_MANIFEST} debe crear metadata.name=${ROOT_APP_NAME}; actual=${manifest_name:-vacío}."

  [[ "$manifest_namespace" == "argocd" ]] ||
    die "${APP_REGISTRY_ROOT_MANIFEST} debe usar namespace argocd; actual=${manifest_namespace:-vacío}."

  validate_monitoring_app_contract
}

sync_apps_registry_repo() {
  local update_ok=true current_origin=""

  log "Sincronizando caché local de apps-registry"

  install -d -m 0750 "$(dirname "$APP_REGISTRY_CACHE_DIR")"

  if apps_registry_cache_valid; then
    current_origin="$(
      git -C "$APP_REGISTRY_CACHE_DIR" remote get-url origin 2>/dev/null || true
    )"

    if [[ "$current_origin" != "$ROOT_APP_REPO_URL" ]]; then
      warn "Actualizando origin: ${current_origin:-ausente} -> ${ROOT_APP_REPO_URL}"
      git -C "$APP_REGISTRY_CACHE_DIR" remote set-url origin "$ROOT_APP_REPO_URL"
    fi

    if ! git_with_optional_credentials \
      -C "$APP_REGISTRY_CACHE_DIR" \
      fetch \
      --prune \
      --depth="$APP_REGISTRY_CLONE_DEPTH" \
      origin \
      "$ROOT_APP_REVISION"; then
      update_ok=false
    else
      git -C "$APP_REGISTRY_CACHE_DIR" \
        checkout -B "$ROOT_APP_REVISION" "origin/${ROOT_APP_REVISION}"

      git -C "$APP_REGISTRY_CACHE_DIR" \
        reset --hard "origin/${ROOT_APP_REVISION}"

      git -C "$APP_REGISTRY_CACHE_DIR" clean -fd
    fi
  else
    rm -rf "$APP_REGISTRY_CACHE_DIR"

    if ! git_with_optional_credentials \
      clone \
      --branch "$ROOT_APP_REVISION" \
      --single-branch \
      --depth "$APP_REGISTRY_CLONE_DEPTH" \
      "$ROOT_APP_REPO_URL" \
      "$APP_REGISTRY_CACHE_DIR"; then
      update_ok=false
    fi
  fi

  if [[ "$update_ok" != "true" ]]; then
    if apps_registry_cache_valid &&
      [[ "$APP_REGISTRY_ALLOW_CACHE_FALLBACK" == "true" ]]; then
      warn "Gitea no estuvo disponible; se utilizará el último clon válido de apps-registry."
    else
      die "No fue posible clonar/actualizar ${ROOT_APP_REPO_URL} y no hay un caché válido."
    fi
  fi

  validate_apps_registry_cache
  record_apps_registry_commit
}


platform_infra_cache_valid() {
  [[ -d "${PLATFORM_INFRA_CACHE_DIR}/.git" ]] &&
    git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1
}

record_platform_infra_commit() {
  local commit
  commit="$(
    git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD 2>/dev/null || true
  )"

  [[ -n "$commit" ]] || return 0

  printf '%s\n' "$commit" > "$PLATFORM_INFRA_COMMIT_FILE"
  chmod 0600 "$PLATFORM_INFRA_COMMIT_FILE"
  ok "platform-infra cacheado en commit ${commit:0:12}."
}

sync_platform_infra_repo() {
  local update_ok=true current_origin=""

  log "Sincronizando caché local de platform-infra"

  install -d -m 0750 "$(dirname "$PLATFORM_INFRA_CACHE_DIR")"

  if platform_infra_cache_valid; then
    current_origin="$(
      git -C "$PLATFORM_INFRA_CACHE_DIR" remote get-url origin 2>/dev/null || true
    )"

    if [[ "$current_origin" != "$PLATFORM_INFRA_REPO_URL" ]]; then
      warn "Actualizando origin platform-infra: ${current_origin:-ausente} -> ${PLATFORM_INFRA_REPO_URL}"
      git -C "$PLATFORM_INFRA_CACHE_DIR" remote set-url origin "$PLATFORM_INFRA_REPO_URL"
    fi

    if ! git_with_optional_credentials \
      -C "$PLATFORM_INFRA_CACHE_DIR" \
      fetch \
      --prune \
      --depth="$PLATFORM_INFRA_CLONE_DEPTH" \
      origin \
      "$PLATFORM_INFRA_REVISION"; then
      update_ok=false
    else
      git -C "$PLATFORM_INFRA_CACHE_DIR" \
        checkout -B "$PLATFORM_INFRA_REVISION" "origin/${PLATFORM_INFRA_REVISION}"

      git -C "$PLATFORM_INFRA_CACHE_DIR" \
        reset --hard "origin/${PLATFORM_INFRA_REVISION}"

      git -C "$PLATFORM_INFRA_CACHE_DIR" clean -fd
    fi
  else
    rm -rf "$PLATFORM_INFRA_CACHE_DIR"

    if ! git_with_optional_credentials \
      clone \
      --branch "$PLATFORM_INFRA_REVISION" \
      --single-branch \
      --depth "$PLATFORM_INFRA_CLONE_DEPTH" \
      "$PLATFORM_INFRA_REPO_URL" \
      "$PLATFORM_INFRA_CACHE_DIR"; then
      update_ok=false
    fi
  fi

  if [[ "$update_ok" != "true" ]]; then
    if platform_infra_cache_valid &&
      [[ "$PLATFORM_INFRA_ALLOW_CACHE_FALLBACK" == "true" ]]; then
      warn "Gitea no estuvo disponible; se utilizará el último clon válido de platform-infra."
    else
      die "No fue posible clonar/actualizar ${PLATFORM_INFRA_REPO_URL} y no hay un caché válido."
    fi
  fi

  record_platform_infra_commit
}

governance_applications_tsv() {
  local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}"
  local manifest="" kind="" app_name="" repo_url="" source_path="" revision="" namespace=""

  [[ -d "$apps_dir" ]] || die "No existe ${apps_dir}."

  while IFS= read -r manifest; do
    kind="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.kind}' 2>/dev/null || true
    )"

    [[ "$kind" == "Application" ]] || continue

    repo_url="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.spec.source.repoURL}' 2>/dev/null || true
    )"

    [[ "$repo_url" == "$PLATFORM_INFRA_REPO_URL" ]] || continue

    app_name="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.metadata.name}' 2>/dev/null || true
    )"
    source_path="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.spec.source.path}' 2>/dev/null || true
    )"
    revision="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
    )"
    namespace="$(
      kubectl create --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.spec.destination.namespace}' 2>/dev/null || true
    )"

    [[ -n "$app_name" && -n "$source_path" ]] ||
      die "Application governance inválida: ${manifest}"

    printf '%s\t%s\t%s\t%s\n' \
      "$app_name" \
      "$source_path" \
      "${revision:-$PLATFORM_INFRA_REVISION}" \
      "${namespace:-default}"
  done < <(
    find "$apps_dir" \
      -maxdepth 1 \
      -type f \
      \( -name '*.yaml' -o -name '*.yml' \) \
      | sort
  )
}

validate_monitoring_governance_contract() {
  local rendered_file="${PLATFORM_INFRA_RENDER_DIR}/monitoring-governance.yaml"

  [[ -f "$rendered_file" ]] || return 0

  if grep -Eq '^kind:[[:space:]]*LimitRange[[:space:]]*$' "$rendered_file"; then
    ok "monitoring-governance incluye LimitRange."
    return 0
  fi

  if [[ "$ENFORCE_MONITORING_LIMITRANGE" == "true" ]]; then
    die "monitoring-governance no incluye LimitRange. Agrega limitrange.yaml y decláralo en kustomization.yaml para que ResourceQuota no rechace contenedores sin requests/limits."
  fi

  warn "monitoring-governance no incluye LimitRange; monitoring puede fallar por ResourceQuota."
}

validate_and_render_platform_infra() {
  platform_infra_cache_valid ||
    die "El caché ${PLATFORM_INFRA_CACHE_DIR} no es un repositorio Git válido."

  install -d -m 0750 "$PLATFORM_INFRA_RENDER_DIR"

  local app_name="" source_path="" revision="" namespace=""
  local source_dir="" render_file="" count=0

  while IFS=$'\t' read -r app_name source_path revision namespace; do
    [[ -n "$app_name" ]] || continue
    count=$((count + 1))

    source_dir="${PLATFORM_INFRA_CACHE_DIR}/${source_path}"
    render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml"

    [[ -d "$source_dir" ]] ||
      die "${app_name} referencia una ruta inexistente: ${source_dir}"

    if [[ "$revision" != "$PLATFORM_INFRA_REVISION" ]]; then
      warn "${app_name} usa targetRevision=${revision}; el caché local está en ${PLATFORM_INFRA_REVISION}."
    fi

    kubectl kustomize "$source_dir" > "${render_file}.tmp"
    [[ -s "${render_file}.tmp" ]] ||
      die "Kustomize no generó recursos para ${app_name} (${source_path})."

    {
      printf '# Source: %s\n' "$PLATFORM_INFRA_REPO_URL"
      printf '# Revision: %s\n' "$PLATFORM_INFRA_REVISION"
      printf '# Commit: %s\n' \
        "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD)"
      printf '# Application: %s\n' "$app_name"
      cat "${render_file}.tmp"
    } > "$render_file"

    rm -f "${render_file}.tmp"
    chmod 0600 "$render_file"

    ok "Kustomize validado: ${app_name} -> ${source_path}"
  done < <(governance_applications_tsv)

  (( count > 0 )) ||
    die "No se encontraron Applications que referencien ${PLATFORM_INFRA_REPO_URL}."

  validate_monitoring_governance_contract
}

apply_governance_snapshots() {
  [[ "$APPLY_GOVERNANCE_SNAPSHOTS" == "true" ]] || return 0

  log "Aplicando snapshots governance antes de los workloads"

  local app_name="" source_path="" revision="" namespace=""
  local render_file=""

  while IFS=$'\t' read -r app_name source_path revision namespace; do
    [[ -n "$app_name" ]] || continue

    kubectl create namespace "$namespace" \
      --dry-run=client \
      -o yaml |
      kubectl apply -f - >/dev/null

    render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml"
    [[ -s "$render_file" ]] ||
      die "No existe snapshot renderizado para ${app_name}: ${render_file}"

    kubectl apply \
      --server-side \
      --force-conflicts \
      --field-manager=deploy-k3d-lab-governance \
      -f "$render_file"
  done < <(governance_applications_tsv)

  ok "Governance aplicada desde snapshots Kustomize."
}

sync_gitops_repositories() {
  sync_apps_registry_repo
  sync_platform_infra_repo
  validate_and_render_platform_infra
}

root_application_source_manifest() {
  local repo_manifest="${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}"

  if [[ -f "$repo_manifest" ]]; then
    printf '%s\n' "$repo_manifest"
  else
    printf '%s\n' "$ROOT_APP_MANIFEST"
  fi
}

configure_single_argocd_repository() {
  local secret_name="$1"
  local repo_url="$2"
  local repo_username="$3"
  local repo_password="$4"

  kubectl create secret generic "$secret_name" \
    -n argocd \
    --from-literal=type=git \
    --from-literal=url="$repo_url" \
    --from-literal=username="$repo_username" \
    --from-literal=password="$repo_password" \
    --dry-run=client \
    -o yaml |
    kubectl label \
      --local -f - \
      'argocd.argoproj.io/secret-type=repository' \
      -o yaml |
    kubectl apply -f -
}

configure_argocd_repository() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  local repo_username="" repo_password=""

  repo_username="$(
    read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true
  )"
  repo_password="$(
    read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true
  )"

  if [[ -z "$repo_username" && -z "$repo_password" ]]; then
    warn "No se configuraron credenciales HTTPS; se asumirá que apps-registry y platform-infra son públicos o que Argo CD ya tiene credenciales."
    return 0
  fi

  [[ -n "$repo_username" && -n "$repo_password" ]] ||
    die "Para repositorios privados se requieren usuario y token/password completos."

  log "Registrando repositorios Git privados en Argo CD"

  configure_single_argocd_repository \
    "$ARGOCD_REPO_SECRET_NAME" \
    "$ROOT_APP_REPO_URL" \
    "$repo_username" \
    "$repo_password"

  configure_single_argocd_repository \
    "$ARGOCD_PLATFORM_REPO_SECRET_NAME" \
    "$PLATFORM_INFRA_REPO_URL" \
    "$repo_username" \
    "$repo_password"

  ok "apps-registry y platform-infra configurados en Argo CD."
}

write_root_application_manifest() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  if [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]]; then
    ok "Application raíz obtenida del clon: ${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}"
    return 0
  fi

  warn "No se encontró ${APP_REGISTRY_ROOT_MANIFEST} en el clon; generando manifiesto fallback."

  cat > "$ROOT_APP_MANIFEST" <<YAML
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: ${ROOT_APP_NAME}
  namespace: argocd
  labels:
    app.kubernetes.io/part-of: lab-bootstrap
    app.kubernetes.io/managed-by: deploy-k3d-lab
spec:
  project: ${ROOT_APP_PROJECT}
  source:
    repoURL: '${ROOT_APP_REPO_URL}'
    path: ${ROOT_APP_REPO_PATH}
    targetRevision: ${ROOT_APP_REVISION}
    directory:
      recurse: false
  destination:
    server: 'https://kubernetes.default.svc'
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
YAML

  chmod 0600 "$ROOT_APP_MANIFEST"
}

apply_root_application() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  local source_manifest
  write_root_application_manifest
  source_manifest="$(root_application_source_manifest)"

  log "Aplicando ${ROOT_APP_NAME} desde ${source_manifest}"
  kubectl apply -f "$source_manifest"

  kubectl annotate application "$ROOT_APP_NAME" \
    -n argocd \
    argocd.argoproj.io/refresh=hard \
    --overwrite >/dev/null

  ok "Application raíz aplicada."
}

root_application_diagnostics() {
  kubectl get application "$ROOT_APP_NAME" \
    -n argocd \
    -o wide 2>/dev/null || true

  kubectl get application "$ROOT_APP_NAME" \
    -n argocd \
    -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \
    2>/dev/null || true

  kubectl get pods -n argocd -o wide || true
}

wait_for_root_application() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  log "Esperando sincronización de ${ROOT_APP_NAME}"

  local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS))
  local sync_status="" health_status="" operation_phase=""

  while (( SECONDS < deadline )); do
    sync_status="$(
      kubectl get application "$ROOT_APP_NAME" \
        -n argocd \
        -o jsonpath='{.status.sync.status}' 2>/dev/null || true
    )"

    health_status="$(
      kubectl get application "$ROOT_APP_NAME" \
        -n argocd \
        -o jsonpath='{.status.health.status}' 2>/dev/null || true
    )"

    operation_phase="$(
      kubectl get application "$ROOT_APP_NAME" \
        -n argocd \
        -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true
    )"

    if [[ "$operation_phase" == "Error" || "$operation_phase" == "Failed" ]]; then
      root_application_diagnostics
      die "${ROOT_APP_NAME} terminó con operationState=${operation_phase}."
    fi

    if [[ "$sync_status" == "Synced" ]]; then
      ok "${ROOT_APP_NAME}: Synced / ${health_status:-Unknown}."
      return 0
    fi

    sleep 5
  done

  root_application_diagnostics
  die "${ROOT_APP_NAME} no quedó Synced dentro de ${ROOT_APP_WAIT_SECONDS}s."
}

discover_expected_gitops_apps() {
  local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}"
  local manifest="" kind="" app_name=""

  EXPECTED_APPS_ARRAY=()

  [[ -d "$apps_dir" ]] ||
    die "No existe el directorio de Applications: ${apps_dir}"

  while IFS= read -r manifest; do
    [[ -n "$manifest" ]] || continue

    kind="$(
      kubectl create \
        --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.kind}' 2>/dev/null || true
    )"

    app_name="$(
      kubectl create \
        --dry-run=client \
        -f "$manifest" \
        -o jsonpath='{.metadata.name}' 2>/dev/null || true
    )"

    if [[ "$kind" == "Application" &&
          -n "$app_name" &&
          "$app_name" != "$ROOT_APP_NAME" ]]; then
      EXPECTED_APPS_ARRAY+=("$app_name")
    fi
  done < <(
    find "$apps_dir" \
      -maxdepth 1 \
      -type f \
      \( -name '*.yaml' -o -name '*.yml' \) \
      | sort
  )

  (( ${#EXPECTED_APPS_ARRAY[@]} > 0 )) ||
    die "No se descubrieron recursos kind: Application dentro de ${apps_dir}."
}

expected_gitops_apps_array() {
  local normalized=""

  if [[ -n "${EXPECTED_GITOPS_APPS//[[:space:],]/}" ]]; then
    normalized="${EXPECTED_GITOPS_APPS//,/ }"
    read -r -a EXPECTED_APPS_ARRAY <<< "$normalized"
  else
    discover_expected_gitops_apps
  fi

  ok "Applications esperadas: ${EXPECTED_APPS_ARRAY[*]}"
}

wait_for_expected_gitops_apps() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  expected_gitops_apps_array

  log "Esperando Applications hijas del repositorio"

  local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS))
  local app="" missing_count=0

  while (( SECONDS < deadline )); do
    missing_count=0

    for app in "${EXPECTED_APPS_ARRAY[@]}"; do
      [[ -n "$app" ]] || continue
      kubectl get application "$app" -n argocd >/dev/null 2>&1 ||
        missing_count=$((missing_count + 1))
    done

    if (( missing_count == 0 )); then
      ok "Se crearon todas las Applications esperadas."
      return 0
    fi

    sleep 5
  done

  warn "Applications encontradas:"
  kubectl get applications.argoproj.io -n argocd || true
  die "Faltan ${missing_count} Applications esperadas después de aplicar ${ROOT_APP_NAME}."
}

wait_for_gitops_apps_health() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  expected_gitops_apps_array

  log "Esperando que las Applications hijas se estabilicen"

  local deadline=$((SECONDS + GITOPS_APPS_WAIT_SECONDS))
  local app="" sync_status="" health_status="" pending=0

  while (( SECONDS < deadline )); do
    pending=0

    for app in "${EXPECTED_APPS_ARRAY[@]}"; do
      [[ -n "$app" ]] || continue

      sync_status="$(
        kubectl get application "$app" \
          -n argocd \
          -o jsonpath='{.status.sync.status}' 2>/dev/null || true
      )"

      health_status="$(
        kubectl get application "$app" \
          -n argocd \
          -o jsonpath='{.status.health.status}' 2>/dev/null || true
      )"

      if [[ "$sync_status" != "Synced" || "$health_status" != "Healthy" ]]; then
        pending=$((pending + 1))
      fi
    done

    if (( pending == 0 )); then
      ok "Todas las Applications esperadas están Synced/Healthy."
      return 0
    fi

    sleep 10
  done

  kubectl get applications.argoproj.io -n argocd || true

  if [[ "$GITOPS_STRICT_HEALTH" == "true" ]]; then
    die "${pending} Applications no quedaron Synced/Healthy dentro de ${GITOPS_APPS_WAIT_SECONDS}s."
  fi

  warn "${pending} Applications siguen Progressing, OutOfSync o Degraded; Argo CD continuará reconciliándolas."
}


monitoring_app_diagnostics() {
  local app="$MONITORING_APP_NAME"
  local namespace="$MONITORING_NAMESPACE"
  local revision="" sync_status="" health_status="" phase="" message=""

  log "Diagnóstico de ${app}"

  kubectl get application "$app" -n argocd -o wide || {
    die "No existe Application/${app} en argocd."
  }

  revision="$(
    kubectl get application "$app" \
      -n argocd \
      -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
  )"
  sync_status="$(
    kubectl get application "$app" \
      -n argocd \
      -o jsonpath='{.status.sync.status}' 2>/dev/null || true
  )"
  health_status="$(
    kubectl get application "$app" \
      -n argocd \
      -o jsonpath='{.status.health.status}' 2>/dev/null || true
  )"
  phase="$(
    kubectl get application "$app" \
      -n argocd \
      -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true
  )"
  message="$(
    kubectl get application "$app" \
      -n argocd \
      -o jsonpath='{.status.operationState.message}' 2>/dev/null || true
  )"

  printf 'Chart revision : %s\n' "${revision:-desconocida}"
  printf 'Sync           : %s\n' "${sync_status:-desconocido}"
  printf 'Health         : %s\n' "${health_status:-desconocido}"
  printf 'Operation      : %s\n' "${phase:-ninguna}"
  printf 'Message        : %s\n' "${message:-sin mensaje}"

  echo
  echo "=== Jobs/Pods de admission webhooks ==="
  kubectl get jobs,pods \
    -n "$namespace" \
    -o wide 2>/dev/null |
    grep -E 'NAME|admission-(create|patch)' || true

  echo
  echo "=== Eventos recientes de monitoring ==="
  kubectl get events \
    -n "$namespace" \
    --sort-by='.lastTimestamp' 2>/dev/null |
    tail -60 || true

  echo
  echo "=== Condiciones de Argo CD ==="
  kubectl get application "$app" \
    -n argocd \
    -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \
    2>/dev/null || true

  if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]] &&
     [[ "$message" == *"admission-create"* ||
        "$sync_status" == "OutOfSync" ]]; then
    warn "Se detectó kube-prometheus-stack ${revision}, afectado por la carrera de ttlSecondsAfterFinished: 0 con hooks de Argo CD."
    warn "Actualiza apps/monitoring-app.yaml al menos a ${MONITORING_FIXED_CHART_VERSION} y configura prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60."
  fi
}


argocd_core_cli() {
  local temp_kubeconfig result=0
  temp_kubeconfig="$(mktemp)"

  cp "$KUBECONFIG_PATH" "$temp_kubeconfig"

  # Argo CD --core obtiene el namespace de control desde el contexto actual
  # del kubeconfig. Sin esto busca argocd-cm en "default".
  KUBECONFIG="$temp_kubeconfig" \
    kubectl config set-context \
      --current \
      --namespace=argocd \
      >/dev/null

  if have argocd; then
    KUBECONFIG="$temp_kubeconfig" \
      argocd "$@" --core || result=$?
  else
    docker run --rm \
      --network host \
      -e KUBECONFIG=/kubeconfig \
      -v "${temp_kubeconfig}:/kubeconfig:ro" \
      --entrypoint /usr/local/bin/argocd \
      "$ARGOCD_CLI_IMAGE" \
      "$@" \
      --core || result=$?
  fi

  rm -f "$temp_kubeconfig"
  return "$result"
}

monitoring_operation_fields() {
  local app="$MONITORING_APP_NAME"

  MONITORING_TARGET_REVISION="$(
    kubectl get application "$app" -n argocd \
      -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
  )"
  MONITORING_OPERATION_REVISION="$(
    kubectl get application "$app" -n argocd \
      -o jsonpath='{.status.operationState.syncResult.revision}' 2>/dev/null || true
  )"
  MONITORING_OPERATION_PHASE="$(
    kubectl get application "$app" -n argocd \
      -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true
  )"
  MONITORING_OPERATION_MESSAGE="$(
    kubectl get application "$app" -n argocd \
      -o jsonpath='{.status.operationState.message}' 2>/dev/null || true
  )"
  MONITORING_SYNC_STATUS="$(
    kubectl get application "$app" -n argocd \
      -o jsonpath='{.status.sync.status}' 2>/dev/null || true
  )"
  MONITORING_HEALTH_STATUS="$(
    kubectl get application "$app" -n argocd \
      -o jsonpath='{.status.health.status}' 2>/dev/null || true
  )"

  export MONITORING_TARGET_REVISION MONITORING_OPERATION_REVISION \
    MONITORING_OPERATION_PHASE MONITORING_OPERATION_MESSAGE \
    MONITORING_SYNC_STATUS MONITORING_HEALTH_STATUS
}

monitoring_operation_is_stale() {
  monitoring_operation_fields
  [[ "$MONITORING_OPERATION_PHASE" == "Running" ]] || return 1
  [[ "$MONITORING_FORCE_RECOVERY" == "true" ]] && return 0

  [[ -n "$MONITORING_OPERATION_REVISION" &&
     -n "$MONITORING_TARGET_REVISION" &&
     "$MONITORING_OPERATION_REVISION" != "$MONITORING_TARGET_REVISION" ]]
}

wait_monitoring_operation_stopped() {
  local deadline=$((SECONDS + MONITORING_OPERATION_STOP_WAIT))

  while (( SECONDS < deadline )); do
    monitoring_operation_fields
    if [[ "$MONITORING_OPERATION_PHASE" != "Running" &&
          "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then
      return 0
    fi
    sleep 3
  done

  return 1
}

cleanup_monitoring_admission_jobs() {
  local job=""

  while IFS= read -r job; do
    [[ -n "$job" ]] || continue
    kubectl delete "$job" -n "$MONITORING_NAMESPACE" --ignore-not-found=true
  done < <(
    kubectl get jobs -n "$MONITORING_NAMESPACE" -o name 2>/dev/null |
      grep -E 'admission-(create|patch)' || true
  )
}

wait_monitoring_synced_healthy() {
  local deadline=$((SECONDS + MONITORING_RECOVERY_WAIT_SECONDS))

  while (( SECONDS < deadline )); do
    monitoring_operation_fields

    if [[ "$MONITORING_SYNC_STATUS" == "Synced" &&
          "$MONITORING_HEALTH_STATUS" == "Healthy" &&
          "$MONITORING_OPERATION_PHASE" != "Running" &&
          "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then
      ok "${MONITORING_APP_NAME}: Synced/Healthy en ${MONITORING_TARGET_REVISION}."
      return 0
    fi

    sleep 10
  done

  monitoring_app_diagnostics
  return 1
}

recover_monitoring_operation() {
  local explicit="${1:-false}"
  local refresh_deadline=0

  kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 ||
    die "No existe Application/${MONITORING_APP_NAME}."

  kubectl get limitrange monitoring-default-resources \
    -n "$MONITORING_NAMESPACE" >/dev/null 2>&1 ||
    die "Falta LimitRange/monitoring-default-resources."

  monitoring_operation_fields

  printf 'Target revision    : %s\n' "${MONITORING_TARGET_REVISION:-desconocida}"
  printf 'Operation revision : %s\n' "${MONITORING_OPERATION_REVISION:-ninguna}"
  printf 'Operation phase    : %s\n' "${MONITORING_OPERATION_PHASE:-ninguna}"
  printf 'Operation message  : %s\n' "${MONITORING_OPERATION_MESSAGE:-sin mensaje}"
  printf 'Sync status        : %s\n' "${MONITORING_SYNC_STATUS:-desconocido}"
  printf 'Health status      : %s\n' "${MONITORING_HEALTH_STATUS:-desconocido}"

  # No se inicia otra sincronización cuando la revisión actual ya terminó bien.
  if [[ "$MONITORING_OPERATION_PHASE" == "Succeeded" &&
        "$MONITORING_OPERATION_REVISION" == "$MONITORING_TARGET_REVISION" ]]; then

    kubectl annotate application "$MONITORING_APP_NAME" -n argocd \
      argocd.argoproj.io/refresh=hard --overwrite >/dev/null

    refresh_deadline=$((SECONDS + 90))

    while (( SECONDS < refresh_deadline )); do
      monitoring_operation_fields

      if [[ "$MONITORING_SYNC_STATUS" == "Synced" &&
            "$MONITORING_HEALTH_STATUS" == "Healthy" ]]; then
        ok "${MONITORING_APP_NAME} ya estaba correctamente Synced/Healthy; no se ejecutó un sync adicional."
        return 0
      fi

      sleep 5
    done

    warn "La operación ${MONITORING_TARGET_REVISION} terminó, pero la Application continúa ${MONITORING_SYNC_STATUS}/${MONITORING_HEALTH_STATUS}; se solicitará una nueva reconciliación."
  fi

  if monitoring_operation_is_stale ||
     [[ "$explicit" == "true" && "$MONITORING_OPERATION_PHASE" == "Running" ]]; then
    warn "Terminando operación obsoleta/atascada de ${MONITORING_APP_NAME}."
    argocd_core_cli app terminate-op "$MONITORING_APP_NAME" ||
      die "No fue posible terminar la operación."

    wait_monitoring_operation_stopped ||
      die "La operación no terminó dentro de ${MONITORING_OPERATION_STOP_WAIT}s."

    cleanup_monitoring_admission_jobs
  fi

  kubectl annotate application "$MONITORING_APP_NAME" -n argocd \
    argocd.argoproj.io/refresh=hard --overwrite >/dev/null

  # La Application tiene auto-sync. Primero damos oportunidad al controller
  # para reconciliar sin crear una operación manual redundante.
  refresh_deadline=$((SECONDS + 90))

  while (( SECONDS < refresh_deadline )); do
    monitoring_operation_fields

    if [[ "$MONITORING_SYNC_STATUS" == "Synced" &&
          "$MONITORING_HEALTH_STATUS" == "Healthy" &&
          "$MONITORING_OPERATION_PHASE" != "Running" &&
          "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then
      ok "${MONITORING_APP_NAME}: reconciliada automáticamente."
      return 0
    fi

    if [[ "$MONITORING_OPERATION_PHASE" == "Running" ]]; then
      wait_monitoring_synced_healthy ||
        die "${MONITORING_APP_NAME} no quedó Synced/Healthy."
      return 0
    fi

    sleep 5
  done

  log "Auto-sync no inició; solicitando sync manual de ${MONITORING_APP_NAME}"

  argocd_core_cli app sync "$MONITORING_APP_NAME" \
    --async --prune --assumeYes ||
    die "No fue posible iniciar el sync."

  wait_monitoring_synced_healthy ||
    die "${MONITORING_APP_NAME} no quedó Synced/Healthy."
}

auto_recover_monitoring_if_stale() {
  [[ "$AUTO_RECOVER_STALE_MONITORING_OPERATION" == "true" ]] || return 0
  kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 || return 0

  if monitoring_operation_is_stale; then
    recover_monitoring_operation false
  fi
}

ensure_root_application_present() {
  [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0

  if ! kubectl get crd applications.argoproj.io >/dev/null 2>&1; then
    warn "El CRD Application aún no existe; se omite la verificación de ${ROOT_APP_NAME}."
    return 0
  fi

  if kubectl get application "$ROOT_APP_NAME" -n argocd >/dev/null 2>&1; then
    return 0
  fi

  warn "${ROOT_APP_NAME} no existe; restaurándolo desde el clon persistente."

  if [[ "$APP_REGISTRY_REFRESH_ON_ENSURE" == "true" ]]; then
    sync_apps_registry_repo
  elif ! apps_registry_cache_valid; then
    sync_apps_registry_repo
  else
    validate_apps_registry_cache
  fi

  if [[ "$PLATFORM_INFRA_REFRESH_ON_ENSURE" == "true" ]]; then
    sync_platform_infra_repo
  elif ! platform_infra_cache_valid; then
    sync_platform_infra_repo
  fi

  validate_and_render_platform_infra
  apply_governance_snapshots
  configure_argocd_repository
  apply_root_application
}

runner_container_exists() {
  docker inspect "$GITEA_RUNNER_CONTAINER" >/dev/null 2>&1
}

runner_is_running() {
  [[ "$(docker inspect -f '{{.State.Running}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true)" == "true" ]]
}

runner_is_registered() {
  [[ -s "${GITEA_RUNNER_DATA_DIR}/.runner" ]]
}

prepare_gitea_runner_directories() {
  # El watchdog se ejecuta como devops y no dispone de TTY para sudo.
  # install-autostart deja estas rutas creadas y con propietario correcto.
  install -d \
    -m 0700 \
    "$GITEA_RUNNER_ROOT" \
    "$GITEA_RUNNER_DATA_DIR" ||
    die "No se pudieron preparar las rutas persistentes del Gitea runner como $(id -un)."
}

detect_gitea_runner_image() {
  local detected_image=""

  if [[ -n "${GITEA_RUNNER_IMAGE_OVERRIDE:-}" ]]; then
    printf '%s\n' "$GITEA_RUNNER_IMAGE_OVERRIDE"
    return 0
  fi

  if [[ -s "$GITEA_RUNNER_IMAGE_FILE" ]]; then
    tr -d '\r\n' < "$GITEA_RUNNER_IMAGE_FILE"
    return 0
  fi

  if runner_container_exists; then
    detected_image="$(
      docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true
    )"
  fi

  printf '%s\n' "${detected_image:-$GITEA_RUNNER_IMAGE}"
}

adopt_existing_gitea_runner() {
  runner_container_exists || return 0

  local current_image data_mount=""
  current_image="$(
    docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER"
  )"

  printf '%s\n' "$current_image" > "$GITEA_RUNNER_IMAGE_FILE"
  chmod 0600 "$GITEA_RUNNER_IMAGE_FILE"

  data_mount="$(
    docker inspect \
      -f '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}' \
      "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true
  )"

  if ! runner_is_registered; then
    warn "Copiando el registro actual del runner desde ${GITEA_RUNNER_CONTAINER}:/data."
    docker cp \
      "${GITEA_RUNNER_CONTAINER}:/data/." \
      "${GITEA_RUNNER_DATA_DIR}/" 2>/dev/null || true
  fi

  if [[ "$data_mount" == "$GITEA_RUNNER_DATA_DIR" ]]; then
    docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null
    return 0
  fi

  warn "El runner actual no usa el volumen persistente ${GITEA_RUNNER_DATA_DIR}."

  if [[ "$MIGRATE_EXISTING_RUNNER" != "true" ]]; then
    warn "Se conserva el contenedor actual. Ejecuta MIGRATE_EXISTING_RUNNER=true $0 runner cuando no haya un workflow activo."
    docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null
    return 0
  fi

  runner_is_registered ||
    die "No se pudo recuperar ${GITEA_RUNNER_DATA_DIR}/.runner; no se eliminará el contenedor actual."

  log "Migrando ${GITEA_RUNNER_CONTAINER} al almacenamiento persistente"
  docker stop "$GITEA_RUNNER_CONTAINER" >/dev/null || true
  docker rm "$GITEA_RUNNER_CONTAINER" >/dev/null
}

register_gitea_runner() {
  runner_is_registered && return 0

  [[ -s "$GITEA_RUNNER_TOKEN_FILE" ]] ||
    die "El runner no está registrado. Guarda un token de registro en ${GITEA_RUNNER_TOKEN_FILE} con permisos 0600."

  local runner_image
  runner_image="$(detect_gitea_runner_image)"

  log "Registrando Gitea Actions runner ${GITEA_RUNNER_NAME}"

  docker pull "$runner_image"

  docker run --rm \
    -v "${GITEA_RUNNER_DATA_DIR}:/data" \
    -v "${GITEA_RUNNER_TOKEN_FILE}:/run/secrets/gitea-runner-token:ro" \
    -e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \
    -e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \
    -e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \
    --entrypoint /bin/sh \
    "$runner_image" \
    -ec '
      cd /data
      act_runner register \
        --no-interactive \
        --instance "$GITEA_INSTANCE_URL" \
        --token "$(cat /run/secrets/gitea-runner-token)" \
        --name "$GITEA_RUNNER_NAME" \
        --labels "$GITEA_RUNNER_LABELS"
    '

  runner_is_registered ||
    die "Gitea no generó ${GITEA_RUNNER_DATA_DIR}/.runner."

  printf '%s\n' "$runner_image" > "$GITEA_RUNNER_IMAGE_FILE"
  chmod 0600 "$GITEA_RUNNER_IMAGE_FILE"
}

create_gitea_runner_container() {
  runner_container_exists && return 0

  local runner_image
  runner_image="$(detect_gitea_runner_image)"

  register_gitea_runner

  log "Creando contenedor ${GITEA_RUNNER_CONTAINER}"

  docker run -d \
    --name "$GITEA_RUNNER_CONTAINER" \
    --restart unless-stopped \
    -v "${GITEA_RUNNER_DATA_DIR}:/data" \
    -v /var/run/docker.sock:/var/run/docker.sock \
    -e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \
    -e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \
    -e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \
    "$runner_image" >/dev/null
}

validate_gitea_runner() {
  runner_container_exists ||
    die "No existe el contenedor ${GITEA_RUNNER_CONTAINER}."

  if ! runner_is_running; then
    docker start "$GITEA_RUNNER_CONTAINER" >/dev/null
  fi

  docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null

  local deadline=$((SECONDS + 60))

  while (( SECONDS < deadline )); do
    if runner_is_running; then
      ok "Gitea runner activo: ${GITEA_RUNNER_CONTAINER}."
      return 0
    fi
    sleep 3
  done

  docker logs --tail 150 "$GITEA_RUNNER_CONTAINER" || true
  die "El contenedor ${GITEA_RUNNER_CONTAINER} no quedó activo."
}

ensure_gitea_runner() {
  [[ "$INSTALL_GITEA_RUNNER" == "true" ]] || return 0

  prepare_gitea_runner_directories
  adopt_existing_gitea_runner

  if ! runner_container_exists; then
    if ! runner_is_registered && [[ ! -s "$GITEA_RUNNER_TOKEN_FILE" ]]; then
      if [[ "$GITEA_RUNNER_REQUIRED" == "true" ]]; then
        die "Falta el registro persistente del runner y el token ${GITEA_RUNNER_TOKEN_FILE}."
      fi
      warn "Gitea runner omitido porque no hay registro ni token."
      return 0
    fi

    create_gitea_runner_container
  fi

  validate_gitea_runner
}

apply_bootstrap_manifests() {
  shopt -s nullglob
  local manifests=("$BOOTSTRAP_DIR"/*.yaml "$BOOTSTRAP_DIR"/*.yml)
  shopt -u nullglob

  if (( ${#manifests[@]} == 0 )); then
    warn "No hay manifiestos GitOps en ${BOOTSTRAP_DIR}. Argo CD quedó instalado, pero las Applications deben aplicarse después."
    return 0
  fi

  log "Aplicando manifiestos de bootstrap GitOps"
  kubectl apply -f "$BOOTSTRAP_DIR"
}

print_credentials() {
  local argocd_password='pendiente'

  if kubectl get secret argocd-initial-admin-secret -n argocd >/dev/null 2>&1; then
    argocd_password="$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 --decode)"
  fi

  printf '\n==========================================================\n'
  printf ' LABORATORIO DISPONIBLE\n'
  printf ' ---------------------------------------------------------\n'
  printf ' API Kubernetes : https://%s:%s\n' "$LAB_HOST_IP" "$API_PORT"
  printf ' NPM Argo CD     : http://%s:%s\n' "$LAB_HOST_IP" "$HTTP_PORT"
  printf ' Host Argo CD    : %s\n' "$ARGOCD_HOST"
  printf ' ArgoCD admin    : %s\n' "$argocd_password"
  printf ' Token Headlamp  : kubectl create token headlamp-admin -n headlamp\n'
  printf ' Kubeconfig      : %s\n' "$KUBECONFIG_PATH"
  printf ' Root Application: %s -> %s/%s@%s\n' \
    "$ROOT_APP_NAME" "$ROOT_APP_REPO_URL" "$ROOT_APP_REPO_PATH" "$ROOT_APP_REVISION"
  printf ' Gitea runner    : %s\n' "$GITEA_RUNNER_CONTAINER"
  printf '==========================================================\n'
}

status_report() {
  require_commands
  ensure_docker

  log "Docker"
  docker info --format 'DockerRootDir={{.DockerRootDir}} Driver={{.Driver}}'
  docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \
    --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'

  log "k3d"
  k3d cluster list || true

  if cluster_exists; then
    write_kubeconfig
    log "Kubernetes"
    kubectl get nodes -o wide || true
    kubectl get pods -A || true
    kubectl get ingress -A || true
    kubectl get applications.argoproj.io -n argocd 2>/dev/null || true
  fi

  log "apps-registry local"
  if apps_registry_cache_valid; then
    printf 'Ruta: %s
' "$APP_REGISTRY_CACHE_DIR"
    printf 'Commit: %s
'       "$(git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)"
  else
    warn "No existe un clon válido en ${APP_REGISTRY_CACHE_DIR}."
  fi

  log "platform-infra local"
  if platform_infra_cache_valid; then
    printf 'Ruta: %s
' "$PLATFORM_INFRA_CACHE_DIR"
    printf 'Commit: %s
'       "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)"
  else
    warn "No existe un clon válido en ${PLATFORM_INFRA_CACHE_DIR}."
  fi

  log "Gitea Actions runner"
  if runner_container_exists; then
    docker ps -a       --filter "name=^/${GITEA_RUNNER_CONTAINER}$"       --format 'table {{.Names}}	{{.Image}}	{{.Status}}'
  else
    warn "No existe ${GITEA_RUNNER_CONTAINER}."
  fi
}

backup_cluster_state() {
  local stamp archive
  stamp="$(date '+%Y%m%d-%H%M%S')"
  archive="${BACKUP_DIR}/k3d-lab-state-${stamp}.tar.gz"

  log "Respaldando estado persistente en ${archive}"

  if cluster_exists; then
    k3d cluster stop "$CLUSTER_NAME" || true
  fi

  sudo_cmd tar \
    --acls \
    --xattrs \
    -C "$APP_DATA_PATH" \
    -czf "$archive" \
    k3s storage secrets config bootstrap gitea-runner git rendered 2>/dev/null || \
    sudo_cmd tar -C "$APP_DATA_PATH" -czf "$archive" k3s storage secrets config bootstrap gitea-runner git rendered

  sudo_cmd chown "$(id -un)" "$archive"
  ok "Backup creado: ${archive}"
}

reset_cluster() {
  [[ "${CONFIRM_RESET:-}" == "DELETE-${CLUSTER_NAME}" ]] || \
    die "Reset bloqueado. Usa CONFIRM_RESET=DELETE-${CLUSTER_NAME} $0 reset"

  prepare_directories
  backup_cluster_state

  if cluster_exists; then
    k3d cluster delete "$CLUSTER_NAME"
  fi

  local stamp="$(date '+%Y%m%d-%H%M%S')"
  sudo_cmd mv "$K3S_SERVER_PATH" "${K3S_SERVER_PATH}.reset-${stamp}"
  sudo_cmd mv "${APP_DATA_PATH}/storage" "${APP_DATA_PATH}/storage.reset-${stamp}"
  sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH"
  sudo_cmd install -d -m 0755     "$K3S_SERVER_STORAGE"     "$K3S_AGENT0_STORAGE"     "$K3S_AGENT1_STORAGE"

  create_cluster
}

ensure_cluster_only() {
  require_commands
  ensure_docker
  validate_data_mount
  validate_host_ip

  if ! cluster_exists; then
    if cluster_containers_exist; then
      die "Existen contenedores del lab, pero k3d no reconoce el clúster. Se requiere diagnóstico manual; no se crearán duplicados."
    fi
    die "El clúster ${CLUSTER_NAME} no existe. Ejecuta bootstrap manualmente."
  fi

  start_cluster
  set_restart_policy
  write_kubeconfig
  wait_for_api
  wait_for_nodes
  validate_runtime_after_boot
  ensure_root_application_present
  auto_recover_monitoring_if_stale
  ensure_gitea_runner
}


resolve_service_tool() {
  local command_name="$1"
  local explicit_path="${2:-}"
  local service_home="$3"
  local candidate=""

  for candidate in \
    "$explicit_path" \
    "$(command -v "$command_name" 2>/dev/null || true)" \
    "${service_home}/.local/bin/${command_name}" \
    "${service_home}/bin/${command_name}" \
    "/DATA/.local/bin/${command_name}" \
    "/DATA/bin/${command_name}" \
    "/DATA/AppData/bin/${command_name}" \
    "/usr/local/bin/${command_name}" \
    "/usr/local/sbin/${command_name}" \
    "/usr/bin/${command_name}" \
    "/usr/sbin/${command_name}" \
    "/bin/${command_name}" \
    "/sbin/${command_name}"
  do
    [[ -n "$candidate" ]] || continue
    if [[ -x "$candidate" ]]; then
      readlink -f "$candidate" 2>/dev/null || printf '%s\n' "$candidate"
      return 0
    fi
  done

  return 1
}

append_unique_path_dir() {
  local current_path="$1"
  local new_dir="$2"

  [[ -n "$new_dir" ]] || {
    printf '%s\n' "$current_path"
    return
  }

  case ":${current_path}:" in
    *":${new_dir}:"*)
      printf '%s\n' "$current_path"
      ;;
    *)
      if [[ -n "$current_path" ]]; then
        printf '%s:%s\n' "$current_path" "$new_dir"
      else
        printf '%s\n' "$new_dir"
      fi
      ;;
  esac
}

install_autostart() {
  [[ $(id -u) -eq 0 ]] || die "Ejecuta install-autostart con sudo."

  local service_user="${SUDO_USER:-devops}"
  local service_home service_group docker_root installed_script
  local k3d_path kubectl_path docker_path curl_path tool_path tool_dir

  service_home="$(getent passwd "$service_user" | cut -d: -f6)"
  service_group="$(id -gn "$service_user")"
  docker_root="$(docker info --format '{{.DockerRootDir}}')"
  installed_script="${APP_DATA_PATH}/bin/deploy-k3d-lab"

  [[ -n "$service_home" ]] || die "No fue posible determinar HOME de ${service_user}."
  [[ -n "$service_group" ]] || die "No fue posible determinar el grupo de ${service_user}."
  [[ -n "$docker_root" ]] || die "No fue posible determinar DockerRootDir."

  k3d_path="$(
    resolve_service_tool k3d "${K3D_BIN:-}" "$service_home"
  )" || die "No se encontró k3d para systemd. Usa K3D_BIN=\$(command -v k3d) con sudo -E."

  kubectl_path="$(
    resolve_service_tool kubectl "${KUBECTL_BIN:-}" "$service_home"
  )" || die "No se encontró kubectl para systemd."

  docker_path="$(
    resolve_service_tool docker "${DOCKER_BIN:-}" "$service_home"
  )" || die "No se encontró docker para systemd."

  curl_path="$(
    resolve_service_tool curl "${CURL_BIN:-}" "$service_home"
  )" || die "No se encontró curl para systemd."

  tool_path="${APP_DATA_PATH}/bin"
  for tool_dir in     "$(dirname "$k3d_path")"     "$(dirname "$kubectl_path")"     "$(dirname "$docker_path")"     "$(dirname "$curl_path")"     /usr/local/sbin     /usr/local/bin     /usr/sbin     /usr/bin     /sbin     /bin
  do
    tool_path="$(append_unique_path_dir "$tool_path" "$tool_dir")"
  done

  # ZimaOS no garantiza /usr/local/sbin y su filesystem base puede ser
  # inmutable. Instalamos el ejecutable en /DATA, que es persistente.
  install -d \
    -m 0750 \
    -o "$service_user" \
    "$(dirname "$installed_script")"

  install \
    -m 0750 \
    -o "$service_user" \
    "$0" \
    "$installed_script"

  # El servicio se ejecutará como devops y no puede usar sudo sin TTY.
  # Dejamos listas y escribibles todas las rutas que el modo ensure modifica.
  install -d \
    -m 0750 \
    -o "$service_user" \
    "$APP_DATA_PATH"

  install -d \
    -m 0700 \
    -o "$service_user" \
    "$(dirname "$KUBECONFIG_PATH")"

  # El modo ensure valida y recupera el runner como usuario devops.
  # Preparamos estas rutas durante la instalación privilegiada para que
  # systemd nunca necesite ejecutar sudo de forma no interactiva.
  install -d \
    -m 0700 \
    -o "$service_user" \
    "$GITEA_RUNNER_ROOT" \
    "$GITEA_RUNNER_DATA_DIR"

  if [[ -f "$GITEA_RUNNER_IMAGE_FILE" ]]; then
    chown "$service_user" "$GITEA_RUNNER_IMAGE_FILE"
    chmod 0600 "$GITEA_RUNNER_IMAGE_FILE"
  fi

  if [[ -f "$KUBECONFIG_PATH" ]]; then
    chown "$service_user" "$KUBECONFIG_PATH"
    chmod 0600 "$KUBECONFIG_PATH"
  fi

  install -d -m 0755 \
    /etc/default \
    /etc/systemd/system \
    /etc/systemd/system/docker.service.d

  systemctl enable docker.service >/dev/null
  if systemctl list-unit-files containerd.service >/dev/null 2>&1; then
    systemctl enable containerd.service >/dev/null || true
  fi

  cat > /etc/systemd/system/docker.service.d/10-k3d-storage.conf <<UNIT
[Unit]
RequiresMountsFor=${APP_DATA_PATH} ${docker_root}
After=local-fs.target
UNIT

  cat > /etc/default/k3d-lab <<ENV
CLUSTER_NAME=${CLUSTER_NAME}
APP_DATA_PATH=${APP_DATA_PATH}
KUBECONFIG_PATH=${KUBECONFIG_PATH}
LAB_HOST_IP=${LAB_HOST_IP}
EXPECTED_HOST_IP=${EXPECTED_HOST_IP}
WAIT_SECONDS=${WAIT_SECONDS}
ARGOCD_ROLLOUT_TIMEOUT=${ARGOCD_ROLLOUT_TIMEOUT}
AGENT_RECOVERY_WAIT=${AGENT_RECOVERY_WAIT}
AUTO_REREGISTER_STALE_AGENTS=${AUTO_REREGISTER_STALE_AGENTS}
CHECK_INGRESS_AFTER_BOOT=${CHECK_INGRESS_AFTER_BOOT}
K3D_BIN=${k3d_path}
KUBECTL_BIN=${kubectl_path}
DOCKER_BIN=${docker_path}
CURL_BIN=${curl_path}
LAB_TOOL_PATH=${tool_path}
BOOTSTRAP_ROOT_APP=${BOOTSTRAP_ROOT_APP}
ROOT_APP_NAME=${ROOT_APP_NAME}
ROOT_APP_REPO_URL=${ROOT_APP_REPO_URL}
ROOT_APP_REPO_PATH=${ROOT_APP_REPO_PATH}
ROOT_APP_REVISION=${ROOT_APP_REVISION}
ROOT_APP_PROJECT=${ROOT_APP_PROJECT}
ROOT_APP_WAIT_SECONDS=${ROOT_APP_WAIT_SECONDS}
GITOPS_APPS_WAIT_SECONDS=${GITOPS_APPS_WAIT_SECONDS}
GITOPS_STRICT_HEALTH=${GITOPS_STRICT_HEALTH}
EXPECTED_GITOPS_APPS=${EXPECTED_GITOPS_APPS}
ARGOCD_REPO_USERNAME_FILE=${ARGOCD_REPO_USERNAME_FILE}
ARGOCD_REPO_PASSWORD_FILE=${ARGOCD_REPO_PASSWORD_FILE}
INSTALL_GITEA_RUNNER=${INSTALL_GITEA_RUNNER}
GITEA_RUNNER_REQUIRED=${GITEA_RUNNER_REQUIRED}
GITEA_RUNNER_CONTAINER=${GITEA_RUNNER_CONTAINER}
GITEA_RUNNER_IMAGE=${GITEA_RUNNER_IMAGE}
GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}
GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}
GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}
GITEA_RUNNER_ROOT=${GITEA_RUNNER_ROOT}
GITEA_RUNNER_DATA_DIR=${GITEA_RUNNER_DATA_DIR}
GITEA_RUNNER_IMAGE_FILE=${GITEA_RUNNER_IMAGE_FILE}
GITEA_RUNNER_TOKEN_FILE=${GITEA_RUNNER_TOKEN_FILE}
MIGRATE_EXISTING_RUNNER=false
APP_REGISTRY_CACHE_DIR=${APP_REGISTRY_CACHE_DIR}
APP_REGISTRY_ROOT_MANIFEST=${APP_REGISTRY_ROOT_MANIFEST}
APP_REGISTRY_COMMIT_FILE=${APP_REGISTRY_COMMIT_FILE}
APP_REGISTRY_ALLOW_CACHE_FALLBACK=${APP_REGISTRY_ALLOW_CACHE_FALLBACK}
APP_REGISTRY_REFRESH_ON_ENSURE=${APP_REGISTRY_REFRESH_ON_ENSURE}
APP_REGISTRY_CLONE_DEPTH=${APP_REGISTRY_CLONE_DEPTH}
PLATFORM_INFRA_REPO_URL=${PLATFORM_INFRA_REPO_URL}
PLATFORM_INFRA_REVISION=${PLATFORM_INFRA_REVISION}
PLATFORM_INFRA_CACHE_DIR=${PLATFORM_INFRA_CACHE_DIR}
PLATFORM_INFRA_COMMIT_FILE=${PLATFORM_INFRA_COMMIT_FILE}
PLATFORM_INFRA_ALLOW_CACHE_FALLBACK=${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK}
PLATFORM_INFRA_REFRESH_ON_ENSURE=${PLATFORM_INFRA_REFRESH_ON_ENSURE}
PLATFORM_INFRA_CLONE_DEPTH=${PLATFORM_INFRA_CLONE_DEPTH}
PLATFORM_INFRA_RENDER_DIR=${PLATFORM_INFRA_RENDER_DIR}
APPLY_GOVERNANCE_SNAPSHOTS=${APPLY_GOVERNANCE_SNAPSHOTS}
ENFORCE_MONITORING_LIMITRANGE=${ENFORCE_MONITORING_LIMITRANGE}
MONITORING_GOVERNANCE_PATH=${MONITORING_GOVERNANCE_PATH}
ARGOCD_PLATFORM_REPO_SECRET_NAME=${ARGOCD_PLATFORM_REPO_SECRET_NAME}
MONITORING_APP_NAME=${MONITORING_APP_NAME}
MONITORING_NAMESPACE=${MONITORING_NAMESPACE}
MONITORING_FIXED_CHART_VERSION=${MONITORING_FIXED_CHART_VERSION}
MONITORING_INGRESS_HOST=${MONITORING_INGRESS_HOST}
MONITORING_RECOVERY_WAIT_SECONDS=${MONITORING_RECOVERY_WAIT_SECONDS}
MONITORING_OPERATION_STOP_WAIT=${MONITORING_OPERATION_STOP_WAIT}
AUTO_RECOVER_STALE_MONITORING_OPERATION=${AUTO_RECOVER_STALE_MONITORING_OPERATION}
MONITORING_FORCE_RECOVERY=false
ARGOCD_CLI_IMAGE=${ARGOCD_CLI_IMAGE}
ENV

  cat > /etc/systemd/system/k3d-lab-ensure.service <<UNIT
[Unit]
Description=Recuperar y validar laboratorio k3d
Wants=network-online.target
After=local-fs.target network-online.target docker.service
Requires=docker.service
RequiresMountsFor=${APP_DATA_PATH} ${docker_root}
StartLimitIntervalSec=15min
StartLimitBurst=3

[Service]
Type=oneshot
User=${service_user}
Group=${service_group}
SupplementaryGroups=docker
Environment=HOME=${service_home}
Environment=KUBECONFIG=${KUBECONFIG_PATH}
Environment="PATH=${tool_path}"
EnvironmentFile=-/etc/default/k3d-lab
ExecStart=/bin/bash ${installed_script} ensure
TimeoutStartSec=900
Restart=no
StandardOutput=journal
StandardError=journal
UNIT

  cat > /etc/systemd/system/k3d-lab-ensure.timer <<'UNIT'
[Unit]
Description=Watchdog del laboratorio k3d

[Timer]
OnBootSec=90s
OnUnitInactiveSec=5min
AccuracySec=20s
Persistent=true
Unit=k3d-lab-ensure.service

[Install]
WantedBy=timers.target
UNIT

  systemctl daemon-reload
  systemctl enable --now k3d-lab-ensure.timer

  if ! systemctl start k3d-lab-ensure.service; then
    systemctl status k3d-lab-ensure.service --no-pager -l || true
    journalctl -u k3d-lab-ensure.service -n 200 --no-pager || true
    die "La prueba inicial de k3d-lab-ensure.service falló."
  fi

  ok "Autostart instalado para ${service_user}."
  printf 'Script persistente: %s\n' "$installed_script"
  printf 'k3d para systemd: %s\n' "$k3d_path"
  printf 'kubectl para systemd: %s\n' "$kubectl_path"
  printf 'PATH para systemd: %s\n' "$tool_path"
  printf 'DockerRootDir protegido por systemd: %s\n' "$docker_root"
  printf 'Logs: journalctl -u k3d-lab-ensure.service -n 200 --no-pager\n'
  printf 'Nota: el drop-in de Docker será plenamente verificable después del reinicio de prueba.\n'
}

main() {
  case "$MODE" in
    status)
      status_report
      return
      ;;
    install-autostart)
      install_autostart
      return
      ;;
    ensure)
      ensure_cluster_only
      return
      ;;
    repo-sync)
      require_commands
      ensure_docker
      validate_data_mount
      prepare_directories
      sync_gitops_repositories
      return
      ;;
    runner)
      require_commands
      ensure_docker
      validate_data_mount
      prepare_directories
      ensure_gitea_runner
      return
      ;;
    gitops)
      require_commands
      ensure_docker
      validate_data_mount
      validate_host_ip
      cluster_exists || die "No existe ${CLUSTER_NAME}; ejecuta bootstrap."
      write_kubeconfig
      wait_for_api
      install_argocd
      sync_gitops_repositories
      apply_governance_snapshots
      configure_argocd_repository
      apply_root_application
      wait_for_root_application
      wait_for_expected_gitops_apps
      auto_recover_monitoring_if_stale
      wait_for_gitops_apps_health
      return
      ;;
    monitoring-diagnose)
      require_commands
      ensure_docker
      validate_data_mount
      cluster_exists || die "No existe ${CLUSTER_NAME}."
      write_kubeconfig
      wait_for_api
      monitoring_app_diagnostics
      return
      ;;
    monitoring-recover)
      require_commands
      ensure_docker
      validate_data_mount
      cluster_exists || die "No existe ${CLUSTER_NAME}."
      write_kubeconfig
      wait_for_api
      MONITORING_FORCE_RECOVERY=true recover_monitoring_operation true
      return
      ;;
  esac

  require_commands
  ensure_docker
  validate_data_mount
  validate_host_ip
  prepare_directories
  ensure_cluster_token
  write_k3d_config

  case "$MODE" in
    bootstrap)
      if cluster_exists; then
        start_cluster
      else
        create_cluster
      fi
      ;;
    recover)
      cluster_exists || die "No existe ${CLUSTER_NAME}; utiliza bootstrap para crearlo."
      start_cluster
      ;;
    reset)
      reset_cluster
      ;;
    *)
      die "Modo no válido: ${MODE}. Usa bootstrap, recover, ensure, status, repo-sync, gitops, runner, monitoring-diagnose, monitoring-recover, install-autostart o reset."
      ;;
  esac

  set_restart_policy
  write_kubeconfig
  wait_for_api
  wait_for_nodes

  install_argocd
  apply_argocd_ingress
  install_headlamp_rbac
  apply_postdeploy_rbac
  generate_postdeploy_credentials
  sync_gitops_repositories
  apply_governance_snapshots
  configure_argocd_repository
  write_root_application_manifest
  apply_bootstrap_manifests
  apply_root_application
  wait_for_root_application
  wait_for_expected_gitops_apps
  auto_recover_monitoring_if_stale
  wait_for_gitops_apps_health
  ensure_gitea_runner
  validate_runtime_after_boot

  log "Validación final"
  kubectl get nodes -o wide
  kubectl get pods -n argocd
  kubectl get ingress -A
  kubectl get applications.argoproj.io -n argocd 2>/dev/null || true

  print_credentials
}

main "$@"