#!/usr/bin/env bash # Laboratorio candidato v4.5.7: ZimaOS + Docker + k3d + Argo CD + Multi-Repo + Governance + Monitoring + Portainer + CI/CD # # Modos: # bootstrap Crea el clúster si no existe; si existe, lo inicia y reconcilia. # recover Recupera únicamente un clúster existente. # ensure Health check no destructivo para systemd; no instala Argo CD. # status Muestra diagnóstico del laboratorio. # repo-sync Clona/actualiza apps-registry y platform-infra. # gitops Actualiza el repo y aplica root-apps-registry. # monitoring-diagnose Diagnostica kube-prometheus-stack/Grafana. # monitoring-recover Finaliza una operación obsoleta y resincroniza Grafana. # postdeploy-secrets Regenera y valida credenciales de Gitea Actions. # portainer Valida o recupera el Portainer Edge Agent. # portainer-adopt Captura de forma persistente el Edge Agent actual. # retire-headlamp Elimina residuos de Headlamp cuando ya no existe en Gitea. # runner Adopta, crea o recupera el Gitea Actions runner. # install-autostart Instala servicio/timer systemd, orden de montajes y autorreparación. # reset Reconstrucción limpia; requiere confirmación explícita. # # Ejemplo: # LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh bootstrap set -Eeuo pipefail IFS=$'\n\t' MODE="${1:-bootstrap}" if [[ -z "${BASH_VERSION:-}" ]]; then printf 'ERROR: este script requiere Bash. Ejecútalo con: bash deploy-lab.sh %s\n' "$MODE" >&2 exit 1 fi if [[ $(id -u) -eq 0 && "$MODE" != "install-autostart" ]]; then printf 'ERROR: ejecuta el modo %s como usuario normal, sin sudo.\n' "$MODE" >&2 printf 'Ejemplo: LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh %s\n' "$MODE" >&2 exit 1 fi CLUSTER_NAME="${CLUSTER_NAME:-lab-cluster}" APP_DATA_PATH="${APP_DATA_PATH:-/DATA/AppData/k3d-lab}" KUBECONFIG_PATH="${KUBECONFIG_PATH:-/DATA/.kube/config}" CONFIG_FILE="${CONFIG_FILE:-${APP_DATA_PATH}/config/k3d-lab.yaml}" BOOTSTRAP_DIR="${BOOTSTRAP_DIR:-${APP_DATA_PATH}/bootstrap}" BACKUP_DIR="${BACKUP_DIR:-${APP_DATA_PATH}/backups}" API_PORT="${API_PORT:-46421}" HTTP_PORT="${HTTP_PORT:-90}" HTTPS_PORT="${HTTPS_PORT:-9443}" AGENT_COUNT="${AGENT_COUNT:-2}" K3D_SUBNET="${K3D_SUBNET:-auto}" K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.35.5-k3s1}" ARGOCD_VERSION="${ARGOCD_VERSION:-v3.2.0}" ARGOCD_HOST="${ARGOCD_HOST:-argocd.cruzcloud.net}" EXPECTED_HOST_IP="${EXPECTED_HOST_IP:-192.168.68.61}" ALLOW_IP_CHANGE="${ALLOW_IP_CHANGE:-false}" # Portainer Edge Agent para administrar Kubernetes. INSTALL_PORTAINER_EDGE_AGENT="${INSTALL_PORTAINER_EDGE_AGENT:-true}" PORTAINER_REQUIRED="${PORTAINER_REQUIRED:-true}" PORTAINER_NAMESPACE="${PORTAINER_NAMESPACE:-portainer}" PORTAINER_AGENT_DEPLOYMENT="${PORTAINER_AGENT_DEPLOYMENT:-portainer-agent}" PORTAINER_AGENT_SERVICE="${PORTAINER_AGENT_SERVICE:-portainer-agent}" PORTAINER_AGENT_SERVICEACCOUNT="${PORTAINER_AGENT_SERVICEACCOUNT:-portainer-sa-clusteradmin}" PORTAINER_AGENT_CLUSTERROLEBINDING="${PORTAINER_AGENT_CLUSTERROLEBINDING:-portainer-crb-clusteradmin}" PORTAINER_AGENT_CONFIGMAP="${PORTAINER_AGENT_CONFIGMAP:-portainer-agent-edge}" PORTAINER_AGENT_SECRET="${PORTAINER_AGENT_SECRET:-portainer-agent-edge-key}" PORTAINER_AGENT_IMAGE="${PORTAINER_AGENT_IMAGE:-portainer/agent:2.43.0}" PORTAINER_SERVER_URL="${PORTAINER_SERVER_URL:-https://192.168.68.61:9444}" PORTAINER_TUNNEL_HOST="${PORTAINER_TUNNEL_HOST:-192.168.68.61}" PORTAINER_TUNNEL_PORT="${PORTAINER_TUNNEL_PORT:-8000}" PORTAINER_EDGE_INSECURE_POLL="${PORTAINER_EDGE_INSECURE_POLL:-1}" PORTAINER_ROLLOUT_TIMEOUT="${PORTAINER_ROLLOUT_TIMEOUT:-180}" PORTAINER_BUNDLE_FILE="${PORTAINER_BUNDLE_FILE:-${APP_DATA_PATH}/secrets/portainer-edge-agent.bundle.json}" PORTAINER_CAPTURE_ON_SUCCESS="${PORTAINER_CAPTURE_ON_SUCCESS:-true}" REMOVE_LEGACY_HEADLAMP="${REMOVE_LEGACY_HEADLAMP:-true}" APPLY_POSTDEPLOY_RBAC="${APPLY_POSTDEPLOY_RBAC:-true}" POSTDEPLOY_SERVICE_ACCOUNT="${POSTDEPLOY_SERVICE_ACCOUNT:-gitea-postdeploy-validator}" POSTDEPLOY_TOKEN_SECRET="${POSTDEPLOY_TOKEN_SECRET:-gitea-postdeploy-validator-token}" POSTDEPLOY_NAMESPACE="${POSTDEPLOY_NAMESPACE:-ecommerce}" POSTDEPLOY_TOKEN_WAIT_SECONDS="${POSTDEPLOY_TOKEN_WAIT_SECONDS:-90}" WAIT_SECONDS="${WAIT_SECONDS:-240}" ARGOCD_ROLLOUT_TIMEOUT="${ARGOCD_ROLLOUT_TIMEOUT:-420}" AGENT_RECOVERY_WAIT="${AGENT_RECOVERY_WAIT:-75}" AUTO_REREGISTER_STALE_AGENTS="${AUTO_REREGISTER_STALE_AGENTS:-true}" CHECK_INGRESS_AFTER_BOOT="${CHECK_INGRESS_AFTER_BOOT:-true}" # Arranque conservador basado en la secuencia previamente confirmada: # Docker restaura k3d y el runner con unless-stopped; systemd solo interviene # después de una ventana amplia si la API sigue caída. K3D_AUTOSTART="${K3D_AUTOSTART:-false}" AUTOSTART_BOOT_GRACE_SECONDS="${AUTOSTART_BOOT_GRACE_SECONDS:-300}" AUTOSTART_DOCKER_RECOVERY_SECONDS="${AUTOSTART_DOCKER_RECOVERY_SECONDS:-180}" AUTOSTART_K3D_START_TIMEOUT="${AUTOSTART_K3D_START_TIMEOUT:-360}" K3D_CONTAINER_RESTART_POLICY="${K3D_CONTAINER_RESTART_POLICY:-unless-stopped}" GITEA_RUNNER_RESTART_POLICY="${GITEA_RUNNER_RESTART_POLICY:-unless-stopped}" # Bootstrap GitOps / App-of-Apps. BOOTSTRAP_ROOT_APP="${BOOTSTRAP_ROOT_APP:-true}" ROOT_APP_NAME="${ROOT_APP_NAME:-root-apps-registry}" ROOT_APP_REPO_URL="${ROOT_APP_REPO_URL:-https://gitea.cruzcloud.net/devops/apps-registry.git}" ROOT_APP_REPO_PATH="${ROOT_APP_REPO_PATH:-apps}" ROOT_APP_REVISION="${ROOT_APP_REVISION:-main}" ROOT_APP_PROJECT="${ROOT_APP_PROJECT:-default}" ROOT_APP_MANIFEST="${ROOT_APP_MANIFEST:-${BOOTSTRAP_DIR}/00-root-apps-registry.yaml}" ROOT_APP_WAIT_SECONDS="${ROOT_APP_WAIT_SECONDS:-600}" GITOPS_APPS_WAIT_SECONDS="${GITOPS_APPS_WAIT_SECONDS:-900}" GITOPS_STRICT_HEALTH="${GITOPS_STRICT_HEALTH:-false}" EXPECTED_GITOPS_APPS="${EXPECTED_GITOPS_APPS:-}" ARGOCD_REPO_SECRET_NAME="${ARGOCD_REPO_SECRET_NAME:-apps-registry-repository}" ARGOCD_REPO_USERNAME_FILE="${ARGOCD_REPO_USERNAME_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-username}" ARGOCD_REPO_PASSWORD_FILE="${ARGOCD_REPO_PASSWORD_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-password}" # Copia local persistente del repositorio GitOps. APP_REGISTRY_CACHE_DIR="${APP_REGISTRY_CACHE_DIR:-${APP_DATA_PATH}/git/apps-registry}" APP_REGISTRY_ROOT_MANIFEST="${APP_REGISTRY_ROOT_MANIFEST:-application.yaml}" APP_REGISTRY_COMMIT_FILE="${APP_REGISTRY_COMMIT_FILE:-${APP_DATA_PATH}/git/apps-registry.commit}" APP_REGISTRY_ALLOW_CACHE_FALLBACK="${APP_REGISTRY_ALLOW_CACHE_FALLBACK:-true}" APP_REGISTRY_REFRESH_ON_ENSURE="${APP_REGISTRY_REFRESH_ON_ENSURE:-false}" APP_REGISTRY_CLONE_DEPTH="${APP_REGISTRY_CLONE_DEPTH:-1}" # Repositorio de governance/Kustomize. PLATFORM_INFRA_REPO_URL="${PLATFORM_INFRA_REPO_URL:-https://gitea.cruzcloud.net/devops/platform-infra.git}" PLATFORM_INFRA_REVISION="${PLATFORM_INFRA_REVISION:-main}" PLATFORM_INFRA_CACHE_DIR="${PLATFORM_INFRA_CACHE_DIR:-${APP_DATA_PATH}/git/platform-infra}" PLATFORM_INFRA_COMMIT_FILE="${PLATFORM_INFRA_COMMIT_FILE:-${APP_DATA_PATH}/git/platform-infra.commit}" PLATFORM_INFRA_ALLOW_CACHE_FALLBACK="${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK:-true}" PLATFORM_INFRA_REFRESH_ON_ENSURE="${PLATFORM_INFRA_REFRESH_ON_ENSURE:-false}" PLATFORM_INFRA_CLONE_DEPTH="${PLATFORM_INFRA_CLONE_DEPTH:-1}" PLATFORM_INFRA_RENDER_DIR="${PLATFORM_INFRA_RENDER_DIR:-${APP_DATA_PATH}/rendered/platform-infra}" APPLY_GOVERNANCE_SNAPSHOTS="${APPLY_GOVERNANCE_SNAPSHOTS:-true}" ENFORCE_MONITORING_LIMITRANGE="${ENFORCE_MONITORING_LIMITRANGE:-true}" MONITORING_GOVERNANCE_PATH="${MONITORING_GOVERNANCE_PATH:-monitoring-governance}" ARGOCD_PLATFORM_REPO_SECRET_NAME="${ARGOCD_PLATFORM_REPO_SECRET_NAME:-platform-infra-repository}" # Diagnóstico de monitoring/Grafana. MONITORING_APP_NAME="${MONITORING_APP_NAME:-monitoring-app}" MONITORING_NAMESPACE="${MONITORING_NAMESPACE:-monitoring}" MONITORING_FIXED_CHART_VERSION="${MONITORING_FIXED_CHART_VERSION:-58.3.3}" MONITORING_INGRESS_HOST="${MONITORING_INGRESS_HOST:-grafana.cruzcloud.net}" MONITORING_RECOVERY_WAIT_SECONDS="${MONITORING_RECOVERY_WAIT_SECONDS:-900}" MONITORING_OPERATION_STOP_WAIT="${MONITORING_OPERATION_STOP_WAIT:-180}" AUTO_RECOVER_STALE_MONITORING_OPERATION="${AUTO_RECOVER_STALE_MONITORING_OPERATION:-true}" MONITORING_FORCE_RECOVERY="${MONITORING_FORCE_RECOVERY:-false}" ARGOCD_CLI_IMAGE="${ARGOCD_CLI_IMAGE:-quay.io/argoproj/argocd:${ARGOCD_VERSION}}" # Gitea Actions runner externo al clúster. INSTALL_GITEA_RUNNER="${INSTALL_GITEA_RUNNER:-true}" GITEA_RUNNER_REQUIRED="${GITEA_RUNNER_REQUIRED:-true}" GITEA_RUNNER_CONTAINER="${GITEA_RUNNER_CONTAINER:-gitea-runner}" GITEA_RUNNER_IMAGE="${GITEA_RUNNER_IMAGE:-docker.io/gitea/act_runner:latest}" GITEA_INSTANCE_URL="${GITEA_INSTANCE_URL:-https://gitea.cruzcloud.net}" GITEA_RUNNER_NAME="${GITEA_RUNNER_NAME:-zimaos-lab-runner}" GITEA_RUNNER_LABELS="${GITEA_RUNNER_LABELS:-ubuntu-latest:docker://node:20-bookworm}" GITEA_RUNNER_ROOT="${GITEA_RUNNER_ROOT:-${APP_DATA_PATH}/gitea-runner}" GITEA_RUNNER_DATA_DIR="${GITEA_RUNNER_DATA_DIR:-${GITEA_RUNNER_ROOT}/data}" GITEA_RUNNER_IMAGE_FILE="${GITEA_RUNNER_IMAGE_FILE:-${GITEA_RUNNER_ROOT}/image.txt}" GITEA_RUNNER_TOKEN_FILE="${GITEA_RUNNER_TOKEN_FILE:-${APP_DATA_PATH}/secrets/gitea-runner-registration-token}" MIGRATE_EXISTING_RUNNER="${MIGRATE_EXISTING_RUNNER:-false}" K3S_SERVER_PATH="${APP_DATA_PATH}/k3s/server" K3S_SERVER_STORAGE="${APP_DATA_PATH}/storage/server-0" K3S_AGENT0_STORAGE="${APP_DATA_PATH}/storage/agent-0" K3S_AGENT1_STORAGE="${APP_DATA_PATH}/storage/agent-1" TOKEN_FILE="${APP_DATA_PATH}/secrets/k3s-cluster-token" POSTDEPLOY_RBAC_FILE="${APP_DATA_PATH}/gitea-postdeploy-validator-rbac.yaml" if [[ -t 1 ]]; then C_RESET=$'\033[0m' C_BLUE=$'\033[1;34m' C_GREEN=$'\033[1;32m' C_YELLOW=$'\033[1;33m' C_RED=$'\033[1;31m' else C_RESET='' C_BLUE='' C_GREEN='' C_YELLOW='' C_RED='' fi log() { printf '\n%s[%s] %s%s\n' "$C_BLUE" "$(date '+%F %T')" "$*" "$C_RESET"; } ok() { printf '%sOK:%s %s\n' "$C_GREEN" "$C_RESET" "$*"; } warn() { printf '%sAVISO:%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } die() { printf '%sERROR:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; exit 1; } on_error() { local exit_code=$? printf '%sERROR:%s fallo en la línea %s: %s\n' \ "$C_RED" "$C_RESET" "${BASH_LINENO[0]:-desconocida}" "${BASH_COMMAND:-desconocido}" >&2 exit "$exit_code" } trap on_error ERR have() { command -v "$1" >/dev/null 2>&1; } sudo_cmd() { if [[ $(id -u) -eq 0 ]]; then "$@" else sudo "$@" fi } require_commands() { local command_name for command_name in docker k3d kubectl curl git awk sed grep findmnt ss tar base64 tr flock; do have "$command_name" || die "No se encontró el comando requerido: ${command_name}" done } detect_host_ip() { ip -4 route get 1.1.1.1 2>/dev/null | awk '{ for (i = 1; i <= NF; i++) { if ($i == "src") { print $(i + 1) exit } } }' } LAB_HOST_IP="${LAB_HOST_IP:-$(detect_host_ip)}" export LAB_HOST_IP CLUSTER_NAME APP_DATA_PATH K3D_SUBNET K3S_IMAGE AGENT_COUNT API_PORT HTTP_PORT HTTPS_PORT validate_host_ip() { [[ -n "$LAB_HOST_IP" ]] || die "No fue posible detectar LAB_HOST_IP. Defínela manualmente." if [[ -n "$EXPECTED_HOST_IP" && "$LAB_HOST_IP" != "$EXPECTED_HOST_IP" ]]; then if [[ "$ALLOW_IP_CHANGE" == "true" ]]; then warn "La IP detectada ${LAB_HOST_IP} no coincide con ${EXPECTED_HOST_IP}; se continuará por ALLOW_IP_CHANGE=true." else die "La IP detectada es ${LAB_HOST_IP}, pero NPM/Gitea esperan ${EXPECTED_HOST_IP}. Reserva la IP o usa ALLOW_IP_CHANGE=true conscientemente." fi fi } ensure_docker() { if have systemctl && ! systemctl is-active --quiet docker.service; then if [[ $(id -u) -eq 0 || -t 0 ]]; then log "Iniciando Docker" sudo_cmd systemctl start docker.service else die "Docker no está activo. systemd debe iniciarlo antes de ejecutar el watchdog." fi fi docker info >/dev/null 2>&1 || die "Docker no responde para el usuario $(id -un). Valida docker.service y el grupo docker." } validate_data_mount() { findmnt -T /DATA >/dev/null 2>&1 || die "/DATA no está montado. No se iniciará k3d para evitar escribir en el filesystem raíz." [[ -d "$APP_DATA_PATH" ]] || die "No existe la ruta persistente ${APP_DATA_PATH}." # En ZimaOS el usuario devops no puede escribir directamente en /DATA, # pero sí debe poder escribir en su ruta del laboratorio. local test_file="${APP_DATA_PATH}/.k3d-write-test.$$" touch "$test_file" 2>/dev/null || die "${APP_DATA_PATH} no permite escritura para $(id -un)." rm -f "$test_file" } prepare_directories() { log "Preparando almacenamiento persistente" sudo_cmd install -d -m 0750 -o "$(id -un)" \ "$APP_DATA_PATH" \ "${APP_DATA_PATH}/config" \ "${APP_DATA_PATH}/secrets" \ "${APP_DATA_PATH}/run" \ "$BOOTSTRAP_DIR" \ "$BACKUP_DIR" \ "$GITEA_RUNNER_ROOT" \ "$GITEA_RUNNER_DATA_DIR" \ "$(dirname "$APP_REGISTRY_CACHE_DIR")" \ "$(dirname "$PLATFORM_INFRA_CACHE_DIR")" \ "$PLATFORM_INFRA_RENDER_DIR" \ "$(dirname "$KUBECONFIG_PATH")" # K3s y los workloads escriben como root dentro de los nodos. sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH" sudo_cmd install -d -m 0755 \ "$K3S_SERVER_STORAGE" \ "$K3S_AGENT0_STORAGE" \ "$K3S_AGENT1_STORAGE" sudo_cmd chown "$(id -un)" "$(dirname "$KUBECONFIG_PATH")" } generate_cluster_token() { local token="" if have openssl; then token="$(openssl rand -hex 32)" elif have od; then token="$( od -An -N32 -tx1 /dev/urandom | awk '{ for (i = 1; i <= NF; i++) { printf "%s", $i } } END { printf "\n" }' )" elif have hexdump; then token="$(hexdump -vn32 -e '1/1 "%02x"' /dev/urandom)" else die "No hay un generador aleatorio disponible. Se requiere openssl, od o hexdump." fi [[ "$token" =~ ^[0-9a-fA-F]{64}$ ]] || die "No fue posible generar un token hexadecimal seguro." printf '%s\n' "$token" } ensure_cluster_token() { if [[ ! -s "$TOKEN_FILE" ]]; then log "Generando token persistente del clúster" umask 077 generate_cluster_token > "$TOKEN_FILE" chmod 0600 "$TOKEN_FILE" fi K3S_CLUSTER_TOKEN="$(<"$TOKEN_FILE")" [[ "$K3S_CLUSTER_TOKEN" =~ ^[0-9a-fA-F]{64}$ ]] || die "El token persistente de K3s no es válido." export K3S_CLUSTER_TOKEN } write_k3d_config() { log "Generando configuración declarativa de k3d" cat > "$CONFIG_FILE" <<'YAML' apiVersion: k3d.io/v1alpha5 kind: Simple metadata: name: ${CLUSTER_NAME} servers: 1 agents: ${AGENT_COUNT} image: ${K3S_IMAGE} subnet: "${K3D_SUBNET}" token: "${K3S_CLUSTER_TOKEN}" kubeAPI: host: "${LAB_HOST_IP}" hostIP: "0.0.0.0" hostPort: "${API_PORT}" ports: - port: "${HTTP_PORT}:80" nodeFilters: - loadbalancer - port: "${HTTPS_PORT}:443" nodeFilters: - loadbalancer volumes: # Conserva SQLite, certificados y token de K3s fuera de los contenedores. - volume: "${APP_DATA_PATH}/k3s/server:/var/lib/rancher/k3s/server" nodeFilters: - server:0 # Conserva datos de PVC creados por local-path-provisioner. - volume: "${APP_DATA_PATH}/storage/server-0:/var/lib/rancher/k3s/storage" nodeFilters: - server:0 - volume: "${APP_DATA_PATH}/storage/agent-0:/var/lib/rancher/k3s/storage" nodeFilters: - agent:0 - volume: "${APP_DATA_PATH}/storage/agent-1:/var/lib/rancher/k3s/storage" nodeFilters: - agent:1 options: k3d: wait: true timeout: "240s" disableRollback: false k3s: extraArgs: - arg: "--tls-san=${LAB_HOST_IP}" nodeFilters: - server:* kubeconfig: updateDefaultKubeconfig: false switchCurrentContext: false YAML chmod 0600 "$CONFIG_FILE" } cluster_exists() { k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | grep -Fxq "$CLUSTER_NAME" } cluster_containers_exist() { docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" --format '{{.ID}}' | grep -q . } ports_in_use_by_non_cluster_process() { local port for port in "$API_PORT" "$HTTP_PORT" "$HTTPS_PORT"; do if ss -lntH "sport = :${port}" 2>/dev/null | grep -q .; then return 0 fi done return 1 } create_cluster() { if cluster_containers_exist; then die "Existen contenedores etiquetados para ${CLUSTER_NAME}, pero k3d no reconoce el clúster. No se crearán duplicados." fi if ports_in_use_by_non_cluster_process; then sudo_cmd ss -lntp | grep -E ":(${API_PORT}|${HTTP_PORT}|${HTTPS_PORT})\\b" || true die "Uno o más puertos del laboratorio están ocupados." fi log "Creando clúster ${CLUSTER_NAME} con IPAM estático" docker pull "$K3S_IMAGE" k3d cluster create --config "$CONFIG_FILE" } start_cluster() { log "Iniciando clúster existente ${CLUSTER_NAME}" k3d cluster start \ "$CLUSTER_NAME" \ --wait \ --timeout "${AUTOSTART_K3D_START_TIMEOUT}s" } cluster_api_ready() { [[ -s "$KUBECONFIG_PATH" ]] || return 1 KUBECONFIG="$KUBECONFIG_PATH" \ kubectl get --raw='/readyz' \ --request-timeout='5s' >/dev/null 2>&1 } wait_for_cluster_api_quiet() { local timeout_seconds="$1" local deadline=$((SECONDS + timeout_seconds)) while (( SECONDS < deadline )); do if cluster_api_ready; then return 0 fi sleep 5 done return 1 } acquire_ensure_lock() { install -d -m 0750 "${APP_DATA_PATH}/run" exec 9>"${APP_DATA_PATH}/run/k3d-lab-ensure.lock" if ! flock -n 9; then warn "Ya existe otra recuperación del laboratorio en ejecución; se omite esta iteración." exit 0 fi } wait_for_boot_grace() { [[ "$K3D_AUTOSTART" == "true" ]] || return 0 local uptime_seconds remaining uptime_seconds="$( awk '{print int($1)}' /proc/uptime 2>/dev/null || printf '0\n' )" if (( uptime_seconds < AUTOSTART_BOOT_GRACE_SECONDS )); then remaining=$((AUTOSTART_BOOT_GRACE_SECONDS - uptime_seconds)) log "Esperando ${remaining}s para que ZimaOS y Docker restauren las aplicaciones" sleep "$remaining" fi } set_restart_policy() { mapfile -t cluster_containers < <( docker ps -aq --filter "label=k3d.cluster=${CLUSTER_NAME}" ) if (( ${#cluster_containers[@]} > 0 )); then docker update \ --restart="$K3D_CONTAINER_RESTART_POLICY" \ "${cluster_containers[@]}" >/dev/null ok "Política Docker ${K3D_CONTAINER_RESTART_POLICY} aplicada a ${#cluster_containers[@]} contenedores k3d." fi if docker inspect "$GITEA_RUNNER_CONTAINER" >/dev/null 2>&1; then docker update \ --restart="$GITEA_RUNNER_RESTART_POLICY" \ "$GITEA_RUNNER_CONTAINER" >/dev/null ok "Política Docker ${GITEA_RUNNER_RESTART_POLICY} aplicada a ${GITEA_RUNNER_CONTAINER}." fi } start_runner_if_needed() { local runner_running docker inspect "$GITEA_RUNNER_CONTAINER" >/dev/null 2>&1 || return 0 runner_running="$( docker inspect \ -f '{{.State.Running}}' \ "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true )" if [[ "$runner_running" != "true" ]]; then log "Iniciando ${GITEA_RUNNER_CONTAINER}" docker start "$GITEA_RUNNER_CONTAINER" >/dev/null fi } recover_cluster_conservatively() { if cluster_api_ready; then ok "La API ya está disponible; no se ejecutará k3d cluster start." return 0 fi log "La API aún no responde; esperando restauración natural de Docker" if wait_for_cluster_api_quiet "$AUTOSTART_DOCKER_RECOVERY_SECONDS"; then ok "La API se recuperó mediante las políticas de Docker." return 0 fi warn "La API no se recuperó en ${AUTOSTART_DOCKER_RECOVERY_SECONDS}s; se ejecutará un único k3d cluster start." if ! start_cluster; then warn "k3d cluster start devolvió error; se validará igualmente la API porque K3s puede continuar iniciando." fi wait_for_cluster_api_quiet "$AUTOSTART_K3D_START_TIMEOUT" || die "La API de Kubernetes no se recuperó después del arranque conservador." } write_kubeconfig() { local temp_file context_name configured_server temp_file="$(mktemp)" k3d kubeconfig get "$CLUSTER_NAME" > "$temp_file" [[ -s "$temp_file" ]] || die "k3d no generó el kubeconfig." context_name="$( KUBECONFIG="$temp_file" \ kubectl config get-contexts -o name 2>/dev/null | head -1 )" [[ -n "$context_name" ]] || die "El kubeconfig generado no contiene ningún contexto." # Algunas combinaciones de k3d/kubectl pueden producir un archivo válido # sin current-context. Lo fijamos explícitamente antes de instalarlo. KUBECONFIG="$temp_file" \ kubectl config use-context "$context_name" >/dev/null configured_server="$( KUBECONFIG="$temp_file" \ kubectl config view \ --minify \ -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null )" [[ -n "$configured_server" ]] || die "El kubeconfig generado no contiene un API Server válido." # El watchdog se ejecuta sin TTY y no debe invocar sudo. # install-autostart garantiza previamente el propietario de esta ruta. install -d \ -m 0700 \ "$(dirname "$KUBECONFIG_PATH")" || die "No se pudo preparar $(dirname "$KUBECONFIG_PATH") como $(id -un)." install \ -m 0600 \ "$temp_file" \ "$KUBECONFIG_PATH" || die "No se pudo actualizar ${KUBECONFIG_PATH} como $(id -un)." rm -f "$temp_file" export KUBECONFIG="$KUBECONFIG_PATH" [[ "$(kubectl config current-context 2>/dev/null || true)" == "$context_name" ]] || die "No fue posible activar el contexto ${context_name}." ok "Kubeconfig actualizado: ${KUBECONFIG_PATH}" ok "Contexto activo: ${context_name}" ok "API Server: ${configured_server}" } wait_for_api() { log "Esperando API de Kubernetes" local deadline=$((SECONDS + WAIT_SECONDS)) while (( SECONDS < deadline )); do if kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1; then ok "API de Kubernetes disponible." return 0 fi sleep 5 done docker logs --tail 200 "k3d-${CLUSTER_NAME}-server-0" || true die "La API no quedó disponible dentro de ${WAIT_SECONDS}s." } node_ready() { local node_name="$1" [[ "$(kubectl get node "$node_name" \ -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' \ 2>/dev/null || true)" == "True" ]] } wait_agent_ready() { local node_name="$1" local timeout_seconds="$2" local deadline=$((SECONDS + timeout_seconds)) while (( SECONDS < deadline )); do if node_ready "$node_name"; then return 0 fi sleep 5 done return 1 } recover_not_ready_agents() { local index node_name container_name docker_ip kubernetes_ip # El servidor y el API deben estar sanos antes de reparar agentes. kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1 || die "No se repararán agentes porque el API Server no está disponible." for (( index = 0; index < AGENT_COUNT; index++ )); do node_name="k3d-${CLUSTER_NAME}-agent-${index}" container_name="$node_name" if node_ready "$node_name"; then continue fi warn "${node_name} no existe o no está Ready; reiniciando el contenedor." docker restart "$container_name" >/dev/null || true if wait_agent_ready "$node_name" "$AGENT_RECOVERY_WAIT"; then ok "${node_name} se recuperó después del reinicio." continue fi docker_ip="$( docker inspect \ -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ "$container_name" 2>/dev/null || true )" kubernetes_ip="$( kubectl get node "$node_name" \ -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' \ 2>/dev/null || true )" warn "${node_name} continúa sin estar Ready. DockerIP=${docker_ip:-ausente}, KubernetesIP=${kubernetes_ip:-ausente}." if [[ "$AUTO_REREGISTER_STALE_AGENTS" != "true" ]]; then continue fi # En un lab, si el agente lleva el periodo de gracia completo sin responder, # se elimina únicamente su objeto Node y password secret para permitir # un registro limpio con la IP estática actual. warn "Forzando re-registro seguro de ${node_name}." kubectl delete node "$node_name" --ignore-not-found=true >/dev/null kubectl delete secret \ -n kube-system \ "${node_name}.node-password.k3s" \ --ignore-not-found=true >/dev/null docker restart "$container_name" >/dev/null if wait_agent_ready "$node_name" "$((AGENT_RECOVERY_WAIT * 2))"; then ok "${node_name} volvió a registrarse y está Ready." else docker logs --tail 180 "$container_name" || true die "${node_name} no se recuperó después del re-registro." fi done if ! docker ps --format '{{.Names}}' | grep -Fxq "k3d-${CLUSTER_NAME}-serverlb"; then warn "El load balancer no está activo; intentando iniciarlo." docker start "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true fi } nodes_are_ready() { local expected_nodes=$((AGENT_COUNT + 1)) local actual_nodes ready_nodes actual_nodes="$(kubectl get nodes --no-headers 2>/dev/null | awk 'NF {count++} END {print count+0}')" ready_nodes="$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep -c '^True$' || true)" [[ "$actual_nodes" -eq "$expected_nodes" && "$ready_nodes" -eq "$expected_nodes" ]] } wait_for_nodes_once() { local deadline=$((SECONDS + WAIT_SECONDS)) while (( SECONDS < deadline )); do if nodes_are_ready; then return 0 fi sleep 5 done return 1 } wait_for_nodes() { log "Esperando $((AGENT_COUNT + 1)) nodos registrados y Ready" if wait_for_nodes_once; then kubectl get nodes -o wide return 0 fi recover_not_ready_agents if wait_for_nodes_once; then kubectl get nodes -o wide return 0 fi kubectl get nodes -o wide || true docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \ --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' || true die "No se registraron y quedaron Ready los $((AGENT_COUNT + 1)) nodos esperados." } validate_runtime_after_boot() { [[ "$CHECK_INGRESS_AFTER_BOOT" == "true" ]] || return 0 log "Validando Traefik, Argo CD y el puerto publicado" if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then if ! kubectl rollout status deployment/traefik \ -n kube-system \ --timeout="${AGENT_RECOVERY_WAIT}s"; then warn "Traefik no quedó disponible; reiniciando el Deployment." kubectl rollout restart deployment/traefik -n kube-system kubectl rollout status deployment/traefik \ -n kube-system \ --timeout="${WAIT_SECONDS}s" fi fi if kubectl get deployment/argocd-server -n argocd >/dev/null 2>&1; then if ! kubectl rollout status deployment/argocd-server \ -n argocd \ --timeout="${AGENT_RECOVERY_WAIT}s"; then warn "argocd-server no quedó disponible; reiniciando el Deployment." kubectl rollout restart deployment/argocd-server -n argocd kubectl rollout status deployment/argocd-server \ -n argocd \ --timeout="${WAIT_SECONDS}s" fi fi if ! curl -sS \ --max-time 8 \ -o /dev/null \ -H "Host: ${ARGOCD_HOST}" \ "http://127.0.0.1:${HTTP_PORT}/"; then warn "El Ingress no respondió en 127.0.0.1:${HTTP_PORT}; reiniciando serverlb y Traefik." docker restart "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then kubectl rollout restart deployment/traefik -n kube-system kubectl rollout status deployment/traefik \ -n kube-system \ --timeout="${WAIT_SECONDS}s" fi sleep 10 curl -sS \ --max-time 8 \ -o /dev/null \ -H "Host: ${ARGOCD_HOST}" \ "http://127.0.0.1:${HTTP_PORT}/" || die "El Ingress sigue sin responder en el puerto ${HTTP_PORT}." fi ok "Runtime, Traefik e Ingress disponibles." } argocd_rollout_diagnostics() { local resource="$1" local namespace="$2" warn "Diagnóstico del rollout ${resource} en ${namespace}" kubectl get "$resource" -n "$namespace" -o wide || true kubectl get pods -n "$namespace" -o wide || true kubectl get events -n "$namespace" \ --sort-by='.lastTimestamp' 2>/dev/null | tail -60 || true } cleanup_terminating_argocd_server_pods() { local desired available terminating_pods pod desired="$( kubectl get deployment/argocd-server \ -n argocd \ -o jsonpath='{.spec.replicas}' 2>/dev/null || true )" available="$( kubectl get deployment/argocd-server \ -n argocd \ -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true )" desired="${desired:-1}" available="${available:-0}" # Solo se fuerza la eliminación cuando la nueva réplica ya está disponible. if (( available < desired )); then return 1 fi terminating_pods="$( kubectl get pods \ -n argocd \ -l app.kubernetes.io/name=argocd-server \ -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' \ 2>/dev/null || true )" [[ -n "$terminating_pods" ]] || return 1 while IFS= read -r pod; do [[ -n "$pod" ]] || continue warn "Eliminando Pod antiguo de argocd-server atascado en Terminating: ${pod}" kubectl delete pod "$pod" \ -n argocd \ --grace-period=0 \ --force || true done <<< "$terminating_pods" return 0 } wait_argocd_rollout() { local kind="$1" local name="$2" local namespace="${3:-argocd}" local resource="${kind}/${name}" if kubectl rollout status "$resource" \ -n "$namespace" \ --timeout="${ARGOCD_ROLLOUT_TIMEOUT}s"; then return 0 fi argocd_rollout_diagnostics "$resource" "$namespace" if [[ "$resource" == "deployment/argocd-server" ]] && cleanup_terminating_argocd_server_pods; then log "Reintentando rollout de argocd-server después de limpiar la réplica antigua" kubectl rollout status "$resource" \ -n "$namespace" \ --timeout="180s" return fi die "El rollout ${resource} no quedó disponible." } install_argocd() { log "Instalando/reconciliando Argo CD ${ARGOCD_VERSION}" kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - kubectl apply \ --server-side \ --force-conflicts \ -n argocd \ -f "https://raw.githubusercontent.com/argoproj/argo-cd/${ARGOCD_VERSION}/manifests/install.yaml" local current_insecure current_insecure="$( kubectl get configmap argocd-cmd-params-cm \ -n argocd \ -o jsonpath='{.data.server\.insecure}' \ 2>/dev/null || true )" if [[ "$current_insecure" != "true" ]]; then kubectl patch configmap argocd-cmd-params-cm \ -n argocd \ --type merge \ -p '{"data":{"server.insecure":"true"}}' # Reiniciar únicamente cuando la configuración cambió. kubectl rollout restart deployment/argocd-server -n argocd else ok "argocd-server ya está configurado con server.insecure=true; no se reinicia innecesariamente." fi local deployments=( argocd-server argocd-repo-server argocd-redis argocd-dex-server argocd-applicationset-controller argocd-notifications-controller ) local deployment for deployment in "${deployments[@]}"; do if kubectl get deployment "$deployment" -n argocd >/dev/null 2>&1; then wait_argocd_rollout deployment "$deployment" argocd fi done if kubectl get statefulset argocd-application-controller -n argocd >/dev/null 2>&1; then wait_argocd_rollout statefulset argocd-application-controller argocd fi } apply_argocd_ingress() { log "Aplicando Ingress de Argo CD" cat </dev/null 2>&1 && kubectl get configmap "$PORTAINER_AGENT_CONFIGMAP" \ -n "$PORTAINER_NAMESPACE" >/dev/null 2>&1 && kubectl get secret "$PORTAINER_AGENT_SECRET" \ -n "$PORTAINER_NAMESPACE" >/dev/null 2>&1 } capture_portainer_edge_bundle() { [[ "$INSTALL_PORTAINER_EDGE_AGENT" == "true" ]] || return 0 have python3 || die "Se requiere python3 para generar el respaldo declarativo del Portainer Edge Agent." portainer_resources_exist || die "No existe un Portainer Edge Agent completo para adoptar." local temp_dir output_tmp temp_dir="$(mktemp -d)" output_tmp="${PORTAINER_BUNDLE_FILE}.tmp" kubectl get namespace "$PORTAINER_NAMESPACE" -o json \ > "${temp_dir}/00-namespace.json" kubectl get serviceaccount "$PORTAINER_AGENT_SERVICEACCOUNT" \ -n "$PORTAINER_NAMESPACE" -o json \ > "${temp_dir}/10-serviceaccount.json" kubectl get clusterrolebinding "$PORTAINER_AGENT_CLUSTERROLEBINDING" \ -o json \ > "${temp_dir}/20-clusterrolebinding.json" kubectl get service "$PORTAINER_AGENT_SERVICE" \ -n "$PORTAINER_NAMESPACE" -o json \ > "${temp_dir}/30-service.json" kubectl get configmap "$PORTAINER_AGENT_CONFIGMAP" \ -n "$PORTAINER_NAMESPACE" -o json \ > "${temp_dir}/40-configmap.json" kubectl get secret "$PORTAINER_AGENT_SECRET" \ -n "$PORTAINER_NAMESPACE" -o json \ > "${temp_dir}/50-secret.json" kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" -o json \ > "${temp_dir}/60-deployment.json" python3 - "$temp_dir" "$output_tmp" <<'PY' import json import pathlib import sys source = pathlib.Path(sys.argv[1]) destination = pathlib.Path(sys.argv[2]) items = [] for filename in sorted(source.glob("*.json")): obj = json.loads(filename.read_text(encoding="utf-8")) metadata = obj.get("metadata", {}) clean_meta = { "name": metadata["name"], } if metadata.get("namespace"): clean_meta["namespace"] = metadata["namespace"] if metadata.get("labels"): clean_meta["labels"] = metadata["labels"] annotations = { key: value for key, value in metadata.get("annotations", {}).items() if key not in { "kubectl.kubernetes.io/last-applied-configuration", "deployment.kubernetes.io/revision", "argocd.argoproj.io/tracking-id", } } if annotations: clean_meta["annotations"] = annotations clean = { "apiVersion": obj["apiVersion"], "kind": obj["kind"], "metadata": clean_meta, } kind = obj["kind"] if kind == "Namespace": pass elif kind == "Secret": clean["type"] = obj.get("type", "Opaque") if obj.get("data"): clean["data"] = obj["data"] if obj.get("stringData"): clean["stringData"] = obj["stringData"] elif kind == "ConfigMap": if obj.get("data"): clean["data"] = obj["data"] if obj.get("binaryData"): clean["binaryData"] = obj["binaryData"] else: spec = obj.get("spec", {}) if kind == "Service": for key in ( "clusterIP", "clusterIPs", "ipFamilies", "ipFamilyPolicy", "healthCheckNodePort", "allocateLoadBalancerNodePorts", ): spec.pop(key, None) for port in spec.get("ports", []): port.pop("nodePort", None) if kind == "ServiceAccount": spec.pop("secrets", None) clean["spec"] = spec # ClusterRoleBinding no usa spec. if kind == "ClusterRoleBinding": clean.pop("spec", None) clean["roleRef"] = obj["roleRef"] clean["subjects"] = obj["subjects"] if kind == "ServiceAccount": clean.pop("spec", None) if "automountServiceAccountToken" in obj: clean["automountServiceAccountToken"] = obj[ "automountServiceAccountToken" ] if obj.get("imagePullSecrets"): clean["imagePullSecrets"] = obj["imagePullSecrets"] items.append(clean) bundle = { "apiVersion": "v1", "kind": "List", "items": items, } destination.write_text( json.dumps(bundle, indent=2, sort_keys=False) + "\n", encoding="utf-8", ) PY install -m 0600 "$output_tmp" "$PORTAINER_BUNDLE_FILE" rm -rf "$temp_dir" "$output_tmp" ok "Portainer Edge Agent adoptado en ${PORTAINER_BUNDLE_FILE}." warn "El bundle contiene EDGE_KEY; mantenlo con permisos 0600 y fuera de Gitea." } restore_portainer_edge_agent() { [[ -s "$PORTAINER_BUNDLE_FILE" ]] || return 1 log "Restaurando Portainer Edge Agent desde el bundle persistente" kubectl apply -f "$PORTAINER_BUNDLE_FILE" } validate_portainer_edge_agent() { local edge_id insecure actual_image local secret_keys secret_encoded_bytes local desired_replicas ready_replicas available_replicas local pod_count ready_pod_count local configmap_referenced=false secret_referenced=false portainer_resources_exist || return 1 edge_id="$( kubectl get configmap "$PORTAINER_AGENT_CONFIGMAP" \ -n "$PORTAINER_NAMESPACE" \ -o jsonpath='{.data.EDGE_ID}' 2>/dev/null || true )" insecure="$( kubectl get configmap "$PORTAINER_AGENT_CONFIGMAP" \ -n "$PORTAINER_NAMESPACE" \ -o jsonpath='{.data.EDGE_INSECURE_POLL}' 2>/dev/null || true )" actual_image="$( kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" \ -o jsonpath='{.spec.template.spec.containers[0].image}' \ 2>/dev/null || true )" secret_keys="$( kubectl get secret "$PORTAINER_AGENT_SECRET" \ -n "$PORTAINER_NAMESPACE" \ -o go-template='{{range $key, $value := .data}}{{printf "%s\n" $key}}{{end}}' \ 2>/dev/null || true )" secret_encoded_bytes="$( kubectl get secret "$PORTAINER_AGENT_SECRET" \ -n "$PORTAINER_NAMESPACE" \ -o go-template='{{range $key, $value := .data}}{{printf "%d\n" (len $value)}}{{end}}' \ 2>/dev/null | awk '{sum += $1} END {print sum + 0}' )" [[ -n "$edge_id" ]] || die "ConfigMap/${PORTAINER_AGENT_CONFIGMAP} no contiene EDGE_ID." [[ -n "$secret_keys" && "${secret_encoded_bytes:-0}" -gt 0 ]] || die "Secret/${PORTAINER_AGENT_SECRET} no contiene una credencial Edge válida." # Portainer puede cambiar entre env, envFrom, secretKeyRef y volúmenes # según la versión del instalador. La validación no depende de una forma # concreta; solo registra si los objetos aparecen referenciados. if kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" -o json | grep -Fq "\"${PORTAINER_AGENT_CONFIGMAP}\""; then configmap_referenced=true fi if kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" -o json | grep -Fq "\"${PORTAINER_AGENT_SECRET}\""; then secret_referenced=true fi if [[ "$configmap_referenced" == "true" ]]; then ok "Deployment referencia ConfigMap/${PORTAINER_AGENT_CONFIGMAP}." else warn "No se detectó una referencia textual al ConfigMap; se validará el runtime." fi if [[ "$secret_referenced" == "true" ]]; then ok "Deployment referencia Secret/${PORTAINER_AGENT_SECRET}." else warn "No se detectó una referencia textual al Secret; se validará el runtime." fi if [[ "$PORTAINER_EDGE_INSECURE_POLL" == "1" && "$insecure" != "1" ]]; then warn "EDGE_INSECURE_POLL esperado=1; actual=${insecure:-vacío}." fi if [[ -n "$actual_image" && "$actual_image" != "$PORTAINER_AGENT_IMAGE" ]]; then warn "Imagen Portainer Agent actual=${actual_image}; referencia=${PORTAINER_AGENT_IMAGE}." fi kubectl rollout status "deployment/${PORTAINER_AGENT_DEPLOYMENT}" \ -n "$PORTAINER_NAMESPACE" \ --timeout="${PORTAINER_ROLLOUT_TIMEOUT}s" desired_replicas="$( kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" \ -o jsonpath='{.spec.replicas}' )" ready_replicas="$( kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" \ -o jsonpath='{.status.readyReplicas}' )" available_replicas="$( kubectl get deployment "$PORTAINER_AGENT_DEPLOYMENT" \ -n "$PORTAINER_NAMESPACE" \ -o jsonpath='{.status.availableReplicas}' )" pod_count="$( kubectl get pods \ -n "$PORTAINER_NAMESPACE" \ -l app=portainer-agent \ --no-headers 2>/dev/null | wc -l | tr -d ' ' )" ready_pod_count="$( kubectl get pods \ -n "$PORTAINER_NAMESPACE" \ -l app=portainer-agent \ -o go-template='{{range .items}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{printf "1\n"}}{{end}}{{end}}{{end}}' \ 2>/dev/null | wc -l | tr -d ' ' )" [[ "${desired_replicas:-0}" -gt 0 ]] || die "Deployment/${PORTAINER_AGENT_DEPLOYMENT} tiene réplicas deseadas inválidas." [[ "${ready_replicas:-0}" -eq "$desired_replicas" ]] || die "Portainer Agent Ready=${ready_replicas:-0}; esperado=${desired_replicas}." [[ "${available_replicas:-0}" -eq "$desired_replicas" ]] || die "Portainer Agent Available=${available_replicas:-0}; esperado=${desired_replicas}." # Algunos manifiestos cambian las etiquetas. El rollout es la fuente # principal; el conteo por selector se usa como información adicional. if [[ "${pod_count:-0}" -gt 0 ]]; then printf 'Pods Portainer detectados : %s\n' "$pod_count" printf 'Pods Portainer Ready : %s\n' "$ready_pod_count" fi ok "Portainer Edge Agent disponible en namespace ${PORTAINER_NAMESPACE}." } ensure_portainer_edge_agent() { [[ "$INSTALL_PORTAINER_EDGE_AGENT" == "true" ]] || return 0 validate_portainer_server if ! portainer_resources_exist; then if ! restore_portainer_edge_agent; then portainer_fail_or_warn \ "No existe Portainer Edge Agent ni bundle persistente. Ejecuta el comando generado por Portainer y luego: $0 portainer-adopt" return fi fi if ! validate_portainer_edge_agent; then portainer_fail_or_warn "Portainer Edge Agent no quedó disponible." return fi if [[ "$PORTAINER_CAPTURE_ON_SUCCESS" == "true" && ! -s "$PORTAINER_BUNDLE_FILE" ]]; then capture_portainer_edge_bundle fi } portainer_adopt_mode() { require_commands ensure_docker validate_data_mount validate_host_ip prepare_directories cluster_exists || die "No existe ${CLUSTER_NAME}." write_kubeconfig wait_for_api validate_portainer_server validate_portainer_edge_agent capture_portainer_edge_bundle } headlamp_git_references() { local found=0 path if [[ -d "${APP_REGISTRY_CACHE_DIR}/workloads/headlamp" ]]; then printf '%s\n' "${APP_REGISTRY_CACHE_DIR}/workloads/headlamp" found=1 fi if [[ -d "$APP_REGISTRY_CACHE_DIR" ]]; then while IFS= read -r path; do printf '%s\n' "$path" found=1 done < <( grep -RIl \ --exclude-dir=.git \ --include='*.yaml' \ --include='*.yml' \ -E 'name:[[:space:]]*headlamp-app([[:space:]]|$)' \ "$APP_REGISTRY_CACHE_DIR" 2>/dev/null || true ) fi if [[ -d "$PLATFORM_INFRA_CACHE_DIR" ]]; then while IFS= read -r path; do printf '%s\n' "$path" found=1 done < <( grep -RIl \ --exclude-dir=.git \ --include='*.yaml' \ --include='*.yml' \ -E 'name:[[:space:]]*headlamp-governance([[:space:]]|$)' \ "$PLATFORM_INFRA_CACHE_DIR" 2>/dev/null || true ) while IFS= read -r path; do printf '%s\n' "$path" found=1 done < <( find "$PLATFORM_INFRA_CACHE_DIR" \ -path '*/.git' -prune -o \ -type d -name 'headlamp-governance' -print \ 2>/dev/null || true ) fi return "$found" } retire_headlamp_mode() { require_commands ensure_docker validate_data_mount validate_host_ip prepare_directories cluster_exists || die "No existe ${CLUSTER_NAME}." write_kubeconfig wait_for_api sync_gitops_repositories local references references="$(headlamp_git_references || true)" if [[ -n "$references" ]]; then printf '%s\n' "$references" >&2 die "Headlamp todavía está declarado en Gitea. Elimina esas rutas, haz commit/push y repite retire-headlamp." fi log "Retirando residuos de Headlamp" kubectl delete application \ headlamp-app \ headlamp-governance \ -n argocd \ --ignore-not-found=true \ --wait=false kubectl delete clusterrolebinding \ headlamp-admin-binding \ --ignore-not-found=true kubectl delete namespace headlamp \ --ignore-not-found=true \ --wait=false ok "Headlamp retirado. Portainer queda como consola Kubernetes." } apply_postdeploy_rbac() { [[ "$APPLY_POSTDEPLOY_RBAC" == "true" ]] || return 0 log "Aplicando RBAC de solo lectura para validación postdeploy" kubectl create namespace "$POSTDEPLOY_NAMESPACE" \ --dry-run=client \ -o yaml | kubectl apply -f - cat </dev/null 2>&1; then secret_sa_uid="$( kubectl get secret "$POSTDEPLOY_TOKEN_SECRET" \ -n "$POSTDEPLOY_NAMESPACE" \ -o jsonpath='{.metadata.annotations.kubernetes\.io/service-account\.uid}' \ 2>/dev/null || true )" if [[ -n "$secret_sa_uid" && "$secret_sa_uid" != "$sa_uid" ]]; then warn "El token postdeploy pertenece a un ServiceAccount anterior; recreándolo." kubectl delete secret "$POSTDEPLOY_TOKEN_SECRET" \ -n "$POSTDEPLOY_NAMESPACE" fi fi if ! kubectl get secret "$POSTDEPLOY_TOKEN_SECRET" \ -n "$POSTDEPLOY_NAMESPACE" >/dev/null 2>&1; then cat </dev/null || true )" ca_b64="$( kubectl get secret "$POSTDEPLOY_TOKEN_SECRET" \ -n "$POSTDEPLOY_NAMESPACE" \ -o jsonpath='{.data.ca\.crt}' 2>/dev/null || true )" if [[ -n "$token_b64" && -n "$ca_b64" ]]; then break fi sleep 2 done [[ -n "$token_b64" ]] || die "El controlador no generó el token en Secret/${POSTDEPLOY_TOKEN_SECRET}." [[ -n "$ca_b64" ]] || die "El controlador no generó ca.crt en Secret/${POSTDEPLOY_TOKEN_SECRET}." token="$( printf '%s' "$token_b64" | base64 --decode )" [[ -n "$token" ]] || die "El token postdeploy quedó vacío." printf '%s\n' "$token" > "${tmp_dir}/K8S_TOKEN.txt" printf '%s\n' "$ca_b64" > "${tmp_dir}/K8S_CA_B64.txt" printf '%s\n' "$server" > "${tmp_dir}/K8S_SERVER.txt" printf '%s\n' "$LAB_HOST_IP" > "${tmp_dir}/K8S_TLS_SERVER_NAME.txt" cat > "$kubeconfig_file" < "${tmp_dir}/KUBE_CONFIG_DATA.txt" printf '\n' >> "${tmp_dir}/KUBE_CONFIG_DATA.txt" for output_file in \ K8S_TOKEN.txt \ K8S_CA_B64.txt \ K8S_SERVER.txt \ K8S_TLS_SERVER_NAME.txt \ KUBE_CONFIG_DATA.txt do sudo_cmd install \ -m 0600 \ -o "$owner" \ "${tmp_dir}/${output_file}" \ "${secret_dir}/${output_file}" done sudo_cmd install \ -m 0600 \ -o "$owner" \ "$kubeconfig_file" \ "${secret_dir}/postdeploy-validator.kubeconfig" rm -rf "$tmp_dir" warn "Credenciales postdeploy regeneradas en ${secret_dir}." warn "Actualiza en Gitea el secret KUBE_CONFIG_DATA con KUBE_CONFIG_DATA.txt." ok "Token persistente asociado a Secret/${POSTDEPLOY_TOKEN_SECRET}." } validate_postdeploy_credentials() { local secret_dir="${APP_DATA_PATH}/gitea-actions-secrets" local expected_server="https://${LAB_HOST_IP}:${API_PORT}" local server_file="${secret_dir}/K8S_SERVER.txt" local tls_file="${secret_dir}/K8S_TLS_SERVER_NAME.txt" local kubeconfig_file="${secret_dir}/postdeploy-validator.kubeconfig" local actual_server actual_tls kube_server kube_tls can_read [[ -s "$server_file" ]] || die "No existe ${server_file}." [[ -s "$tls_file" ]] || die "No existe ${tls_file}." [[ -s "$kubeconfig_file" ]] || die "No existe ${kubeconfig_file}." actual_server="$(tr -d '\r\n\t ' < "$server_file")" actual_tls="$(tr -d '\r\n\t ' < "$tls_file")" [[ "$actual_server" == "$expected_server" ]] || die "K8S_SERVER incorrecto. Actual=${actual_server}; esperado=${expected_server}." [[ "$actual_tls" == "$LAB_HOST_IP" ]] || die "K8S_TLS_SERVER_NAME incorrecto. Actual=${actual_tls}; esperado=${LAB_HOST_IP}." kube_server="$( KUBECONFIG="$kubeconfig_file" \ kubectl config view \ --minify \ -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true )" kube_tls="$( KUBECONFIG="$kubeconfig_file" \ kubectl config view \ --minify \ -o jsonpath='{.clusters[0].cluster.tls-server-name}' 2>/dev/null || true )" [[ "$kube_server" == "$expected_server" ]] || die "El kubeconfig postdeploy apunta a ${kube_server:-vacío}; esperado=${expected_server}." [[ "$kube_tls" == "$LAB_HOST_IP" ]] || die "El kubeconfig postdeploy usa tls-server-name=${kube_tls:-vacío}; esperado=${LAB_HOST_IP}." can_read="$( KUBECONFIG="$kubeconfig_file" \ kubectl auth can-i get deployments \ -n "$POSTDEPLOY_NAMESPACE" 2>/dev/null || true )" [[ "$can_read" == "yes" ]] || die "El ServiceAccount postdeploy no puede leer Deployments en ecommerce." KUBECONFIG="$kubeconfig_file" \ kubectl get deployments \ -n "$POSTDEPLOY_NAMESPACE" \ --request-timeout=15s \ >/dev/null ok "Credenciales postdeploy verificadas contra ${expected_server}." printf 'K8S_SERVER esperado : %s\n' "$expected_server" printf 'K8S_TLS_SERVER_NAME esperado : %s\n' "$LAB_HOST_IP" printf 'Archivos para Gitea : %s\n' "$secret_dir" } read_optional_secret() { local variable_name="$1" local file_path="$2" local value="${!variable_name:-}" if [[ -n "$value" ]]; then printf '%s' "$value" return 0 fi if [[ -s "$file_path" ]]; then tr -d '\r\n' < "$file_path" return 0 fi return 1 } git_with_optional_credentials() { local username="" password="" askpass_dir="" askpass_script="" result=0 username="$( read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true )" password="$( read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true )" if [[ -z "$username" && -z "$password" ]]; then GIT_TERMINAL_PROMPT=0 git "$@" return fi [[ -n "$username" && -n "$password" ]] || die "El clon privado requiere usuario y token/password completos." askpass_dir="$(mktemp -d)" askpass_script="${askpass_dir}/askpass.sh" cat > "$askpass_script" <<'ASKPASS' #!/bin/sh case "$1" in *sername*) printf '%s\n' "$GIT_BOOTSTRAP_USERNAME" ;; *assword*) printf '%s\n' "$GIT_BOOTSTRAP_PASSWORD" ;; *) printf '\n' ;; esac ASKPASS chmod 0700 "$askpass_script" GIT_TERMINAL_PROMPT=0 \ GIT_ASKPASS="$askpass_script" \ GIT_BOOTSTRAP_USERNAME="$username" \ GIT_BOOTSTRAP_PASSWORD="$password" \ git "$@" || result=$? rm -rf "$askpass_dir" return "$result" } apps_registry_cache_valid() { [[ -d "${APP_REGISTRY_CACHE_DIR}/.git" ]] && git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 } record_apps_registry_commit() { local commit commit="$( git -C "$APP_REGISTRY_CACHE_DIR" rev-parse HEAD 2>/dev/null || true )" [[ -n "$commit" ]] || return 0 printf '%s\n' "$commit" > "$APP_REGISTRY_COMMIT_FILE" chmod 0600 "$APP_REGISTRY_COMMIT_FILE" ok "apps-registry cacheado en commit ${commit:0:12}." } find_application_manifest_by_name() { local app_name="$1" local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" local manifest="" manifest_name="" while IFS= read -r manifest; do manifest_name="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.metadata.name}' 2>/dev/null || true )" if [[ "$manifest_name" == "$app_name" ]]; then printf '%s\n' "$manifest" return 0 fi done < <( find "$apps_dir" -maxdepth 1 -type f \ \( -name '*.yaml' -o -name '*.yml' \) | sort ) return 1 } validate_monitoring_app_contract() { local manifest="" revision="" ttl="" ingress_enabled="" ingress_host="" local request_cpu="" request_memory="" limit_cpu="" limit_memory="" local parameters_dump="" manifest="$(find_application_manifest_by_name "$MONITORING_APP_NAME" || true)" [[ -n "$manifest" ]] || die "No se encontró Application/${MONITORING_APP_NAME} en ${ROOT_APP_REPO_PATH}/." revision="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true )" [[ -n "$revision" ]] || die "${manifest} no contiene spec.source.targetRevision." if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]]; then die "${MONITORING_APP_NAME} usa ${revision}. Usa ${MONITORING_FIXED_CHART_VERSION} o superior." fi ttl="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished}' \ 2>/dev/null || true )" request_cpu="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.cpu}' \ 2>/dev/null || true )" request_memory="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.memory}' \ 2>/dev/null || true )" limit_cpu="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.cpu}' \ 2>/dev/null || true )" limit_memory="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.memory}' \ 2>/dev/null || true )" parameters_dump="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{range .spec.source.helm.parameters[*]}{.name}={.value}{"\n"}{end}' \ 2>/dev/null || true )" if [[ -z "$ttl" ]] && ! grep -q '^prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60$' \ <<< "$parameters_dump"; then die "${MONITORING_APP_NAME} debe configurar ttlSecondsAfterFinished=60." fi [[ -n "$request_cpu" && -n "$request_memory" && -n "$limit_cpu" && -n "$limit_memory" ]] || die "${MONITORING_APP_NAME} debe definir requests/limits del Job admission." ingress_enabled="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.enabled}' \ 2>/dev/null || true )" ingress_host="$( kubectl create --dry-run=client -f "$manifest" \ -o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.hosts[0]}' \ 2>/dev/null || true )" [[ "$ingress_enabled" == "true" ]] || die "${MONITORING_APP_NAME} debe habilitar grafana.ingress.enabled=true." [[ "$ingress_host" == "$MONITORING_INGRESS_HOST" ]] || die "Grafana debe publicarse en ${MONITORING_INGRESS_HOST}; actual=${ingress_host:-vacío}." ok "Contrato monitoring válido: chart=${revision}, TTL, recursos admission e Ingress Grafana." } validate_apps_registry_cache() { apps_registry_cache_valid || die "El caché ${APP_REGISTRY_CACHE_DIR} no es un repositorio Git válido." [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]] || die "No existe ${APP_REGISTRY_ROOT_MANIFEST} en el repositorio apps-registry." [[ -d "${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" ]] || die "No existe la ruta ${ROOT_APP_REPO_PATH}/ en el repositorio apps-registry." kubectl apply \ --dry-run=client \ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \ >/dev/null local manifest_name manifest_namespace manifest_name="$( kubectl create \ --dry-run=client \ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \ -o jsonpath='{.metadata.name}' 2>/dev/null || true )" manifest_namespace="$( kubectl create \ --dry-run=client \ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \ -o jsonpath='{.metadata.namespace}' 2>/dev/null || true )" [[ "$manifest_name" == "$ROOT_APP_NAME" ]] || die "${APP_REGISTRY_ROOT_MANIFEST} debe crear metadata.name=${ROOT_APP_NAME}; actual=${manifest_name:-vacío}." [[ "$manifest_namespace" == "argocd" ]] || die "${APP_REGISTRY_ROOT_MANIFEST} debe usar namespace argocd; actual=${manifest_namespace:-vacío}." validate_monitoring_app_contract } sync_apps_registry_repo() { local update_ok=true current_origin="" log "Sincronizando caché local de apps-registry" install -d -m 0750 "$(dirname "$APP_REGISTRY_CACHE_DIR")" if apps_registry_cache_valid; then current_origin="$( git -C "$APP_REGISTRY_CACHE_DIR" remote get-url origin 2>/dev/null || true )" if [[ "$current_origin" != "$ROOT_APP_REPO_URL" ]]; then warn "Actualizando origin: ${current_origin:-ausente} -> ${ROOT_APP_REPO_URL}" git -C "$APP_REGISTRY_CACHE_DIR" remote set-url origin "$ROOT_APP_REPO_URL" fi if ! git_with_optional_credentials \ -C "$APP_REGISTRY_CACHE_DIR" \ fetch \ --prune \ --depth="$APP_REGISTRY_CLONE_DEPTH" \ origin \ "$ROOT_APP_REVISION"; then update_ok=false else git -C "$APP_REGISTRY_CACHE_DIR" \ checkout -B "$ROOT_APP_REVISION" "origin/${ROOT_APP_REVISION}" git -C "$APP_REGISTRY_CACHE_DIR" \ reset --hard "origin/${ROOT_APP_REVISION}" git -C "$APP_REGISTRY_CACHE_DIR" clean -fd fi else rm -rf "$APP_REGISTRY_CACHE_DIR" if ! git_with_optional_credentials \ clone \ --branch "$ROOT_APP_REVISION" \ --single-branch \ --depth "$APP_REGISTRY_CLONE_DEPTH" \ "$ROOT_APP_REPO_URL" \ "$APP_REGISTRY_CACHE_DIR"; then update_ok=false fi fi if [[ "$update_ok" != "true" ]]; then if apps_registry_cache_valid && [[ "$APP_REGISTRY_ALLOW_CACHE_FALLBACK" == "true" ]]; then warn "Gitea no estuvo disponible; se utilizará el último clon válido de apps-registry." else die "No fue posible clonar/actualizar ${ROOT_APP_REPO_URL} y no hay un caché válido." fi fi validate_apps_registry_cache record_apps_registry_commit } platform_infra_cache_valid() { [[ -d "${PLATFORM_INFRA_CACHE_DIR}/.git" ]] && git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 } record_platform_infra_commit() { local commit commit="$( git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD 2>/dev/null || true )" [[ -n "$commit" ]] || return 0 printf '%s\n' "$commit" > "$PLATFORM_INFRA_COMMIT_FILE" chmod 0600 "$PLATFORM_INFRA_COMMIT_FILE" ok "platform-infra cacheado en commit ${commit:0:12}." } sync_platform_infra_repo() { local update_ok=true current_origin="" log "Sincronizando caché local de platform-infra" install -d -m 0750 "$(dirname "$PLATFORM_INFRA_CACHE_DIR")" if platform_infra_cache_valid; then current_origin="$( git -C "$PLATFORM_INFRA_CACHE_DIR" remote get-url origin 2>/dev/null || true )" if [[ "$current_origin" != "$PLATFORM_INFRA_REPO_URL" ]]; then warn "Actualizando origin platform-infra: ${current_origin:-ausente} -> ${PLATFORM_INFRA_REPO_URL}" git -C "$PLATFORM_INFRA_CACHE_DIR" remote set-url origin "$PLATFORM_INFRA_REPO_URL" fi if ! git_with_optional_credentials \ -C "$PLATFORM_INFRA_CACHE_DIR" \ fetch \ --prune \ --depth="$PLATFORM_INFRA_CLONE_DEPTH" \ origin \ "$PLATFORM_INFRA_REVISION"; then update_ok=false else git -C "$PLATFORM_INFRA_CACHE_DIR" \ checkout -B "$PLATFORM_INFRA_REVISION" "origin/${PLATFORM_INFRA_REVISION}" git -C "$PLATFORM_INFRA_CACHE_DIR" \ reset --hard "origin/${PLATFORM_INFRA_REVISION}" git -C "$PLATFORM_INFRA_CACHE_DIR" clean -fd fi else rm -rf "$PLATFORM_INFRA_CACHE_DIR" if ! git_with_optional_credentials \ clone \ --branch "$PLATFORM_INFRA_REVISION" \ --single-branch \ --depth "$PLATFORM_INFRA_CLONE_DEPTH" \ "$PLATFORM_INFRA_REPO_URL" \ "$PLATFORM_INFRA_CACHE_DIR"; then update_ok=false fi fi if [[ "$update_ok" != "true" ]]; then if platform_infra_cache_valid && [[ "$PLATFORM_INFRA_ALLOW_CACHE_FALLBACK" == "true" ]]; then warn "Gitea no estuvo disponible; se utilizará el último clon válido de platform-infra." else die "No fue posible clonar/actualizar ${PLATFORM_INFRA_REPO_URL} y no hay un caché válido." fi fi record_platform_infra_commit } governance_applications_tsv() { local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" local manifest="" kind="" app_name="" repo_url="" source_path="" revision="" namespace="" [[ -d "$apps_dir" ]] || die "No existe ${apps_dir}." while IFS= read -r manifest; do kind="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.kind}' 2>/dev/null || true )" [[ "$kind" == "Application" ]] || continue repo_url="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.spec.source.repoURL}' 2>/dev/null || true )" [[ "$repo_url" == "$PLATFORM_INFRA_REPO_URL" ]] || continue app_name="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.metadata.name}' 2>/dev/null || true )" source_path="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.spec.source.path}' 2>/dev/null || true )" revision="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true )" namespace="$( kubectl create --dry-run=client \ -f "$manifest" \ -o jsonpath='{.spec.destination.namespace}' 2>/dev/null || true )" [[ -n "$app_name" && -n "$source_path" ]] || die "Application governance inválida: ${manifest}" printf '%s\t%s\t%s\t%s\n' \ "$app_name" \ "$source_path" \ "${revision:-$PLATFORM_INFRA_REVISION}" \ "${namespace:-default}" done < <( find "$apps_dir" \ -maxdepth 1 \ -type f \ \( -name '*.yaml' -o -name '*.yml' \) \ | sort ) } validate_monitoring_governance_contract() { local rendered_file="${PLATFORM_INFRA_RENDER_DIR}/monitoring-governance.yaml" [[ -f "$rendered_file" ]] || return 0 if grep -Eq '^kind:[[:space:]]*LimitRange[[:space:]]*$' "$rendered_file"; then ok "monitoring-governance incluye LimitRange." return 0 fi if [[ "$ENFORCE_MONITORING_LIMITRANGE" == "true" ]]; then die "monitoring-governance no incluye LimitRange. Agrega limitrange.yaml y decláralo en kustomization.yaml para que ResourceQuota no rechace contenedores sin requests/limits." fi warn "monitoring-governance no incluye LimitRange; monitoring puede fallar por ResourceQuota." } validate_and_render_platform_infra() { platform_infra_cache_valid || die "El caché ${PLATFORM_INFRA_CACHE_DIR} no es un repositorio Git válido." install -d -m 0750 "$PLATFORM_INFRA_RENDER_DIR" local app_name="" source_path="" revision="" namespace="" local source_dir="" render_file="" count=0 while IFS=$'\t' read -r app_name source_path revision namespace; do [[ -n "$app_name" ]] || continue count=$((count + 1)) source_dir="${PLATFORM_INFRA_CACHE_DIR}/${source_path}" render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml" [[ -d "$source_dir" ]] || die "${app_name} referencia una ruta inexistente: ${source_dir}" if [[ "$revision" != "$PLATFORM_INFRA_REVISION" ]]; then warn "${app_name} usa targetRevision=${revision}; el caché local está en ${PLATFORM_INFRA_REVISION}." fi kubectl kustomize "$source_dir" > "${render_file}.tmp" [[ -s "${render_file}.tmp" ]] || die "Kustomize no generó recursos para ${app_name} (${source_path})." { printf '# Source: %s\n' "$PLATFORM_INFRA_REPO_URL" printf '# Revision: %s\n' "$PLATFORM_INFRA_REVISION" printf '# Commit: %s\n' \ "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD)" printf '# Application: %s\n' "$app_name" cat "${render_file}.tmp" } > "$render_file" rm -f "${render_file}.tmp" chmod 0600 "$render_file" ok "Kustomize validado: ${app_name} -> ${source_path}" done < <(governance_applications_tsv) (( count > 0 )) || die "No se encontraron Applications que referencien ${PLATFORM_INFRA_REPO_URL}." validate_monitoring_governance_contract } apply_governance_snapshots() { [[ "$APPLY_GOVERNANCE_SNAPSHOTS" == "true" ]] || return 0 log "Aplicando snapshots governance antes de los workloads" local app_name="" source_path="" revision="" namespace="" local render_file="" while IFS=$'\t' read -r app_name source_path revision namespace; do [[ -n "$app_name" ]] || continue kubectl create namespace "$namespace" \ --dry-run=client \ -o yaml | kubectl apply -f - >/dev/null render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml" [[ -s "$render_file" ]] || die "No existe snapshot renderizado para ${app_name}: ${render_file}" kubectl apply \ --server-side \ --force-conflicts \ --field-manager=deploy-k3d-lab-governance \ -f "$render_file" done < <(governance_applications_tsv) ok "Governance aplicada desde snapshots Kustomize." } sync_gitops_repositories() { sync_apps_registry_repo sync_platform_infra_repo validate_and_render_platform_infra } root_application_source_manifest() { local repo_manifest="${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" if [[ -f "$repo_manifest" ]]; then printf '%s\n' "$repo_manifest" else printf '%s\n' "$ROOT_APP_MANIFEST" fi } configure_single_argocd_repository() { local secret_name="$1" local repo_url="$2" local repo_username="$3" local repo_password="$4" kubectl create secret generic "$secret_name" \ -n argocd \ --from-literal=type=git \ --from-literal=url="$repo_url" \ --from-literal=username="$repo_username" \ --from-literal=password="$repo_password" \ --dry-run=client \ -o yaml | kubectl label \ --local -f - \ 'argocd.argoproj.io/secret-type=repository' \ -o yaml | kubectl apply -f - } configure_argocd_repository() { [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 local repo_username="" repo_password="" repo_username="$( read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true )" repo_password="$( read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true )" if [[ -z "$repo_username" && -z "$repo_password" ]]; then warn "No se configuraron credenciales HTTPS; se asumirá que apps-registry y platform-infra son públicos o que Argo CD ya tiene credenciales." return 0 fi [[ -n "$repo_username" && -n "$repo_password" ]] || die "Para repositorios privados se requieren usuario y token/password completos." log "Registrando repositorios Git privados en Argo CD" configure_single_argocd_repository \ "$ARGOCD_REPO_SECRET_NAME" \ "$ROOT_APP_REPO_URL" \ "$repo_username" \ "$repo_password" configure_single_argocd_repository \ "$ARGOCD_PLATFORM_REPO_SECRET_NAME" \ "$PLATFORM_INFRA_REPO_URL" \ "$repo_username" \ "$repo_password" ok "apps-registry y platform-infra configurados en Argo CD." } write_root_application_manifest() { [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 if [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]]; then ok "Application raíz obtenida del clon: ${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" return 0 fi warn "No se encontró ${APP_REGISTRY_ROOT_MANIFEST} en el clon; generando manifiesto fallback." cat > "$ROOT_APP_MANIFEST" </dev/null ok "Application raíz aplicada." } root_application_diagnostics() { kubectl get application "$ROOT_APP_NAME" \ -n argocd \ -o wide 2>/dev/null || true kubectl get application "$ROOT_APP_NAME" \ -n argocd \ -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \ 2>/dev/null || true kubectl get pods -n argocd -o wide || true } wait_for_root_application() { [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 log "Esperando sincronización de ${ROOT_APP_NAME}" local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS)) local sync_status="" health_status="" operation_phase="" while (( SECONDS < deadline )); do sync_status="$( kubectl get application "$ROOT_APP_NAME" \ -n argocd \ -o jsonpath='{.status.sync.status}' 2>/dev/null || true )" health_status="$( kubectl get application "$ROOT_APP_NAME" \ -n argocd \ -o jsonpath='{.status.health.status}' 2>/dev/null || true )" operation_phase="$( kubectl get application "$ROOT_APP_NAME" \ -n argocd \ -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true )" if [[ "$operation_phase" == "Error" || "$operation_phase" == "Failed" ]]; then root_application_diagnostics die "${ROOT_APP_NAME} terminó con operationState=${operation_phase}." fi if [[ "$sync_status" == "Synced" ]]; then ok "${ROOT_APP_NAME}: Synced / ${health_status:-Unknown}." return 0 fi sleep 5 done root_application_diagnostics die "${ROOT_APP_NAME} no quedó Synced dentro de ${ROOT_APP_WAIT_SECONDS}s." } discover_expected_gitops_apps() { local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" local manifest="" kind="" app_name="" EXPECTED_APPS_ARRAY=() [[ -d "$apps_dir" ]] || die "No existe el directorio de Applications: ${apps_dir}" while IFS= read -r manifest; do [[ -n "$manifest" ]] || continue kind="$( kubectl create \ --dry-run=client \ -f "$manifest" \ -o jsonpath='{.kind}' 2>/dev/null || true )" app_name="$( kubectl create \ --dry-run=client \ -f "$manifest" \ -o jsonpath='{.metadata.name}' 2>/dev/null || true )" if [[ "$kind" == "Application" && -n "$app_name" && "$app_name" != "$ROOT_APP_NAME" ]]; then EXPECTED_APPS_ARRAY+=("$app_name") fi done < <( find "$apps_dir" \ -maxdepth 1 \ -type f \ \( -name '*.yaml' -o -name '*.yml' \) \ | sort ) (( ${#EXPECTED_APPS_ARRAY[@]} > 0 )) || die "No se descubrieron recursos kind: Application dentro de ${apps_dir}." } expected_gitops_apps_array() { local normalized="" if [[ -n "${EXPECTED_GITOPS_APPS//[[:space:],]/}" ]]; then normalized="${EXPECTED_GITOPS_APPS//,/ }" read -r -a EXPECTED_APPS_ARRAY <<< "$normalized" else discover_expected_gitops_apps fi ok "Applications esperadas: ${EXPECTED_APPS_ARRAY[*]}" } wait_for_expected_gitops_apps() { [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 expected_gitops_apps_array log "Esperando Applications hijas del repositorio" local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS)) local app="" missing_count=0 while (( SECONDS < deadline )); do missing_count=0 for app in "${EXPECTED_APPS_ARRAY[@]}"; do [[ -n "$app" ]] || continue kubectl get application "$app" -n argocd >/dev/null 2>&1 || missing_count=$((missing_count + 1)) done if (( missing_count == 0 )); then ok "Se crearon todas las Applications esperadas." return 0 fi sleep 5 done warn "Applications encontradas:" kubectl get applications.argoproj.io -n argocd || true die "Faltan ${missing_count} Applications esperadas después de aplicar ${ROOT_APP_NAME}." } wait_for_gitops_apps_health() { [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 expected_gitops_apps_array log "Esperando que las Applications hijas se estabilicen" local deadline=$((SECONDS + GITOPS_APPS_WAIT_SECONDS)) local app="" sync_status="" health_status="" pending=0 while (( SECONDS < deadline )); do pending=0 for app in "${EXPECTED_APPS_ARRAY[@]}"; do [[ -n "$app" ]] || continue sync_status="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.status.sync.status}' 2>/dev/null || true )" health_status="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.status.health.status}' 2>/dev/null || true )" if [[ "$sync_status" != "Synced" || "$health_status" != "Healthy" ]]; then pending=$((pending + 1)) fi done if (( pending == 0 )); then ok "Todas las Applications esperadas están Synced/Healthy." return 0 fi sleep 10 done kubectl get applications.argoproj.io -n argocd || true if [[ "$GITOPS_STRICT_HEALTH" == "true" ]]; then die "${pending} Applications no quedaron Synced/Healthy dentro de ${GITOPS_APPS_WAIT_SECONDS}s." fi warn "${pending} Applications siguen Progressing, OutOfSync o Degraded; Argo CD continuará reconciliándolas." } monitoring_app_diagnostics() { local app="$MONITORING_APP_NAME" local namespace="$MONITORING_NAMESPACE" local revision="" sync_status="" health_status="" phase="" message="" log "Diagnóstico de ${app}" kubectl get application "$app" -n argocd -o wide || { die "No existe Application/${app} en argocd." } revision="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true )" sync_status="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.status.sync.status}' 2>/dev/null || true )" health_status="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.status.health.status}' 2>/dev/null || true )" phase="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true )" message="$( kubectl get application "$app" \ -n argocd \ -o jsonpath='{.status.operationState.message}' 2>/dev/null || true )" printf 'Chart revision : %s\n' "${revision:-desconocida}" printf 'Sync : %s\n' "${sync_status:-desconocido}" printf 'Health : %s\n' "${health_status:-desconocido}" printf 'Operation : %s\n' "${phase:-ninguna}" printf 'Message : %s\n' "${message:-sin mensaje}" echo echo "=== Jobs/Pods de admission webhooks ===" kubectl get jobs,pods \ -n "$namespace" \ -o wide 2>/dev/null | grep -E 'NAME|admission-(create|patch)' || true echo echo "=== Eventos recientes de monitoring ===" kubectl get events \ -n "$namespace" \ --sort-by='.lastTimestamp' 2>/dev/null | tail -60 || true echo echo "=== Condiciones de Argo CD ===" kubectl get application "$app" \ -n argocd \ -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \ 2>/dev/null || true if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]] && [[ "$message" == *"admission-create"* || "$sync_status" == "OutOfSync" ]]; then warn "Se detectó kube-prometheus-stack ${revision}, afectado por la carrera de ttlSecondsAfterFinished: 0 con hooks de Argo CD." warn "Actualiza apps/monitoring-app.yaml al menos a ${MONITORING_FIXED_CHART_VERSION} y configura prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60." fi } argocd_core_cli() { local temp_kubeconfig result=0 temp_kubeconfig="$(mktemp)" cp "$KUBECONFIG_PATH" "$temp_kubeconfig" # Argo CD --core obtiene el namespace de control desde el contexto actual # del kubeconfig. Sin esto busca argocd-cm en "default". KUBECONFIG="$temp_kubeconfig" \ kubectl config set-context \ --current \ --namespace=argocd \ >/dev/null if have argocd; then KUBECONFIG="$temp_kubeconfig" \ argocd "$@" --core || result=$? else docker run --rm \ --network host \ -e KUBECONFIG=/kubeconfig \ -v "${temp_kubeconfig}:/kubeconfig:ro" \ --entrypoint /usr/local/bin/argocd \ "$ARGOCD_CLI_IMAGE" \ "$@" \ --core || result=$? fi rm -f "$temp_kubeconfig" return "$result" } monitoring_operation_fields() { local app="$MONITORING_APP_NAME" MONITORING_TARGET_REVISION="$( kubectl get application "$app" -n argocd \ -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true )" MONITORING_OPERATION_REVISION="$( kubectl get application "$app" -n argocd \ -o jsonpath='{.status.operationState.syncResult.revision}' 2>/dev/null || true )" MONITORING_OPERATION_PHASE="$( kubectl get application "$app" -n argocd \ -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true )" MONITORING_OPERATION_MESSAGE="$( kubectl get application "$app" -n argocd \ -o jsonpath='{.status.operationState.message}' 2>/dev/null || true )" MONITORING_SYNC_STATUS="$( kubectl get application "$app" -n argocd \ -o jsonpath='{.status.sync.status}' 2>/dev/null || true )" MONITORING_HEALTH_STATUS="$( kubectl get application "$app" -n argocd \ -o jsonpath='{.status.health.status}' 2>/dev/null || true )" export MONITORING_TARGET_REVISION MONITORING_OPERATION_REVISION \ MONITORING_OPERATION_PHASE MONITORING_OPERATION_MESSAGE \ MONITORING_SYNC_STATUS MONITORING_HEALTH_STATUS } monitoring_operation_is_stale() { monitoring_operation_fields [[ "$MONITORING_OPERATION_PHASE" == "Running" ]] || return 1 [[ "$MONITORING_FORCE_RECOVERY" == "true" ]] && return 0 [[ -n "$MONITORING_OPERATION_REVISION" && -n "$MONITORING_TARGET_REVISION" && "$MONITORING_OPERATION_REVISION" != "$MONITORING_TARGET_REVISION" ]] } wait_monitoring_operation_stopped() { local deadline=$((SECONDS + MONITORING_OPERATION_STOP_WAIT)) while (( SECONDS < deadline )); do monitoring_operation_fields if [[ "$MONITORING_OPERATION_PHASE" != "Running" && "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then return 0 fi sleep 3 done return 1 } cleanup_monitoring_admission_jobs() { local job="" while IFS= read -r job; do [[ -n "$job" ]] || continue kubectl delete "$job" -n "$MONITORING_NAMESPACE" --ignore-not-found=true done < <( kubectl get jobs -n "$MONITORING_NAMESPACE" -o name 2>/dev/null | grep -E 'admission-(create|patch)' || true ) } wait_monitoring_synced_healthy() { local deadline=$((SECONDS + MONITORING_RECOVERY_WAIT_SECONDS)) while (( SECONDS < deadline )); do monitoring_operation_fields if [[ "$MONITORING_SYNC_STATUS" == "Synced" && "$MONITORING_HEALTH_STATUS" == "Healthy" && "$MONITORING_OPERATION_PHASE" != "Running" && "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then ok "${MONITORING_APP_NAME}: Synced/Healthy en ${MONITORING_TARGET_REVISION}." return 0 fi sleep 10 done monitoring_app_diagnostics return 1 } recover_monitoring_operation() { local explicit="${1:-false}" local refresh_deadline=0 kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 || die "No existe Application/${MONITORING_APP_NAME}." kubectl get limitrange monitoring-default-resources \ -n "$MONITORING_NAMESPACE" >/dev/null 2>&1 || die "Falta LimitRange/monitoring-default-resources." monitoring_operation_fields printf 'Target revision : %s\n' "${MONITORING_TARGET_REVISION:-desconocida}" printf 'Operation revision : %s\n' "${MONITORING_OPERATION_REVISION:-ninguna}" printf 'Operation phase : %s\n' "${MONITORING_OPERATION_PHASE:-ninguna}" printf 'Operation message : %s\n' "${MONITORING_OPERATION_MESSAGE:-sin mensaje}" printf 'Sync status : %s\n' "${MONITORING_SYNC_STATUS:-desconocido}" printf 'Health status : %s\n' "${MONITORING_HEALTH_STATUS:-desconocido}" # No se inicia otra sincronización cuando la revisión actual ya terminó bien. if [[ "$MONITORING_OPERATION_PHASE" == "Succeeded" && "$MONITORING_OPERATION_REVISION" == "$MONITORING_TARGET_REVISION" ]]; then kubectl annotate application "$MONITORING_APP_NAME" -n argocd \ argocd.argoproj.io/refresh=hard --overwrite >/dev/null refresh_deadline=$((SECONDS + 90)) while (( SECONDS < refresh_deadline )); do monitoring_operation_fields if [[ "$MONITORING_SYNC_STATUS" == "Synced" && "$MONITORING_HEALTH_STATUS" == "Healthy" ]]; then ok "${MONITORING_APP_NAME} ya estaba correctamente Synced/Healthy; no se ejecutó un sync adicional." return 0 fi sleep 5 done warn "La operación ${MONITORING_TARGET_REVISION} terminó, pero la Application continúa ${MONITORING_SYNC_STATUS}/${MONITORING_HEALTH_STATUS}; se solicitará una nueva reconciliación." fi if monitoring_operation_is_stale || [[ "$explicit" == "true" && "$MONITORING_OPERATION_PHASE" == "Running" ]]; then warn "Terminando operación obsoleta/atascada de ${MONITORING_APP_NAME}." argocd_core_cli app terminate-op "$MONITORING_APP_NAME" || die "No fue posible terminar la operación." wait_monitoring_operation_stopped || die "La operación no terminó dentro de ${MONITORING_OPERATION_STOP_WAIT}s." cleanup_monitoring_admission_jobs fi kubectl annotate application "$MONITORING_APP_NAME" -n argocd \ argocd.argoproj.io/refresh=hard --overwrite >/dev/null # La Application tiene auto-sync. Primero damos oportunidad al controller # para reconciliar sin crear una operación manual redundante. refresh_deadline=$((SECONDS + 90)) while (( SECONDS < refresh_deadline )); do monitoring_operation_fields if [[ "$MONITORING_SYNC_STATUS" == "Synced" && "$MONITORING_HEALTH_STATUS" == "Healthy" && "$MONITORING_OPERATION_PHASE" != "Running" && "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then ok "${MONITORING_APP_NAME}: reconciliada automáticamente." return 0 fi if [[ "$MONITORING_OPERATION_PHASE" == "Running" ]]; then wait_monitoring_synced_healthy || die "${MONITORING_APP_NAME} no quedó Synced/Healthy." return 0 fi sleep 5 done log "Auto-sync no inició; solicitando sync manual de ${MONITORING_APP_NAME}" argocd_core_cli app sync "$MONITORING_APP_NAME" \ --async --prune --assumeYes || die "No fue posible iniciar el sync." wait_monitoring_synced_healthy || die "${MONITORING_APP_NAME} no quedó Synced/Healthy." } auto_recover_monitoring_if_stale() { [[ "$AUTO_RECOVER_STALE_MONITORING_OPERATION" == "true" ]] || return 0 kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 || return 0 if monitoring_operation_is_stale; then recover_monitoring_operation false fi } ensure_root_application_present() { [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 if ! kubectl get crd applications.argoproj.io >/dev/null 2>&1; then warn "El CRD Application aún no existe; se omite la verificación de ${ROOT_APP_NAME}." return 0 fi if kubectl get application "$ROOT_APP_NAME" -n argocd >/dev/null 2>&1; then return 0 fi warn "${ROOT_APP_NAME} no existe; restaurándolo desde el clon persistente." if [[ "$APP_REGISTRY_REFRESH_ON_ENSURE" == "true" ]]; then sync_apps_registry_repo elif ! apps_registry_cache_valid; then sync_apps_registry_repo else validate_apps_registry_cache fi if [[ "$PLATFORM_INFRA_REFRESH_ON_ENSURE" == "true" ]]; then sync_platform_infra_repo elif ! platform_infra_cache_valid; then sync_platform_infra_repo fi validate_and_render_platform_infra apply_governance_snapshots configure_argocd_repository apply_root_application } runner_container_exists() { docker inspect "$GITEA_RUNNER_CONTAINER" >/dev/null 2>&1 } runner_is_running() { [[ "$(docker inspect -f '{{.State.Running}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true)" == "true" ]] } runner_is_registered() { [[ -s "${GITEA_RUNNER_DATA_DIR}/.runner" ]] } prepare_gitea_runner_directories() { # El watchdog se ejecuta como devops y no dispone de TTY para sudo. # install-autostart deja estas rutas creadas y con propietario correcto. install -d \ -m 0700 \ "$GITEA_RUNNER_ROOT" \ "$GITEA_RUNNER_DATA_DIR" || die "No se pudieron preparar las rutas persistentes del Gitea runner como $(id -un)." } detect_gitea_runner_image() { local detected_image="" if [[ -n "${GITEA_RUNNER_IMAGE_OVERRIDE:-}" ]]; then printf '%s\n' "$GITEA_RUNNER_IMAGE_OVERRIDE" return 0 fi if [[ -s "$GITEA_RUNNER_IMAGE_FILE" ]]; then tr -d '\r\n' < "$GITEA_RUNNER_IMAGE_FILE" return 0 fi if runner_container_exists; then detected_image="$( docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true )" fi printf '%s\n' "${detected_image:-$GITEA_RUNNER_IMAGE}" } adopt_existing_gitea_runner() { runner_container_exists || return 0 local current_image data_mount="" current_image="$( docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER" )" printf '%s\n' "$current_image" > "$GITEA_RUNNER_IMAGE_FILE" chmod 0600 "$GITEA_RUNNER_IMAGE_FILE" data_mount="$( docker inspect \ -f '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}' \ "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true )" if ! runner_is_registered; then warn "Copiando el registro actual del runner desde ${GITEA_RUNNER_CONTAINER}:/data." docker cp \ "${GITEA_RUNNER_CONTAINER}:/data/." \ "${GITEA_RUNNER_DATA_DIR}/" 2>/dev/null || true fi if [[ "$data_mount" == "$GITEA_RUNNER_DATA_DIR" ]]; then docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null return 0 fi warn "El runner actual no usa el volumen persistente ${GITEA_RUNNER_DATA_DIR}." if [[ "$MIGRATE_EXISTING_RUNNER" != "true" ]]; then warn "Se conserva el contenedor actual. Ejecuta MIGRATE_EXISTING_RUNNER=true $0 runner cuando no haya un workflow activo." docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null return 0 fi runner_is_registered || die "No se pudo recuperar ${GITEA_RUNNER_DATA_DIR}/.runner; no se eliminará el contenedor actual." log "Migrando ${GITEA_RUNNER_CONTAINER} al almacenamiento persistente" docker stop "$GITEA_RUNNER_CONTAINER" >/dev/null || true docker rm "$GITEA_RUNNER_CONTAINER" >/dev/null } register_gitea_runner() { runner_is_registered && return 0 [[ -s "$GITEA_RUNNER_TOKEN_FILE" ]] || die "El runner no está registrado. Guarda un token de registro en ${GITEA_RUNNER_TOKEN_FILE} con permisos 0600." local runner_image runner_image="$(detect_gitea_runner_image)" log "Registrando Gitea Actions runner ${GITEA_RUNNER_NAME}" docker pull "$runner_image" docker run --rm \ -v "${GITEA_RUNNER_DATA_DIR}:/data" \ -v "${GITEA_RUNNER_TOKEN_FILE}:/run/secrets/gitea-runner-token:ro" \ -e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \ -e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \ -e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \ --entrypoint /bin/sh \ "$runner_image" \ -ec ' cd /data act_runner register \ --no-interactive \ --instance "$GITEA_INSTANCE_URL" \ --token "$(cat /run/secrets/gitea-runner-token)" \ --name "$GITEA_RUNNER_NAME" \ --labels "$GITEA_RUNNER_LABELS" ' runner_is_registered || die "Gitea no generó ${GITEA_RUNNER_DATA_DIR}/.runner." printf '%s\n' "$runner_image" > "$GITEA_RUNNER_IMAGE_FILE" chmod 0600 "$GITEA_RUNNER_IMAGE_FILE" } create_gitea_runner_container() { runner_container_exists && return 0 local runner_image runner_image="$(detect_gitea_runner_image)" register_gitea_runner log "Creando contenedor ${GITEA_RUNNER_CONTAINER}" docker run -d \ --name "$GITEA_RUNNER_CONTAINER" \ --restart unless-stopped \ -v "${GITEA_RUNNER_DATA_DIR}:/data" \ -v /var/run/docker.sock:/var/run/docker.sock \ -e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \ -e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \ -e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \ "$runner_image" >/dev/null } validate_gitea_runner() { runner_container_exists || die "No existe el contenedor ${GITEA_RUNNER_CONTAINER}." if ! runner_is_running; then docker start "$GITEA_RUNNER_CONTAINER" >/dev/null fi docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null local deadline=$((SECONDS + 60)) while (( SECONDS < deadline )); do if runner_is_running; then ok "Gitea runner activo: ${GITEA_RUNNER_CONTAINER}." return 0 fi sleep 3 done docker logs --tail 150 "$GITEA_RUNNER_CONTAINER" || true die "El contenedor ${GITEA_RUNNER_CONTAINER} no quedó activo." } ensure_gitea_runner() { [[ "$INSTALL_GITEA_RUNNER" == "true" ]] || return 0 prepare_gitea_runner_directories adopt_existing_gitea_runner if ! runner_container_exists; then if ! runner_is_registered && [[ ! -s "$GITEA_RUNNER_TOKEN_FILE" ]]; then if [[ "$GITEA_RUNNER_REQUIRED" == "true" ]]; then die "Falta el registro persistente del runner y el token ${GITEA_RUNNER_TOKEN_FILE}." fi warn "Gitea runner omitido porque no hay registro ni token." return 0 fi create_gitea_runner_container fi validate_gitea_runner } apply_bootstrap_manifests() { shopt -s nullglob local manifests=("$BOOTSTRAP_DIR"/*.yaml "$BOOTSTRAP_DIR"/*.yml) shopt -u nullglob if (( ${#manifests[@]} == 0 )); then warn "No hay manifiestos GitOps en ${BOOTSTRAP_DIR}. Argo CD quedó instalado, pero las Applications deben aplicarse después." return 0 fi log "Aplicando manifiestos de bootstrap GitOps" kubectl apply -f "$BOOTSTRAP_DIR" } print_credentials() { local argocd_password='pendiente' if kubectl get secret argocd-initial-admin-secret -n argocd >/dev/null 2>&1; then argocd_password="$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 --decode)" fi printf '\n==========================================================\n' printf ' LABORATORIO DISPONIBLE\n' printf ' ---------------------------------------------------------\n' printf ' API Kubernetes : https://%s:%s\n' "$LAB_HOST_IP" "$API_PORT" printf ' NPM Argo CD : http://%s:%s\n' "$LAB_HOST_IP" "$HTTP_PORT" printf ' Host Argo CD : %s\n' "$ARGOCD_HOST" printf ' ArgoCD admin : %s\n' "$argocd_password" printf ' Portainer API : %s\n' "$PORTAINER_SERVER_URL" printf ' Portainer Edge : deployment/%s namespace/%s\n' \ "$PORTAINER_AGENT_DEPLOYMENT" "$PORTAINER_NAMESPACE" printf ' Kubeconfig : %s\n' "$KUBECONFIG_PATH" printf ' Root Application: %s -> %s/%s@%s\n' \ "$ROOT_APP_NAME" "$ROOT_APP_REPO_URL" "$ROOT_APP_REPO_PATH" "$ROOT_APP_REVISION" printf ' Gitea runner : %s\n' "$GITEA_RUNNER_CONTAINER" printf '==========================================================\n' } status_report() { require_commands ensure_docker log "Docker" docker info --format 'DockerRootDir={{.DockerRootDir}} Driver={{.Driver}}' docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \ --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' log "k3d" k3d cluster list || true if cluster_exists; then write_kubeconfig log "Kubernetes" kubectl get nodes -o wide || true kubectl get pods -A || true kubectl get ingress -A || true kubectl get applications.argoproj.io -n argocd 2>/dev/null || true fi log "apps-registry local" if apps_registry_cache_valid; then printf 'Ruta: %s ' "$APP_REGISTRY_CACHE_DIR" printf 'Commit: %s ' "$(git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)" else warn "No existe un clon válido en ${APP_REGISTRY_CACHE_DIR}." fi log "platform-infra local" if platform_infra_cache_valid; then printf 'Ruta: %s ' "$PLATFORM_INFRA_CACHE_DIR" printf 'Commit: %s ' "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)" else warn "No existe un clon válido en ${PLATFORM_INFRA_CACHE_DIR}." fi log "Gitea Actions runner" if runner_container_exists; then docker ps -a --filter "name=^/${GITEA_RUNNER_CONTAINER}$" --format 'table {{.Names}} {{.Image}} {{.Status}}' else warn "No existe ${GITEA_RUNNER_CONTAINER}." fi log "Portainer" printf 'Server: %s\n' "$PORTAINER_SERVER_URL" printf 'Tunnel: %s:%s\n' "$PORTAINER_TUNNEL_HOST" "$PORTAINER_TUNNEL_PORT" if cluster_exists && kubectl get namespace "$PORTAINER_NAMESPACE" >/dev/null 2>&1; then kubectl get deployment,service,configmap,secret -n "$PORTAINER_NAMESPACE" 2>/dev/null || true else warn "No existe el namespace ${PORTAINER_NAMESPACE}." fi } backup_cluster_state() { local stamp archive stamp="$(date '+%Y%m%d-%H%M%S')" archive="${BACKUP_DIR}/k3d-lab-state-${stamp}.tar.gz" log "Respaldando estado persistente en ${archive}" if cluster_exists; then k3d cluster stop "$CLUSTER_NAME" || true fi sudo_cmd tar \ --acls \ --xattrs \ -C "$APP_DATA_PATH" \ -czf "$archive" \ k3s storage secrets config bootstrap gitea-runner git rendered 2>/dev/null || \ sudo_cmd tar -C "$APP_DATA_PATH" -czf "$archive" k3s storage secrets config bootstrap gitea-runner git rendered sudo_cmd chown "$(id -un)" "$archive" ok "Backup creado: ${archive}" } reset_cluster() { [[ "${CONFIRM_RESET:-}" == "DELETE-${CLUSTER_NAME}" ]] || \ die "Reset bloqueado. Usa CONFIRM_RESET=DELETE-${CLUSTER_NAME} $0 reset" prepare_directories backup_cluster_state if cluster_exists; then k3d cluster delete "$CLUSTER_NAME" fi local stamp="$(date '+%Y%m%d-%H%M%S')" sudo_cmd mv "$K3S_SERVER_PATH" "${K3S_SERVER_PATH}.reset-${stamp}" sudo_cmd mv "${APP_DATA_PATH}/storage" "${APP_DATA_PATH}/storage.reset-${stamp}" sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH" sudo_cmd install -d -m 0755 "$K3S_SERVER_STORAGE" "$K3S_AGENT0_STORAGE" "$K3S_AGENT1_STORAGE" create_cluster } ensure_cluster_only() { require_commands ensure_docker validate_data_mount validate_host_ip acquire_ensure_lock wait_for_boot_grace if ! cluster_exists; then if cluster_containers_exist; then die "Existen contenedores del lab, pero k3d no reconoce el clúster. Se requiere diagnóstico manual; no se crearán duplicados." fi die "El clúster ${CLUSTER_NAME} no existe. Ejecuta bootstrap manualmente." fi set_restart_policy start_runner_if_needed recover_cluster_conservatively write_kubeconfig wait_for_api wait_for_nodes validate_runtime_after_boot ensure_root_application_present auto_recover_monitoring_if_stale ensure_gitea_runner ensure_portainer_edge_agent } resolve_service_tool() { local command_name="$1" local explicit_path="${2:-}" local service_home="$3" local candidate="" for candidate in \ "$explicit_path" \ "$(command -v "$command_name" 2>/dev/null || true)" \ "${service_home}/.local/bin/${command_name}" \ "${service_home}/bin/${command_name}" \ "/DATA/.local/bin/${command_name}" \ "/DATA/bin/${command_name}" \ "/DATA/AppData/bin/${command_name}" \ "/usr/local/bin/${command_name}" \ "/usr/local/sbin/${command_name}" \ "/usr/bin/${command_name}" \ "/usr/sbin/${command_name}" \ "/bin/${command_name}" \ "/sbin/${command_name}" do [[ -n "$candidate" ]] || continue if [[ -x "$candidate" ]]; then readlink -f "$candidate" 2>/dev/null || printf '%s\n' "$candidate" return 0 fi done return 1 } append_unique_path_dir() { local current_path="$1" local new_dir="$2" [[ -n "$new_dir" ]] || { printf '%s\n' "$current_path" return } case ":${current_path}:" in *":${new_dir}:"*) printf '%s\n' "$current_path" ;; *) if [[ -n "$current_path" ]]; then printf '%s:%s\n' "$current_path" "$new_dir" else printf '%s\n' "$new_dir" fi ;; esac } install_autostart() { [[ $(id -u) -eq 0 ]] || die "Ejecuta install-autostart con sudo." local service_user="${SUDO_USER:-devops}" local service_home service_group docker_root installed_script local k3d_path kubectl_path docker_path curl_path tool_path tool_dir local service_result service_status service_home="$(getent passwd "$service_user" | cut -d: -f6)" service_group="$(id -gn "$service_user")" docker_root="$(docker info --format '{{.DockerRootDir}}')" installed_script="${APP_DATA_PATH}/bin/deploy-k3d-lab" [[ -n "$service_home" ]] || die "No fue posible determinar HOME de ${service_user}." [[ -n "$service_group" ]] || die "No fue posible determinar el grupo de ${service_user}." [[ -n "$docker_root" ]] || die "No fue posible determinar DockerRootDir." k3d_path="$( resolve_service_tool k3d "${K3D_BIN:-}" "$service_home" )" || die "No se encontró k3d para systemd. Usa K3D_BIN=\$(command -v k3d) con sudo -E." kubectl_path="$( resolve_service_tool kubectl "${KUBECTL_BIN:-}" "$service_home" )" || die "No se encontró kubectl para systemd." docker_path="$( resolve_service_tool docker "${DOCKER_BIN:-}" "$service_home" )" || die "No se encontró docker para systemd." curl_path="$( resolve_service_tool curl "${CURL_BIN:-}" "$service_home" )" || die "No se encontró curl para systemd." tool_path="${APP_DATA_PATH}/bin" for tool_dir in "$(dirname "$k3d_path")" "$(dirname "$kubectl_path")" "$(dirname "$docker_path")" "$(dirname "$curl_path")" /usr/local/sbin /usr/local/bin /usr/sbin /usr/bin /sbin /bin do tool_path="$(append_unique_path_dir "$tool_path" "$tool_dir")" done # ZimaOS no garantiza /usr/local/sbin y su filesystem base puede ser # inmutable. Instalamos el ejecutable en /DATA, que es persistente. install -d \ -m 0750 \ -o "$service_user" \ "$(dirname "$installed_script")" install \ -m 0750 \ -o "$service_user" \ "$0" \ "$installed_script" # El servicio se ejecutará como devops y no puede usar sudo sin TTY. # Dejamos listas y escribibles todas las rutas que el modo ensure modifica. install -d \ -m 0750 \ -o "$service_user" \ "$APP_DATA_PATH" install -d \ -m 0700 \ -o "$service_user" \ "$(dirname "$KUBECONFIG_PATH")" # El modo ensure valida y recupera el runner como usuario devops. # Preparamos estas rutas durante la instalación privilegiada para que # systemd nunca necesite ejecutar sudo de forma no interactiva. install -d \ -m 0700 \ -o "$service_user" \ "$GITEA_RUNNER_ROOT" \ "$GITEA_RUNNER_DATA_DIR" if [[ -f "$GITEA_RUNNER_IMAGE_FILE" ]]; then chown "$service_user" "$GITEA_RUNNER_IMAGE_FILE" chmod 0600 "$GITEA_RUNNER_IMAGE_FILE" fi if [[ -f "$KUBECONFIG_PATH" ]]; then chown "$service_user" "$KUBECONFIG_PATH" chmod 0600 "$KUBECONFIG_PATH" fi install -d -m 0755 \ /etc/default \ /etc/systemd/system systemctl enable docker.service >/dev/null if systemctl list-unit-files containerd.service >/dev/null 2>&1; then systemctl enable containerd.service >/dev/null || true fi # v4.5.5: el laboratorio nunca debe modificar las dependencias de # docker.service. Un RequiresMountsFor sobre almacenamiento administrado por # ZimaOS puede impedir el arranque global de Docker y, con ello, de todas las # aplicaciones del NAS. Eliminamos el drop-in creado por versiones previas. rm -f /etc/systemd/system/docker.service.d/10-k3d-storage.conf rmdir /etc/systemd/system/docker.service.d 2>/dev/null || true set_restart_policy start_runner_if_needed cat > /etc/default/k3d-lab < /etc/systemd/system/k3d-lab-ensure.service < /etc/systemd/system/k3d-lab-ensure.timer <<'UNIT' [Unit] Description=Watchdog del laboratorio k3d [Timer] # Docker restaura primero k3d y el runner usando unless-stopped. El watchdog # interviene después y solo llama k3d cluster start si la API sigue caída. OnBootSec=5min OnUnitInactiveSec=10min AccuracySec=20s Persistent=false Unit=k3d-lab-ensure.service [Install] WantedBy=timers.target UNIT # Evitamos que el timer y la prueba inicial intenten arrancar el # oneshot al mismo tiempo. También limpiamos contadores históricos de # StartLimit antes de probar la unidad recién instalada. systemctl stop k3d-lab-ensure.timer >/dev/null 2>&1 || true systemctl stop k3d-lab-ensure.service >/dev/null 2>&1 || true systemctl reset-failed k3d-lab-ensure.service >/dev/null 2>&1 || true systemctl daemon-reload systemctl enable --now k3d-lab-ensure.timer >/dev/null systemctl is-active --quiet k3d-lab-ensure.timer || die "El timer k3d-lab-ensure.timer no quedó activo." # No ejecutamos inmediatamente el oneshot durante la instalación. Así # evitamos competir con un clúster que todavía se está recuperando. ok "Autostart instalado para ${service_user}." printf 'Script persistente: %s\n' "$installed_script" printf 'k3d para systemd: %s\n' "$k3d_path" printf 'kubectl para systemd: %s\n' "$kubectl_path" printf 'PATH para systemd: %s\n' "$tool_path" printf 'DockerRootDir detectado (sin modificar docker.service): %s\n' "$docker_root" printf 'Logs: journalctl -u k3d-lab-ensure.service -n 200 --no-pager\n' printf 'Seguridad: docker.service no contiene dependencias del laboratorio.\n' printf 'Orden estable: Docker unless-stopped primero; watchdog después de 5 minutos.\n' } main() { case "$MODE" in status) status_report return ;; install-autostart) install_autostart return ;; ensure) ensure_cluster_only return ;; repo-sync) require_commands ensure_docker validate_data_mount prepare_directories sync_gitops_repositories return ;; runner) require_commands ensure_docker validate_data_mount prepare_directories ensure_gitea_runner return ;; gitops) require_commands ensure_docker validate_data_mount validate_host_ip cluster_exists || die "No existe ${CLUSTER_NAME}; ejecuta bootstrap." write_kubeconfig wait_for_api install_argocd sync_gitops_repositories apply_governance_snapshots configure_argocd_repository apply_root_application wait_for_root_application wait_for_expected_gitops_apps auto_recover_monitoring_if_stale wait_for_gitops_apps_health return ;; monitoring-diagnose) require_commands ensure_docker validate_data_mount cluster_exists || die "No existe ${CLUSTER_NAME}." write_kubeconfig wait_for_api monitoring_app_diagnostics return ;; monitoring-recover) require_commands ensure_docker validate_data_mount cluster_exists || die "No existe ${CLUSTER_NAME}." write_kubeconfig wait_for_api MONITORING_FORCE_RECOVERY=true recover_monitoring_operation true return ;; postdeploy-secrets) require_commands ensure_docker validate_data_mount validate_host_ip prepare_directories cluster_exists || die "No existe ${CLUSTER_NAME}." write_kubeconfig wait_for_api apply_postdeploy_rbac generate_postdeploy_credentials validate_postdeploy_credentials return ;; portainer) require_commands ensure_docker validate_data_mount validate_host_ip prepare_directories cluster_exists || die "No existe ${CLUSTER_NAME}." write_kubeconfig wait_for_api ensure_portainer_edge_agent return ;; portainer-adopt) portainer_adopt_mode return ;; retire-headlamp) retire_headlamp_mode return ;; esac require_commands ensure_docker validate_data_mount validate_host_ip prepare_directories ensure_cluster_token write_k3d_config case "$MODE" in bootstrap) if cluster_exists; then start_cluster else create_cluster fi ;; recover) cluster_exists || die "No existe ${CLUSTER_NAME}; utiliza bootstrap para crearlo." start_cluster ;; reset) reset_cluster ;; *) die "Modo no válido: ${MODE}. Usa bootstrap, recover, ensure, status, repo-sync, gitops, runner, monitoring-diagnose, monitoring-recover, postdeploy-secrets, portainer, portainer-adopt, retire-headlamp, install-autostart o reset." ;; esac set_restart_policy write_kubeconfig wait_for_api wait_for_nodes install_argocd apply_argocd_ingress apply_postdeploy_rbac generate_postdeploy_credentials sync_gitops_repositories apply_governance_snapshots configure_argocd_repository write_root_application_manifest apply_bootstrap_manifests apply_root_application wait_for_root_application wait_for_expected_gitops_apps auto_recover_monitoring_if_stale wait_for_gitops_apps_health ensure_gitea_runner ensure_portainer_edge_agent validate_runtime_after_boot log "Validación final" kubectl get nodes -o wide kubectl get pods -n argocd kubectl get ingress -A kubectl get applications.argoproj.io -n argocd 2>/dev/null || true print_credentials } main "$@"