From 5b1a63ab2e1875cb66f36db3de03769c48ea5490 Mon Sep 17 00:00:00 2001 From: devops Date: Fri, 17 Jul 2026 13:55:35 +0000 Subject: [PATCH] Update deploy-lab.sh --- deploy-lab.sh | 1369 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 1312 insertions(+), 57 deletions(-) diff --git a/deploy-lab.sh b/deploy-lab.sh index 7dc8ffc..d3c4f20 100644 --- a/deploy-lab.sh +++ b/deploy-lab.sh @@ -1,53 +1,749 @@ -#!/bin/bash -# Script Maestro Final: Laboratorio Estable (ZimaOS + k3d + ArgoCD + Headlamp) -set -e +#!/usr/bin/env bash +# Laboratorio estable v3.8: ZimaOS + Docker + k3d + Argo CD + Traefik +# +# Modos: +# bootstrap Crea el clúster si no existe; si existe, lo inicia y reconcilia. +# recover Recupera únicamente un clúster existente. +# ensure Health check no destructivo para systemd; no instala Argo CD. +# status Muestra diagnóstico del laboratorio. +# install-autostart Instala servicio/timer systemd, orden de montajes y autorreparación. +# reset Reconstrucción limpia; requiere confirmación explícita. +# +# Ejemplo: +# LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh bootstrap -CLUSTER_NAME="lab-cluster" -APP_DATA_PATH="/DATA/AppData/k3d-lab" +set -Eeuo pipefail +IFS=$'\n\t' -echo "--- 1. Preparando entorno ---" -sudo mkdir -p $APP_DATA_PATH/{gitea,argocd} -sudo chmod -R 777 $APP_DATA_PATH/ +MODE="${1:-bootstrap}" -echo "--- 2. Limpiando clúster previo ---" -k3d cluster delete $CLUSTER_NAME || true +if [[ -z "${BASH_VERSION:-}" ]]; then + printf 'ERROR: este script requiere Bash. Ejecútalo con: bash deploy-lab.sh %s\n' "$MODE" >&2 + exit 1 +fi -echo "--- 3. Creando clúster ---" -k3d cluster create $CLUSTER_NAME \ - --api-port 46421 \ - -p "90:80@loadbalancer" \ - -p "9443:443@loadbalancer" \ - -v "$APP_DATA_PATH/gitea:/var/lib/gitea" \ - -v "$APP_DATA_PATH/argocd:/var/lib/argocd" \ - --agents 2 +if [[ $(id -u) -eq 0 && "$MODE" != "install-autostart" ]]; then + printf 'ERROR: ejecuta el modo %s como usuario normal, sin sudo.\n' "$MODE" >&2 + printf 'Ejemplo: LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh %s\n' "$MODE" >&2 + exit 1 +fi -echo "--- 4. Configurando reinicio y Kubeconfig ---" -sudo docker update --restart=always $(sudo docker ps -a -q -f name=k3d) -sudo chown -R $USER:$USER /DATA/.kube /DATA/.config -k3d kubeconfig merge $CLUSTER_NAME --kubeconfig-switch-context +CLUSTER_NAME="${CLUSTER_NAME:-lab-cluster}" +APP_DATA_PATH="${APP_DATA_PATH:-/DATA/AppData/k3d-lab}" +KUBECONFIG_PATH="${KUBECONFIG_PATH:-/DATA/.kube/config}" +CONFIG_FILE="${CONFIG_FILE:-${APP_DATA_PATH}/config/k3d-lab.yaml}" +BOOTSTRAP_DIR="${BOOTSTRAP_DIR:-${APP_DATA_PATH}/bootstrap}" +BACKUP_DIR="${BACKUP_DIR:-${APP_DATA_PATH}/backups}" -echo "--- 5. Instalando ArgoCD y RBAC ---" -kubectl create namespace argocd || true -kubectl create serviceaccount argocd-manager -n argocd || true -kubectl create clusterrolebinding argocd-manager-binding \ - --clusterrole=cluster-admin \ - --serviceaccount=argocd:argocd-manager || true +API_PORT="${API_PORT:-46421}" +HTTP_PORT="${HTTP_PORT:-90}" +HTTPS_PORT="${HTTPS_PORT:-9443}" +AGENT_COUNT="${AGENT_COUNT:-2}" +K3D_SUBNET="${K3D_SUBNET:-auto}" +K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.35.5-k3s1}" +ARGOCD_VERSION="${ARGOCD_VERSION:-v3.2.0}" +ARGOCD_HOST="${ARGOCD_HOST:-argocd.cruzcloud.net}" +EXPECTED_HOST_IP="${EXPECTED_HOST_IP:-192.168.68.61}" +ALLOW_IP_CHANGE="${ALLOW_IP_CHANGE:-false}" +INSTALL_HEADLAMP_RBAC="${INSTALL_HEADLAMP_RBAC:-true}" +APPLY_POSTDEPLOY_RBAC="${APPLY_POSTDEPLOY_RBAC:-true}" +WAIT_SECONDS="${WAIT_SECONDS:-240}" +ARGOCD_ROLLOUT_TIMEOUT="${ARGOCD_ROLLOUT_TIMEOUT:-420}" +AGENT_RECOVERY_WAIT="${AGENT_RECOVERY_WAIT:-75}" +AUTO_REREGISTER_STALE_AGENTS="${AUTO_REREGISTER_STALE_AGENTS:-true}" +CHECK_INGRESS_AFTER_BOOT="${CHECK_INGRESS_AFTER_BOOT:-true}" -kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml -sleep 20 # Espera técnica para que los CRDs de ArgoCD se carguen -kubectl patch configmap argocd-cmd-params-cm -n argocd --type merge -p '{"data":{"server.insecure":"true"}}' +K3S_SERVER_PATH="${APP_DATA_PATH}/k3s/server" +K3S_SERVER_STORAGE="${APP_DATA_PATH}/storage/server-0" +K3S_AGENT0_STORAGE="${APP_DATA_PATH}/storage/agent-0" +K3S_AGENT1_STORAGE="${APP_DATA_PATH}/storage/agent-1" +TOKEN_FILE="${APP_DATA_PATH}/secrets/k3s-cluster-token" +POSTDEPLOY_RBAC_FILE="${APP_DATA_PATH}/gitea-postdeploy-validator-rbac.yaml" -echo "--- 6. Creando credenciales para Headlamp ---" -kubectl create namespace headlamp || true -kubectl create serviceaccount headlamp-admin -n headlamp || true -kubectl create clusterrolebinding headlamp-admin-binding --clusterrole=cluster-admin --serviceaccount=headlamp:headlamp-admin || true -kubectl create secret generic headlamp-kubeconfig --from-file=config=/DATA/.kube/config -n headlamp --dry-run=client -o yaml | kubectl apply -f - +if [[ -t 1 ]]; then + C_RESET=$'\033[0m' + C_BLUE=$'\033[1;34m' + C_GREEN=$'\033[1;32m' + C_YELLOW=$'\033[1;33m' + C_RED=$'\033[1;31m' +else + C_RESET='' + C_BLUE='' + C_GREEN='' + C_YELLOW='' + C_RED='' +fi -echo "--- 7. Validando despliegue ---" -kubectl wait --for=condition=Ready pods -n argocd --all --timeout=180s +log() { printf '\n%s[%s] %s%s\n' "$C_BLUE" "$(date '+%F %T')" "$*" "$C_RESET"; } +ok() { printf '%sOK:%s %s\n' "$C_GREEN" "$C_RESET" "$*"; } +warn() { printf '%sAVISO:%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +die() { printf '%sERROR:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; exit 1; } -echo "--- 8. Aplicando Ingress ---" -cat <&2 + exit "$exit_code" +} +trap on_error ERR + +have() { command -v "$1" >/dev/null 2>&1; } + +sudo_cmd() { + if [[ $(id -u) -eq 0 ]]; then + "$@" + else + sudo "$@" + fi +} + +require_commands() { + local command_name + for command_name in docker k3d kubectl curl awk sed grep findmnt ss tar base64 tr; do + have "$command_name" || die "No se encontró el comando requerido: ${command_name}" + done +} + +detect_host_ip() { + ip -4 route get 1.1.1.1 2>/dev/null | awk '{ + for (i = 1; i <= NF; i++) { + if ($i == "src") { + print $(i + 1) + exit + } + } + }' +} + +LAB_HOST_IP="${LAB_HOST_IP:-$(detect_host_ip)}" +export LAB_HOST_IP CLUSTER_NAME APP_DATA_PATH K3D_SUBNET K3S_IMAGE AGENT_COUNT API_PORT HTTP_PORT HTTPS_PORT + +validate_host_ip() { + [[ -n "$LAB_HOST_IP" ]] || die "No fue posible detectar LAB_HOST_IP. Defínela manualmente." + + if [[ -n "$EXPECTED_HOST_IP" && "$LAB_HOST_IP" != "$EXPECTED_HOST_IP" ]]; then + if [[ "$ALLOW_IP_CHANGE" == "true" ]]; then + warn "La IP detectada ${LAB_HOST_IP} no coincide con ${EXPECTED_HOST_IP}; se continuará por ALLOW_IP_CHANGE=true." + else + die "La IP detectada es ${LAB_HOST_IP}, pero NPM/Gitea esperan ${EXPECTED_HOST_IP}. Reserva la IP o usa ALLOW_IP_CHANGE=true conscientemente." + fi + fi +} + +ensure_docker() { + if have systemctl && ! systemctl is-active --quiet docker.service; then + if [[ $(id -u) -eq 0 || -t 0 ]]; then + log "Iniciando Docker" + sudo_cmd systemctl start docker.service + else + die "Docker no está activo. systemd debe iniciarlo antes de ejecutar el watchdog." + fi + fi + + docker info >/dev/null 2>&1 || + die "Docker no responde para el usuario $(id -un). Valida docker.service y el grupo docker." +} + +validate_data_mount() { + findmnt -T /DATA >/dev/null 2>&1 || + die "/DATA no está montado. No se iniciará k3d para evitar escribir en el filesystem raíz." + + [[ -d "$APP_DATA_PATH" ]] || + die "No existe la ruta persistente ${APP_DATA_PATH}." + + # En ZimaOS el usuario devops no puede escribir directamente en /DATA, + # pero sí debe poder escribir en su ruta del laboratorio. + local test_file="${APP_DATA_PATH}/.k3d-write-test.$$" + + touch "$test_file" 2>/dev/null || + die "${APP_DATA_PATH} no permite escritura para $(id -un)." + + rm -f "$test_file" +} + +prepare_directories() { + log "Preparando almacenamiento persistente" + + sudo_cmd install -d -m 0750 -o "$(id -un)" \ + "$APP_DATA_PATH" \ + "${APP_DATA_PATH}/config" \ + "${APP_DATA_PATH}/secrets" \ + "$BOOTSTRAP_DIR" \ + "$BACKUP_DIR" \ + "$(dirname "$KUBECONFIG_PATH")" + + # K3s y los workloads escriben como root dentro de los nodos. + sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH" + sudo_cmd install -d -m 0755 \ + "$K3S_SERVER_STORAGE" \ + "$K3S_AGENT0_STORAGE" \ + "$K3S_AGENT1_STORAGE" + + sudo_cmd chown "$(id -un)" "$(dirname "$KUBECONFIG_PATH")" +} + +generate_cluster_token() { + local token="" + + if have openssl; then + token="$(openssl rand -hex 32)" + elif have od; then + token="$( + od -An -N32 -tx1 /dev/urandom | + awk '{ + for (i = 1; i <= NF; i++) { + printf "%s", $i + } + } + END { + printf "\n" + }' + )" + elif have hexdump; then + token="$(hexdump -vn32 -e '1/1 "%02x"' /dev/urandom)" + else + die "No hay un generador aleatorio disponible. Se requiere openssl, od o hexdump." + fi + + [[ "$token" =~ ^[0-9a-fA-F]{64}$ ]] || + die "No fue posible generar un token hexadecimal seguro." + + printf '%s\n' "$token" +} + +ensure_cluster_token() { + if [[ ! -s "$TOKEN_FILE" ]]; then + log "Generando token persistente del clúster" + umask 077 + generate_cluster_token > "$TOKEN_FILE" + chmod 0600 "$TOKEN_FILE" + fi + + K3S_CLUSTER_TOKEN="$(<"$TOKEN_FILE")" + + [[ "$K3S_CLUSTER_TOKEN" =~ ^[0-9a-fA-F]{64}$ ]] || + die "El token persistente de K3s no es válido." + + export K3S_CLUSTER_TOKEN +} + +write_k3d_config() { + log "Generando configuración declarativa de k3d" + + cat > "$CONFIG_FILE" <<'YAML' +apiVersion: k3d.io/v1alpha5 +kind: Simple + +metadata: + name: ${CLUSTER_NAME} + +servers: 1 +agents: ${AGENT_COUNT} +image: ${K3S_IMAGE} +subnet: "${K3D_SUBNET}" +token: "${K3S_CLUSTER_TOKEN}" + +kubeAPI: + host: "${LAB_HOST_IP}" + hostIP: "0.0.0.0" + hostPort: "${API_PORT}" + +ports: + - port: "${HTTP_PORT}:80" + nodeFilters: + - loadbalancer + - port: "${HTTPS_PORT}:443" + nodeFilters: + - loadbalancer + +volumes: + # Conserva SQLite, certificados y token de K3s fuera de los contenedores. + - volume: "${APP_DATA_PATH}/k3s/server:/var/lib/rancher/k3s/server" + nodeFilters: + - server:0 + + # Conserva datos de PVC creados por local-path-provisioner. + - volume: "${APP_DATA_PATH}/storage/server-0:/var/lib/rancher/k3s/storage" + nodeFilters: + - server:0 + - volume: "${APP_DATA_PATH}/storage/agent-0:/var/lib/rancher/k3s/storage" + nodeFilters: + - agent:0 + - volume: "${APP_DATA_PATH}/storage/agent-1:/var/lib/rancher/k3s/storage" + nodeFilters: + - agent:1 + +options: + k3d: + wait: true + timeout: "240s" + disableRollback: false + k3s: + extraArgs: + - arg: "--tls-san=${LAB_HOST_IP}" + nodeFilters: + - server:* + kubeconfig: + updateDefaultKubeconfig: false + switchCurrentContext: false +YAML + + chmod 0600 "$CONFIG_FILE" +} + +cluster_exists() { + k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | grep -Fxq "$CLUSTER_NAME" +} + +cluster_containers_exist() { + docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" --format '{{.ID}}' | grep -q . +} + +ports_in_use_by_non_cluster_process() { + local port + for port in "$API_PORT" "$HTTP_PORT" "$HTTPS_PORT"; do + if ss -lntH "sport = :${port}" 2>/dev/null | grep -q .; then + return 0 + fi + done + return 1 +} + +create_cluster() { + if cluster_containers_exist; then + die "Existen contenedores etiquetados para ${CLUSTER_NAME}, pero k3d no reconoce el clúster. No se crearán duplicados." + fi + + if ports_in_use_by_non_cluster_process; then + sudo_cmd ss -lntp | grep -E ":(${API_PORT}|${HTTP_PORT}|${HTTPS_PORT})\\b" || true + die "Uno o más puertos del laboratorio están ocupados." + fi + + log "Creando clúster ${CLUSTER_NAME} con IPAM estático" + docker pull "$K3S_IMAGE" + k3d cluster create --config "$CONFIG_FILE" +} + +start_cluster() { + log "Iniciando clúster existente ${CLUSTER_NAME}" + k3d cluster start "$CLUSTER_NAME" --wait --timeout "${WAIT_SECONDS}s" || true +} + +set_restart_policy() { + mapfile -t cluster_containers < <( + docker ps -aq --filter "label=k3d.cluster=${CLUSTER_NAME}" + ) + + if (( ${#cluster_containers[@]} > 0 )); then + docker update --restart=unless-stopped "${cluster_containers[@]}" >/dev/null + ok "Política Docker unless-stopped aplicada a ${#cluster_containers[@]} contenedores del clúster." + fi +} + +write_kubeconfig() { + local temp_file context_name configured_server + temp_file="$(mktemp)" + + k3d kubeconfig get "$CLUSTER_NAME" > "$temp_file" + [[ -s "$temp_file" ]] || die "k3d no generó el kubeconfig." + + context_name="$( + KUBECONFIG="$temp_file" \ + kubectl config get-contexts -o name 2>/dev/null | + head -1 + )" + + [[ -n "$context_name" ]] || + die "El kubeconfig generado no contiene ningún contexto." + + # Algunas combinaciones de k3d/kubectl pueden producir un archivo válido + # sin current-context. Lo fijamos explícitamente antes de instalarlo. + KUBECONFIG="$temp_file" \ + kubectl config use-context "$context_name" >/dev/null + + configured_server="$( + KUBECONFIG="$temp_file" \ + kubectl config view \ + --minify \ + -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null + )" + + [[ -n "$configured_server" ]] || + die "El kubeconfig generado no contiene un API Server válido." + + # El watchdog se ejecuta sin TTY y no debe invocar sudo. + # install-autostart garantiza previamente el propietario de esta ruta. + install -d \ + -m 0700 \ + "$(dirname "$KUBECONFIG_PATH")" || + die "No se pudo preparar $(dirname "$KUBECONFIG_PATH") como $(id -un)." + + install \ + -m 0600 \ + "$temp_file" \ + "$KUBECONFIG_PATH" || + die "No se pudo actualizar ${KUBECONFIG_PATH} como $(id -un)." + + rm -f "$temp_file" + export KUBECONFIG="$KUBECONFIG_PATH" + + [[ "$(kubectl config current-context 2>/dev/null || true)" == "$context_name" ]] || + die "No fue posible activar el contexto ${context_name}." + + ok "Kubeconfig actualizado: ${KUBECONFIG_PATH}" + ok "Contexto activo: ${context_name}" + ok "API Server: ${configured_server}" +} + +wait_for_api() { + log "Esperando API de Kubernetes" + local deadline=$((SECONDS + WAIT_SECONDS)) + + while (( SECONDS < deadline )); do + if kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1; then + ok "API de Kubernetes disponible." + return 0 + fi + sleep 5 + done + + docker logs --tail 200 "k3d-${CLUSTER_NAME}-server-0" || true + die "La API no quedó disponible dentro de ${WAIT_SECONDS}s." +} + +node_ready() { + local node_name="$1" + [[ "$(kubectl get node "$node_name" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' \ + 2>/dev/null || true)" == "True" ]] +} + +wait_agent_ready() { + local node_name="$1" + local timeout_seconds="$2" + local deadline=$((SECONDS + timeout_seconds)) + + while (( SECONDS < deadline )); do + if node_ready "$node_name"; then + return 0 + fi + sleep 5 + done + + return 1 +} + +recover_not_ready_agents() { + local index node_name container_name docker_ip kubernetes_ip + + # El servidor y el API deben estar sanos antes de reparar agentes. + kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1 || + die "No se repararán agentes porque el API Server no está disponible." + + for (( index = 0; index < AGENT_COUNT; index++ )); do + node_name="k3d-${CLUSTER_NAME}-agent-${index}" + container_name="$node_name" + + if node_ready "$node_name"; then + continue + fi + + warn "${node_name} no existe o no está Ready; reiniciando el contenedor." + docker restart "$container_name" >/dev/null || true + + if wait_agent_ready "$node_name" "$AGENT_RECOVERY_WAIT"; then + ok "${node_name} se recuperó después del reinicio." + continue + fi + + docker_ip="$( + docker inspect \ + -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + "$container_name" 2>/dev/null || true + )" + + kubernetes_ip="$( + kubectl get node "$node_name" \ + -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' \ + 2>/dev/null || true + )" + + warn "${node_name} continúa sin estar Ready. DockerIP=${docker_ip:-ausente}, KubernetesIP=${kubernetes_ip:-ausente}." + + if [[ "$AUTO_REREGISTER_STALE_AGENTS" != "true" ]]; then + continue + fi + + # En un lab, si el agente lleva el periodo de gracia completo sin responder, + # se elimina únicamente su objeto Node y password secret para permitir + # un registro limpio con la IP estática actual. + warn "Forzando re-registro seguro de ${node_name}." + kubectl delete node "$node_name" --ignore-not-found=true >/dev/null + kubectl delete secret \ + -n kube-system \ + "${node_name}.node-password.k3s" \ + --ignore-not-found=true >/dev/null + + docker restart "$container_name" >/dev/null + + if wait_agent_ready "$node_name" "$((AGENT_RECOVERY_WAIT * 2))"; then + ok "${node_name} volvió a registrarse y está Ready." + else + docker logs --tail 180 "$container_name" || true + die "${node_name} no se recuperó después del re-registro." + fi + done + + if ! docker ps --format '{{.Names}}' | + grep -Fxq "k3d-${CLUSTER_NAME}-serverlb"; then + warn "El load balancer no está activo; intentando iniciarlo." + docker start "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true + fi +} + +nodes_are_ready() { + local expected_nodes=$((AGENT_COUNT + 1)) + local actual_nodes ready_nodes + + actual_nodes="$(kubectl get nodes --no-headers 2>/dev/null | awk 'NF {count++} END {print count+0}')" + ready_nodes="$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep -c '^True$' || true)" + + [[ "$actual_nodes" -eq "$expected_nodes" && "$ready_nodes" -eq "$expected_nodes" ]] +} + +wait_for_nodes_once() { + local deadline=$((SECONDS + WAIT_SECONDS)) + + while (( SECONDS < deadline )); do + if nodes_are_ready; then + return 0 + fi + sleep 5 + done + + return 1 +} + +wait_for_nodes() { + log "Esperando $((AGENT_COUNT + 1)) nodos registrados y Ready" + + if wait_for_nodes_once; then + kubectl get nodes -o wide + return 0 + fi + + recover_not_ready_agents + + if wait_for_nodes_once; then + kubectl get nodes -o wide + return 0 + fi + + kubectl get nodes -o wide || true + docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \ + --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' || true + die "No se registraron y quedaron Ready los $((AGENT_COUNT + 1)) nodos esperados." +} + + +validate_runtime_after_boot() { + [[ "$CHECK_INGRESS_AFTER_BOOT" == "true" ]] || return 0 + + log "Validando Traefik, Argo CD y el puerto publicado" + + if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then + if ! kubectl rollout status deployment/traefik \ + -n kube-system \ + --timeout="${AGENT_RECOVERY_WAIT}s"; then + warn "Traefik no quedó disponible; reiniciando el Deployment." + kubectl rollout restart deployment/traefik -n kube-system + kubectl rollout status deployment/traefik \ + -n kube-system \ + --timeout="${WAIT_SECONDS}s" + fi + fi + + if kubectl get deployment/argocd-server -n argocd >/dev/null 2>&1; then + if ! kubectl rollout status deployment/argocd-server \ + -n argocd \ + --timeout="${AGENT_RECOVERY_WAIT}s"; then + warn "argocd-server no quedó disponible; reiniciando el Deployment." + kubectl rollout restart deployment/argocd-server -n argocd + kubectl rollout status deployment/argocd-server \ + -n argocd \ + --timeout="${WAIT_SECONDS}s" + fi + fi + + if ! curl -sS \ + --max-time 8 \ + -o /dev/null \ + -H "Host: ${ARGOCD_HOST}" \ + "http://127.0.0.1:${HTTP_PORT}/"; then + + warn "El Ingress no respondió en 127.0.0.1:${HTTP_PORT}; reiniciando serverlb y Traefik." + docker restart "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true + + if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then + kubectl rollout restart deployment/traefik -n kube-system + kubectl rollout status deployment/traefik \ + -n kube-system \ + --timeout="${WAIT_SECONDS}s" + fi + + sleep 10 + curl -sS \ + --max-time 8 \ + -o /dev/null \ + -H "Host: ${ARGOCD_HOST}" \ + "http://127.0.0.1:${HTTP_PORT}/" || + die "El Ingress sigue sin responder en el puerto ${HTTP_PORT}." + fi + + ok "Runtime, Traefik e Ingress disponibles." +} + +argocd_rollout_diagnostics() { + local resource="$1" + local namespace="$2" + + warn "Diagnóstico del rollout ${resource} en ${namespace}" + kubectl get "$resource" -n "$namespace" -o wide || true + kubectl get pods -n "$namespace" -o wide || true + kubectl get events -n "$namespace" \ + --sort-by='.lastTimestamp' 2>/dev/null | + tail -60 || true +} + +cleanup_terminating_argocd_server_pods() { + local desired available terminating_pods pod + + desired="$( + kubectl get deployment/argocd-server \ + -n argocd \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || true + )" + + available="$( + kubectl get deployment/argocd-server \ + -n argocd \ + -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true + )" + + desired="${desired:-1}" + available="${available:-0}" + + # Solo se fuerza la eliminación cuando la nueva réplica ya está disponible. + if (( available < desired )); then + return 1 + fi + + terminating_pods="$( + kubectl get pods \ + -n argocd \ + -l app.kubernetes.io/name=argocd-server \ + -o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null || true + )" + + [[ -n "$terminating_pods" ]] || return 1 + + while IFS= read -r pod; do + [[ -n "$pod" ]] || continue + warn "Eliminando Pod antiguo de argocd-server atascado en Terminating: ${pod}" + kubectl delete pod "$pod" \ + -n argocd \ + --grace-period=0 \ + --force || true + done <<< "$terminating_pods" + + return 0 +} + +wait_argocd_rollout() { + local kind="$1" + local name="$2" + local namespace="${3:-argocd}" + local resource="${kind}/${name}" + + if kubectl rollout status "$resource" \ + -n "$namespace" \ + --timeout="${ARGOCD_ROLLOUT_TIMEOUT}s"; then + return 0 + fi + + argocd_rollout_diagnostics "$resource" "$namespace" + + if [[ "$resource" == "deployment/argocd-server" ]] && + cleanup_terminating_argocd_server_pods; then + + log "Reintentando rollout de argocd-server después de limpiar la réplica antigua" + + kubectl rollout status "$resource" \ + -n "$namespace" \ + --timeout="180s" + return + fi + + die "El rollout ${resource} no quedó disponible." +} + +install_argocd() { + log "Instalando/reconciliando Argo CD ${ARGOCD_VERSION}" + + kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - + + kubectl apply \ + --server-side \ + --force-conflicts \ + -n argocd \ + -f "https://raw.githubusercontent.com/argoproj/argo-cd/${ARGOCD_VERSION}/manifests/install.yaml" + + local current_insecure + current_insecure="$( + kubectl get configmap argocd-cmd-params-cm \ + -n argocd \ + -o jsonpath='{.data.server\.insecure}' \ + 2>/dev/null || true + )" + + if [[ "$current_insecure" != "true" ]]; then + kubectl patch configmap argocd-cmd-params-cm \ + -n argocd \ + --type merge \ + -p '{"data":{"server.insecure":"true"}}' + + # Reiniciar únicamente cuando la configuración cambió. + kubectl rollout restart deployment/argocd-server -n argocd + else + ok "argocd-server ya está configurado con server.insecure=true; no se reinicia innecesariamente." + fi + + local deployments=( + argocd-server + argocd-repo-server + argocd-redis + argocd-dex-server + argocd-applicationset-controller + argocd-notifications-controller + ) + local deployment + + for deployment in "${deployments[@]}"; do + if kubectl get deployment "$deployment" -n argocd >/dev/null 2>&1; then + wait_argocd_rollout deployment "$deployment" argocd + fi + done + + if kubectl get statefulset argocd-application-controller -n argocd >/dev/null 2>&1; then + wait_argocd_rollout statefulset argocd-application-controller argocd + fi +} + +apply_argocd_ingress() { + log "Aplicando Ingress de Argo CD" + + cat < "${tmp_dir}/K8S_TOKEN.txt" + printf '%s\n' "$ca_b64" > "${tmp_dir}/K8S_CA_B64.txt" + printf '%s\n' "$server" > "${tmp_dir}/K8S_SERVER.txt" + printf '%s\n' "$LAB_HOST_IP" > "${tmp_dir}/K8S_TLS_SERVER_NAME.txt" + + cat > "$kubeconfig_file" < "${tmp_dir}/KUBE_CONFIG_DATA.txt" + printf '\n' >> "${tmp_dir}/KUBE_CONFIG_DATA.txt" + + for output_file in \ + K8S_TOKEN.txt \ + K8S_CA_B64.txt \ + K8S_SERVER.txt \ + K8S_TLS_SERVER_NAME.txt \ + KUBE_CONFIG_DATA.txt + do + sudo_cmd install \ + -m 0600 \ + -o "$owner" \ + "${tmp_dir}/${output_file}" \ + "${secret_dir}/${output_file}" + done + + # Los kubeconfig históricos también contienen credenciales. Ajustamos + # únicamente propietario y permisos; no asumimos ningún grupo llamado devops. + for output_file in kubeconfig-gitea.yaml lab-cluster.kubeconfig; do + if [[ -f "${secret_dir}/${output_file}" ]]; then + sudo_cmd chown "$owner" "${secret_dir}/${output_file}" + sudo_cmd chmod 0600 "${secret_dir}/${output_file}" + fi + done + + rm -rf "$tmp_dir" + + warn "Credenciales postdeploy regeneradas en ${secret_dir}." + warn "Actualiza en Gitea K8S_TOKEN/K8S_CA_B64/K8S_SERVER/K8S_TLS_SERVER_NAME o el secret histórico KUBE_CONFIG_DATA." +} + +apply_bootstrap_manifests() { + shopt -s nullglob + local manifests=("$BOOTSTRAP_DIR"/*.yaml "$BOOTSTRAP_DIR"/*.yml) + shopt -u nullglob + + if (( ${#manifests[@]} == 0 )); then + warn "No hay manifiestos GitOps en ${BOOTSTRAP_DIR}. Argo CD quedó instalado, pero las Applications deben aplicarse después." + return 0 + fi + + log "Aplicando manifiestos de bootstrap GitOps" + kubectl apply -f "$BOOTSTRAP_DIR" +} + +print_credentials() { + local argocd_password='pendiente' + + if kubectl get secret argocd-initial-admin-secret -n argocd >/dev/null 2>&1; then + argocd_password="$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 --decode)" + fi + + printf '\n==========================================================\n' + printf ' LABORATORIO DISPONIBLE\n' + printf ' ---------------------------------------------------------\n' + printf ' API Kubernetes : https://%s:%s\n' "$LAB_HOST_IP" "$API_PORT" + printf ' NPM Argo CD : http://%s:%s\n' "$LAB_HOST_IP" "$HTTP_PORT" + printf ' Host Argo CD : %s\n' "$ARGOCD_HOST" + printf ' ArgoCD admin : %s\n' "$argocd_password" + printf ' Token Headlamp : kubectl create token headlamp-admin -n headlamp\n' + printf ' Kubeconfig : %s\n' "$KUBECONFIG_PATH" + printf '==========================================================\n' +} + +status_report() { + require_commands + ensure_docker + + log "Docker" + docker info --format 'DockerRootDir={{.DockerRootDir}} Driver={{.Driver}}' + docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \ + --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' + + log "k3d" + k3d cluster list || true + + if cluster_exists; then + write_kubeconfig + log "Kubernetes" + kubectl get nodes -o wide || true + kubectl get pods -A || true + kubectl get ingress -A || true + kubectl get applications.argoproj.io -n argocd 2>/dev/null || true + fi +} + +backup_cluster_state() { + local stamp archive + stamp="$(date '+%Y%m%d-%H%M%S')" + archive="${BACKUP_DIR}/k3d-lab-state-${stamp}.tar.gz" + + log "Respaldando estado persistente en ${archive}" + + if cluster_exists; then + k3d cluster stop "$CLUSTER_NAME" || true + fi + + sudo_cmd tar \ + --acls \ + --xattrs \ + -C "$APP_DATA_PATH" \ + -czf "$archive" \ + k3s storage secrets config bootstrap 2>/dev/null || \ + sudo_cmd tar -C "$APP_DATA_PATH" -czf "$archive" k3s storage secrets config bootstrap + + sudo_cmd chown "$(id -un)" "$archive" + ok "Backup creado: ${archive}" +} + +reset_cluster() { + [[ "${CONFIRM_RESET:-}" == "DELETE-${CLUSTER_NAME}" ]] || \ + die "Reset bloqueado. Usa CONFIRM_RESET=DELETE-${CLUSTER_NAME} $0 reset" + + prepare_directories + backup_cluster_state + + if cluster_exists; then + k3d cluster delete "$CLUSTER_NAME" + fi + + local stamp="$(date '+%Y%m%d-%H%M%S')" + sudo_cmd mv "$K3S_SERVER_PATH" "${K3S_SERVER_PATH}.reset-${stamp}" + sudo_cmd mv "${APP_DATA_PATH}/storage" "${APP_DATA_PATH}/storage.reset-${stamp}" + sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH" + sudo_cmd install -d -m 0755 "$K3S_SERVER_STORAGE" "$K3S_AGENT0_STORAGE" "$K3S_AGENT1_STORAGE" + + create_cluster +} + +ensure_cluster_only() { + require_commands + ensure_docker + validate_data_mount + validate_host_ip + + if ! cluster_exists; then + if cluster_containers_exist; then + die "Existen contenedores del lab, pero k3d no reconoce el clúster. Se requiere diagnóstico manual; no se crearán duplicados." + fi + die "El clúster ${CLUSTER_NAME} no existe. Ejecuta bootstrap manualmente." + fi + + start_cluster + set_restart_policy + write_kubeconfig + wait_for_api + wait_for_nodes + validate_runtime_after_boot +} + + +resolve_service_tool() { + local command_name="$1" + local explicit_path="${2:-}" + local service_home="$3" + local candidate="" + + for candidate in \ + "$explicit_path" \ + "$(command -v "$command_name" 2>/dev/null || true)" \ + "${service_home}/.local/bin/${command_name}" \ + "${service_home}/bin/${command_name}" \ + "/DATA/.local/bin/${command_name}" \ + "/DATA/bin/${command_name}" \ + "/DATA/AppData/bin/${command_name}" \ + "/usr/local/bin/${command_name}" \ + "/usr/local/sbin/${command_name}" \ + "/usr/bin/${command_name}" \ + "/usr/sbin/${command_name}" \ + "/bin/${command_name}" \ + "/sbin/${command_name}" + do + [[ -n "$candidate" ]] || continue + if [[ -x "$candidate" ]]; then + readlink -f "$candidate" 2>/dev/null || printf '%s\n' "$candidate" + return 0 + fi + done + + return 1 +} + +append_unique_path_dir() { + local current_path="$1" + local new_dir="$2" + + [[ -n "$new_dir" ]] || { + printf '%s\n' "$current_path" + return + } + + case ":${current_path}:" in + *":${new_dir}:"*) + printf '%s\n' "$current_path" + ;; + *) + if [[ -n "$current_path" ]]; then + printf '%s:%s\n' "$current_path" "$new_dir" + else + printf '%s\n' "$new_dir" + fi + ;; + esac +} + +install_autostart() { + [[ $(id -u) -eq 0 ]] || die "Ejecuta install-autostart con sudo." + + local service_user="${SUDO_USER:-devops}" + local service_home service_group docker_root installed_script + local k3d_path kubectl_path docker_path curl_path tool_path tool_dir + + service_home="$(getent passwd "$service_user" | cut -d: -f6)" + service_group="$(id -gn "$service_user")" + docker_root="$(docker info --format '{{.DockerRootDir}}')" + installed_script="${APP_DATA_PATH}/bin/deploy-k3d-lab" + + [[ -n "$service_home" ]] || die "No fue posible determinar HOME de ${service_user}." + [[ -n "$service_group" ]] || die "No fue posible determinar el grupo de ${service_user}." + [[ -n "$docker_root" ]] || die "No fue posible determinar DockerRootDir." + + k3d_path="$( + resolve_service_tool k3d "${K3D_BIN:-}" "$service_home" + )" || die "No se encontró k3d para systemd. Usa K3D_BIN=\$(command -v k3d) con sudo -E." + + kubectl_path="$( + resolve_service_tool kubectl "${KUBECTL_BIN:-}" "$service_home" + )" || die "No se encontró kubectl para systemd." + + docker_path="$( + resolve_service_tool docker "${DOCKER_BIN:-}" "$service_home" + )" || die "No se encontró docker para systemd." + + curl_path="$( + resolve_service_tool curl "${CURL_BIN:-}" "$service_home" + )" || die "No se encontró curl para systemd." + + tool_path="${APP_DATA_PATH}/bin" + for tool_dir in "$(dirname "$k3d_path")" "$(dirname "$kubectl_path")" "$(dirname "$docker_path")" "$(dirname "$curl_path")" /usr/local/sbin /usr/local/bin /usr/sbin /usr/bin /sbin /bin + do + tool_path="$(append_unique_path_dir "$tool_path" "$tool_dir")" + done + + # ZimaOS no garantiza /usr/local/sbin y su filesystem base puede ser + # inmutable. Instalamos el ejecutable en /DATA, que es persistente. + install -d \ + -m 0750 \ + -o "$service_user" \ + "$(dirname "$installed_script")" + + install \ + -m 0750 \ + -o "$service_user" \ + "$0" \ + "$installed_script" + + # El servicio se ejecutará como devops y no puede usar sudo sin TTY. + # Dejamos listas y escribibles todas las rutas que el modo ensure modifica. + install -d \ + -m 0750 \ + -o "$service_user" \ + "$APP_DATA_PATH" + + install -d \ + -m 0700 \ + -o "$service_user" \ + "$(dirname "$KUBECONFIG_PATH")" + + if [[ -f "$KUBECONFIG_PATH" ]]; then + chown "$service_user" "$KUBECONFIG_PATH" + chmod 0600 "$KUBECONFIG_PATH" + fi + + install -d -m 0755 \ + /etc/default \ + /etc/systemd/system \ + /etc/systemd/system/docker.service.d + + systemctl enable docker.service >/dev/null + if systemctl list-unit-files containerd.service >/dev/null 2>&1; then + systemctl enable containerd.service >/dev/null || true + fi + + cat > /etc/systemd/system/docker.service.d/10-k3d-storage.conf < /etc/default/k3d-lab < /etc/systemd/system/k3d-lab-ensure.service < /etc/systemd/system/k3d-lab-ensure.timer <<'UNIT' +[Unit] +Description=Watchdog del laboratorio k3d + +[Timer] +OnBootSec=90s +OnUnitInactiveSec=5min +AccuracySec=20s +Unit=k3d-lab-ensure.service + +[Install] +WantedBy=timers.target +UNIT + + systemctl daemon-reload + systemctl enable --now k3d-lab-ensure.timer + + if ! systemctl start k3d-lab-ensure.service; then + systemctl status k3d-lab-ensure.service --no-pager -l || true + journalctl -u k3d-lab-ensure.service -n 200 --no-pager || true + die "La prueba inicial de k3d-lab-ensure.service falló." + fi + + ok "Autostart instalado para ${service_user}." + printf 'Script persistente: %s\n' "$installed_script" + printf 'k3d para systemd: %s\n' "$k3d_path" + printf 'kubectl para systemd: %s\n' "$kubectl_path" + printf 'PATH para systemd: %s\n' "$tool_path" + printf 'DockerRootDir protegido por systemd: %s\n' "$docker_root" + printf 'Logs: journalctl -u k3d-lab-ensure.service -n 200 --no-pager\n' + printf 'Nota: el drop-in de Docker será plenamente verificable después del reinicio de prueba.\n' +} + +main() { + case "$MODE" in + status) + status_report + return + ;; + install-autostart) + install_autostart + return + ;; + ensure) + ensure_cluster_only + return + ;; + esac + + require_commands + ensure_docker + validate_data_mount + validate_host_ip + prepare_directories + ensure_cluster_token + write_k3d_config + + case "$MODE" in + bootstrap) + if cluster_exists; then + start_cluster + else + create_cluster + fi + ;; + recover) + cluster_exists || die "No existe ${CLUSTER_NAME}; utiliza bootstrap para crearlo." + start_cluster + ;; + reset) + reset_cluster + ;; + *) + die "Modo no válido: ${MODE}. Usa bootstrap, recover, ensure, status, install-autostart o reset." + ;; + esac + + set_restart_policy + write_kubeconfig + wait_for_api + wait_for_nodes + + install_argocd + apply_argocd_ingress + install_headlamp_rbac + apply_postdeploy_rbac + generate_postdeploy_credentials + apply_bootstrap_manifests + validate_runtime_after_boot + + log "Validación final" + kubectl get nodes -o wide + kubectl get pods -n argocd + kubectl get ingress -A + kubectl get applications.argoproj.io -n argocd 2>/dev/null || true + + print_credentials +} + +main "$@"