devops/scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b2c6e2806863a8dfa2e88ab2a4bd73a16826d0da
scripts/deploy-lab.sh
T
devops 62385a475b Update deploy-lab.sh
2026-07-18 01:03:47 +00:00

2946 lines
88 KiB
Bash
Raw Blame History

#!/usr/bin/env bash
# Laboratorio estable v4.4.3: ZimaOS + Docker + k3d + Argo CD + Multi-Repo + Governance + Monitoring Recovery + Runner
#
# Modos:
# bootstrap Crea el clúster si no existe; si existe, lo inicia y reconcilia.
# recover Recupera únicamente un clúster existente.
# ensure Health check no destructivo para systemd; no instala Argo CD.
# status Muestra diagnóstico del laboratorio.
# repo-sync Clona/actualiza apps-registry y platform-infra.
# gitops Actualiza el repo y aplica root-apps-registry.
# monitoring-diagnose Diagnostica kube-prometheus-stack/Grafana.
# monitoring-recover Finaliza una operación obsoleta y resincroniza Grafana.
# runner Adopta, crea o recupera el Gitea Actions runner.
# install-autostart Instala servicio/timer systemd, orden de montajes y autorreparación.
# reset Reconstrucción limpia; requiere confirmación explícita.
#
# Ejemplo:
# LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh bootstrap
set -Eeuo pipefail
IFS=$'\n\t'
MODE="${1:-bootstrap}"
if [[ -z "${BASH_VERSION:-}" ]]; then
printf 'ERROR: este script requiere Bash. Ejecútalo con: bash deploy-lab.sh %s\n' "$MODE" >&2
exit 1
fi
if [[ $(id -u) -eq 0 && "$MODE" != "install-autostart" ]]; then
printf 'ERROR: ejecuta el modo %s como usuario normal, sin sudo.\n' "$MODE" >&2
printf 'Ejemplo: LAB_HOST_IP=192.168.68.61 ./deploy-lab.sh %s\n' "$MODE" >&2
exit 1
fi
CLUSTER_NAME="${CLUSTER_NAME:-lab-cluster}"
APP_DATA_PATH="${APP_DATA_PATH:-/DATA/AppData/k3d-lab}"
KUBECONFIG_PATH="${KUBECONFIG_PATH:-/DATA/.kube/config}"
CONFIG_FILE="${CONFIG_FILE:-${APP_DATA_PATH}/config/k3d-lab.yaml}"
BOOTSTRAP_DIR="${BOOTSTRAP_DIR:-${APP_DATA_PATH}/bootstrap}"
BACKUP_DIR="${BACKUP_DIR:-${APP_DATA_PATH}/backups}"
API_PORT="${API_PORT:-46421}"
HTTP_PORT="${HTTP_PORT:-90}"
HTTPS_PORT="${HTTPS_PORT:-9443}"
AGENT_COUNT="${AGENT_COUNT:-2}"
K3D_SUBNET="${K3D_SUBNET:-auto}"
K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.35.5-k3s1}"
ARGOCD_VERSION="${ARGOCD_VERSION:-v3.2.0}"
ARGOCD_HOST="${ARGOCD_HOST:-argocd.cruzcloud.net}"
EXPECTED_HOST_IP="${EXPECTED_HOST_IP:-192.168.68.61}"
ALLOW_IP_CHANGE="${ALLOW_IP_CHANGE:-false}"
INSTALL_HEADLAMP_RBAC="${INSTALL_HEADLAMP_RBAC:-true}"
APPLY_POSTDEPLOY_RBAC="${APPLY_POSTDEPLOY_RBAC:-true}"
WAIT_SECONDS="${WAIT_SECONDS:-240}"
ARGOCD_ROLLOUT_TIMEOUT="${ARGOCD_ROLLOUT_TIMEOUT:-420}"
AGENT_RECOVERY_WAIT="${AGENT_RECOVERY_WAIT:-75}"
AUTO_REREGISTER_STALE_AGENTS="${AUTO_REREGISTER_STALE_AGENTS:-true}"
CHECK_INGRESS_AFTER_BOOT="${CHECK_INGRESS_AFTER_BOOT:-true}"
# Bootstrap GitOps / App-of-Apps.
BOOTSTRAP_ROOT_APP="${BOOTSTRAP_ROOT_APP:-true}"
ROOT_APP_NAME="${ROOT_APP_NAME:-root-apps-registry}"
ROOT_APP_REPO_URL="${ROOT_APP_REPO_URL:-https://gitea.cruzcloud.net/devops/apps-registry.git}"
ROOT_APP_REPO_PATH="${ROOT_APP_REPO_PATH:-apps}"
ROOT_APP_REVISION="${ROOT_APP_REVISION:-main}"
ROOT_APP_PROJECT="${ROOT_APP_PROJECT:-default}"
ROOT_APP_MANIFEST="${ROOT_APP_MANIFEST:-${BOOTSTRAP_DIR}/00-root-apps-registry.yaml}"
ROOT_APP_WAIT_SECONDS="${ROOT_APP_WAIT_SECONDS:-600}"
GITOPS_APPS_WAIT_SECONDS="${GITOPS_APPS_WAIT_SECONDS:-900}"
GITOPS_STRICT_HEALTH="${GITOPS_STRICT_HEALTH:-false}"
EXPECTED_GITOPS_APPS="${EXPECTED_GITOPS_APPS:-}"
ARGOCD_REPO_SECRET_NAME="${ARGOCD_REPO_SECRET_NAME:-apps-registry-repository}"
ARGOCD_REPO_USERNAME_FILE="${ARGOCD_REPO_USERNAME_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-username}"
ARGOCD_REPO_PASSWORD_FILE="${ARGOCD_REPO_PASSWORD_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-password}"
# Copia local persistente del repositorio GitOps.
APP_REGISTRY_CACHE_DIR="${APP_REGISTRY_CACHE_DIR:-${APP_DATA_PATH}/git/apps-registry}"
APP_REGISTRY_ROOT_MANIFEST="${APP_REGISTRY_ROOT_MANIFEST:-application.yaml}"
APP_REGISTRY_COMMIT_FILE="${APP_REGISTRY_COMMIT_FILE:-${APP_DATA_PATH}/git/apps-registry.commit}"
APP_REGISTRY_ALLOW_CACHE_FALLBACK="${APP_REGISTRY_ALLOW_CACHE_FALLBACK:-true}"
APP_REGISTRY_REFRESH_ON_ENSURE="${APP_REGISTRY_REFRESH_ON_ENSURE:-false}"
APP_REGISTRY_CLONE_DEPTH="${APP_REGISTRY_CLONE_DEPTH:-1}"
# Repositorio de governance/Kustomize.
PLATFORM_INFRA_REPO_URL="${PLATFORM_INFRA_REPO_URL:-https://gitea.cruzcloud.net/devops/platform-infra.git}"
PLATFORM_INFRA_REVISION="${PLATFORM_INFRA_REVISION:-main}"
PLATFORM_INFRA_CACHE_DIR="${PLATFORM_INFRA_CACHE_DIR:-${APP_DATA_PATH}/git/platform-infra}"
PLATFORM_INFRA_COMMIT_FILE="${PLATFORM_INFRA_COMMIT_FILE:-${APP_DATA_PATH}/git/platform-infra.commit}"
PLATFORM_INFRA_ALLOW_CACHE_FALLBACK="${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK:-true}"
PLATFORM_INFRA_REFRESH_ON_ENSURE="${PLATFORM_INFRA_REFRESH_ON_ENSURE:-false}"
PLATFORM_INFRA_CLONE_DEPTH="${PLATFORM_INFRA_CLONE_DEPTH:-1}"
PLATFORM_INFRA_RENDER_DIR="${PLATFORM_INFRA_RENDER_DIR:-${APP_DATA_PATH}/rendered/platform-infra}"
APPLY_GOVERNANCE_SNAPSHOTS="${APPLY_GOVERNANCE_SNAPSHOTS:-true}"
ENFORCE_MONITORING_LIMITRANGE="${ENFORCE_MONITORING_LIMITRANGE:-true}"
MONITORING_GOVERNANCE_PATH="${MONITORING_GOVERNANCE_PATH:-monitoring-governance}"
ARGOCD_PLATFORM_REPO_SECRET_NAME="${ARGOCD_PLATFORM_REPO_SECRET_NAME:-platform-infra-repository}"
# Diagnóstico de monitoring/Grafana.
MONITORING_APP_NAME="${MONITORING_APP_NAME:-monitoring-app}"
MONITORING_NAMESPACE="${MONITORING_NAMESPACE:-monitoring}"
MONITORING_FIXED_CHART_VERSION="${MONITORING_FIXED_CHART_VERSION:-58.3.3}"
MONITORING_INGRESS_HOST="${MONITORING_INGRESS_HOST:-grafana.cruzcloud.net}"
MONITORING_RECOVERY_WAIT_SECONDS="${MONITORING_RECOVERY_WAIT_SECONDS:-900}"
MONITORING_OPERATION_STOP_WAIT="${MONITORING_OPERATION_STOP_WAIT:-180}"
AUTO_RECOVER_STALE_MONITORING_OPERATION="${AUTO_RECOVER_STALE_MONITORING_OPERATION:-true}"
MONITORING_FORCE_RECOVERY="${MONITORING_FORCE_RECOVERY:-false}"
ARGOCD_CLI_IMAGE="${ARGOCD_CLI_IMAGE:-quay.io/argoproj/argocd:${ARGOCD_VERSION}}"
# Gitea Actions runner externo al clúster.
INSTALL_GITEA_RUNNER="${INSTALL_GITEA_RUNNER:-true}"
GITEA_RUNNER_REQUIRED="${GITEA_RUNNER_REQUIRED:-true}"
GITEA_RUNNER_CONTAINER="${GITEA_RUNNER_CONTAINER:-gitea-runner}"
GITEA_RUNNER_IMAGE="${GITEA_RUNNER_IMAGE:-docker.io/gitea/act_runner:latest}"
GITEA_INSTANCE_URL="${GITEA_INSTANCE_URL:-https://gitea.cruzcloud.net}"
GITEA_RUNNER_NAME="${GITEA_RUNNER_NAME:-zimaos-lab-runner}"
GITEA_RUNNER_LABELS="${GITEA_RUNNER_LABELS:-ubuntu-latest:docker://node:20-bookworm}"
GITEA_RUNNER_ROOT="${GITEA_RUNNER_ROOT:-${APP_DATA_PATH}/gitea-runner}"
GITEA_RUNNER_DATA_DIR="${GITEA_RUNNER_DATA_DIR:-${GITEA_RUNNER_ROOT}/data}"
GITEA_RUNNER_IMAGE_FILE="${GITEA_RUNNER_IMAGE_FILE:-${GITEA_RUNNER_ROOT}/image.txt}"
GITEA_RUNNER_TOKEN_FILE="${GITEA_RUNNER_TOKEN_FILE:-${APP_DATA_PATH}/secrets/gitea-runner-registration-token}"
MIGRATE_EXISTING_RUNNER="${MIGRATE_EXISTING_RUNNER:-false}"
K3S_SERVER_PATH="${APP_DATA_PATH}/k3s/server"
K3S_SERVER_STORAGE="${APP_DATA_PATH}/storage/server-0"
K3S_AGENT0_STORAGE="${APP_DATA_PATH}/storage/agent-0"
K3S_AGENT1_STORAGE="${APP_DATA_PATH}/storage/agent-1"
TOKEN_FILE="${APP_DATA_PATH}/secrets/k3s-cluster-token"
POSTDEPLOY_RBAC_FILE="${APP_DATA_PATH}/gitea-postdeploy-validator-rbac.yaml"
if [[ -t 1 ]]; then
C_RESET=$'\033[0m'
C_BLUE=$'\033[1;34m'
C_GREEN=$'\033[1;32m'
C_YELLOW=$'\033[1;33m'
C_RED=$'\033[1;31m'
else
C_RESET=''
C_BLUE=''
C_GREEN=''
C_YELLOW=''
C_RED=''
fi
log() { printf '\n%s[%s] %s%s\n' "$C_BLUE" "$(date '+%F %T')" "$*" "$C_RESET"; }
ok() { printf '%sOK:%s %s\n' "$C_GREEN" "$C_RESET" "$*"; }
warn() { printf '%sAVISO:%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; }
die() { printf '%sERROR:%s %s\n' "$C_RED" "$C_RESET" "$*" >&2; exit 1; }
on_error() {
local exit_code=$?
printf '%sERROR:%s fallo en la línea %s: %s\n' \
"$C_RED" "$C_RESET" "${BASH_LINENO[0]:-desconocida}" "${BASH_COMMAND:-desconocido}" >&2
exit "$exit_code"
}
trap on_error ERR
have() { command -v "$1" >/dev/null 2>&1; }
sudo_cmd() {
if [[ $(id -u) -eq 0 ]]; then
"$@"
else
sudo "$@"
fi
}
require_commands() {
local command_name
for command_name in docker k3d kubectl curl git awk sed grep findmnt ss tar base64 tr; do
have "$command_name" || die "No se encontró el comando requerido: ${command_name}"
done
}
detect_host_ip() {
ip -4 route get 1.1.1.1 2>/dev/null | awk '{
for (i = 1; i <= NF; i++) {
if ($i == "src") {
print $(i + 1)
exit
}
}
}'
}
LAB_HOST_IP="${LAB_HOST_IP:-$(detect_host_ip)}"
export LAB_HOST_IP CLUSTER_NAME APP_DATA_PATH K3D_SUBNET K3S_IMAGE AGENT_COUNT API_PORT HTTP_PORT HTTPS_PORT
validate_host_ip() {
[[ -n "$LAB_HOST_IP" ]] || die "No fue posible detectar LAB_HOST_IP. Defínela manualmente."
if [[ -n "$EXPECTED_HOST_IP" && "$LAB_HOST_IP" != "$EXPECTED_HOST_IP" ]]; then
if [[ "$ALLOW_IP_CHANGE" == "true" ]]; then
warn "La IP detectada ${LAB_HOST_IP} no coincide con ${EXPECTED_HOST_IP}; se continuará por ALLOW_IP_CHANGE=true."
else
die "La IP detectada es ${LAB_HOST_IP}, pero NPM/Gitea esperan ${EXPECTED_HOST_IP}. Reserva la IP o usa ALLOW_IP_CHANGE=true conscientemente."
fi
fi
}
ensure_docker() {
if have systemctl && ! systemctl is-active --quiet docker.service; then
if [[ $(id -u) -eq 0 || -t 0 ]]; then
log "Iniciando Docker"
sudo_cmd systemctl start docker.service
else
die "Docker no está activo. systemd debe iniciarlo antes de ejecutar el watchdog."
fi
fi
docker info >/dev/null 2>&1 ||
die "Docker no responde para el usuario $(id -un). Valida docker.service y el grupo docker."
}
validate_data_mount() {
findmnt -T /DATA >/dev/null 2>&1 ||
die "/DATA no está montado. No se iniciará k3d para evitar escribir en el filesystem raíz."
[[ -d "$APP_DATA_PATH" ]] ||
die "No existe la ruta persistente ${APP_DATA_PATH}."
# En ZimaOS el usuario devops no puede escribir directamente en /DATA,
# pero sí debe poder escribir en su ruta del laboratorio.
local test_file="${APP_DATA_PATH}/.k3d-write-test.$$"
touch "$test_file" 2>/dev/null ||
die "${APP_DATA_PATH} no permite escritura para $(id -un)."
rm -f "$test_file"
}
prepare_directories() {
log "Preparando almacenamiento persistente"
sudo_cmd install -d -m 0750 -o "$(id -un)" \
"$APP_DATA_PATH" \
"${APP_DATA_PATH}/config" \
"${APP_DATA_PATH}/secrets" \
"$BOOTSTRAP_DIR" \
"$BACKUP_DIR" \
"$GITEA_RUNNER_ROOT" \
"$GITEA_RUNNER_DATA_DIR" \
"$(dirname "$APP_REGISTRY_CACHE_DIR")" \
"$(dirname "$PLATFORM_INFRA_CACHE_DIR")" \
"$PLATFORM_INFRA_RENDER_DIR" \
"$(dirname "$KUBECONFIG_PATH")"
# K3s y los workloads escriben como root dentro de los nodos.
sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH"
sudo_cmd install -d -m 0755 \
"$K3S_SERVER_STORAGE" \
"$K3S_AGENT0_STORAGE" \
"$K3S_AGENT1_STORAGE"
sudo_cmd chown "$(id -un)" "$(dirname "$KUBECONFIG_PATH")"
}
generate_cluster_token() {
local token=""
if have openssl; then
token="$(openssl rand -hex 32)"
elif have od; then
token="$(
od -An -N32 -tx1 /dev/urandom |
awk '{
for (i = 1; i <= NF; i++) {
printf "%s", $i
}
}
END {
printf "\n"
}'
)"
elif have hexdump; then
token="$(hexdump -vn32 -e '1/1 "%02x"' /dev/urandom)"
else
die "No hay un generador aleatorio disponible. Se requiere openssl, od o hexdump."
fi
[[ "$token" =~ ^[0-9a-fA-F]{64}$ ]] ||
die "No fue posible generar un token hexadecimal seguro."
printf '%s\n' "$token"
}
ensure_cluster_token() {
if [[ ! -s "$TOKEN_FILE" ]]; then
log "Generando token persistente del clúster"
umask 077
generate_cluster_token > "$TOKEN_FILE"
chmod 0600 "$TOKEN_FILE"
fi
K3S_CLUSTER_TOKEN="$(<"$TOKEN_FILE")"
[[ "$K3S_CLUSTER_TOKEN" =~ ^[0-9a-fA-F]{64}$ ]] ||
die "El token persistente de K3s no es válido."
export K3S_CLUSTER_TOKEN
}
write_k3d_config() {
log "Generando configuración declarativa de k3d"
cat > "$CONFIG_FILE" <<'YAML'
apiVersion: k3d.io/v1alpha5
kind: Simple
metadata:
name: ${CLUSTER_NAME}
servers: 1
agents: ${AGENT_COUNT}
image: ${K3S_IMAGE}
subnet: "${K3D_SUBNET}"
token: "${K3S_CLUSTER_TOKEN}"
kubeAPI:
host: "${LAB_HOST_IP}"
hostIP: "0.0.0.0"
hostPort: "${API_PORT}"
ports:
- port: "${HTTP_PORT}:80"
nodeFilters:
- loadbalancer
- port: "${HTTPS_PORT}:443"
nodeFilters:
- loadbalancer
volumes:
# Conserva SQLite, certificados y token de K3s fuera de los contenedores.
- volume: "${APP_DATA_PATH}/k3s/server:/var/lib/rancher/k3s/server"
nodeFilters:
- server:0
# Conserva datos de PVC creados por local-path-provisioner.
- volume: "${APP_DATA_PATH}/storage/server-0:/var/lib/rancher/k3s/storage"
nodeFilters:
- server:0
- volume: "${APP_DATA_PATH}/storage/agent-0:/var/lib/rancher/k3s/storage"
nodeFilters:
- agent:0
- volume: "${APP_DATA_PATH}/storage/agent-1:/var/lib/rancher/k3s/storage"
nodeFilters:
- agent:1
options:
k3d:
wait: true
timeout: "240s"
disableRollback: false
k3s:
extraArgs:
- arg: "--tls-san=${LAB_HOST_IP}"
nodeFilters:
- server:*
kubeconfig:
updateDefaultKubeconfig: false
switchCurrentContext: false
YAML
chmod 0600 "$CONFIG_FILE"
}
cluster_exists() {
k3d cluster list --no-headers 2>/dev/null | awk '{print $1}' | grep -Fxq "$CLUSTER_NAME"
}
cluster_containers_exist() {
docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" --format '{{.ID}}' | grep -q .
}
ports_in_use_by_non_cluster_process() {
local port
for port in "$API_PORT" "$HTTP_PORT" "$HTTPS_PORT"; do
if ss -lntH "sport = :${port}" 2>/dev/null | grep -q .; then
return 0
fi
done
return 1
}
create_cluster() {
if cluster_containers_exist; then
die "Existen contenedores etiquetados para ${CLUSTER_NAME}, pero k3d no reconoce el clúster. No se crearán duplicados."
fi
if ports_in_use_by_non_cluster_process; then
sudo_cmd ss -lntp | grep -E ":(${API_PORT}|${HTTP_PORT}|${HTTPS_PORT})\\b" || true
die "Uno o más puertos del laboratorio están ocupados."
fi
log "Creando clúster ${CLUSTER_NAME} con IPAM estático"
docker pull "$K3S_IMAGE"
k3d cluster create --config "$CONFIG_FILE"
}
start_cluster() {
log "Iniciando clúster existente ${CLUSTER_NAME}"
k3d cluster start "$CLUSTER_NAME" --wait --timeout "${WAIT_SECONDS}s" || true
}
set_restart_policy() {
mapfile -t cluster_containers < <(
docker ps -aq --filter "label=k3d.cluster=${CLUSTER_NAME}"
)
if (( ${#cluster_containers[@]} > 0 )); then
docker update --restart=unless-stopped "${cluster_containers[@]}" >/dev/null
ok "Política Docker unless-stopped aplicada a ${#cluster_containers[@]} contenedores del clúster."
fi
}
write_kubeconfig() {
local temp_file context_name configured_server
temp_file="$(mktemp)"
k3d kubeconfig get "$CLUSTER_NAME" > "$temp_file"
[[ -s "$temp_file" ]] || die "k3d no generó el kubeconfig."
context_name="$(
KUBECONFIG="$temp_file" \
kubectl config get-contexts -o name 2>/dev/null |
head -1
)"
[[ -n "$context_name" ]] ||
die "El kubeconfig generado no contiene ningún contexto."
# Algunas combinaciones de k3d/kubectl pueden producir un archivo válido
# sin current-context. Lo fijamos explícitamente antes de instalarlo.
KUBECONFIG="$temp_file" \
kubectl config use-context "$context_name" >/dev/null
configured_server="$(
KUBECONFIG="$temp_file" \
kubectl config view \
--minify \
-o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null
)"
[[ -n "$configured_server" ]] ||
die "El kubeconfig generado no contiene un API Server válido."
# El watchdog se ejecuta sin TTY y no debe invocar sudo.
# install-autostart garantiza previamente el propietario de esta ruta.
install -d \
-m 0700 \
"$(dirname "$KUBECONFIG_PATH")" ||
die "No se pudo preparar $(dirname "$KUBECONFIG_PATH") como $(id -un)."
install \
-m 0600 \
"$temp_file" \
"$KUBECONFIG_PATH" ||
die "No se pudo actualizar ${KUBECONFIG_PATH} como $(id -un)."
rm -f "$temp_file"
export KUBECONFIG="$KUBECONFIG_PATH"
[[ "$(kubectl config current-context 2>/dev/null || true)" == "$context_name" ]] ||
die "No fue posible activar el contexto ${context_name}."
ok "Kubeconfig actualizado: ${KUBECONFIG_PATH}"
ok "Contexto activo: ${context_name}"
ok "API Server: ${configured_server}"
}
wait_for_api() {
log "Esperando API de Kubernetes"
local deadline=$((SECONDS + WAIT_SECONDS))
while (( SECONDS < deadline )); do
if kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1; then
ok "API de Kubernetes disponible."
return 0
fi
sleep 5
done
docker logs --tail 200 "k3d-${CLUSTER_NAME}-server-0" || true
die "La API no quedó disponible dentro de ${WAIT_SECONDS}s."
}
node_ready() {
local node_name="$1"
[[ "$(kubectl get node "$node_name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' \
2>/dev/null || true)" == "True" ]]
}
wait_agent_ready() {
local node_name="$1"
local timeout_seconds="$2"
local deadline=$((SECONDS + timeout_seconds))
while (( SECONDS < deadline )); do
if node_ready "$node_name"; then
return 0
fi
sleep 5
done
return 1
}
recover_not_ready_agents() {
local index node_name container_name docker_ip kubernetes_ip
# El servidor y el API deben estar sanos antes de reparar agentes.
kubectl get --raw='/readyz' --request-timeout='5s' >/dev/null 2>&1 ||
die "No se repararán agentes porque el API Server no está disponible."
for (( index = 0; index < AGENT_COUNT; index++ )); do
node_name="k3d-${CLUSTER_NAME}-agent-${index}"
container_name="$node_name"
if node_ready "$node_name"; then
continue
fi
warn "${node_name} no existe o no está Ready; reiniciando el contenedor."
docker restart "$container_name" >/dev/null || true
if wait_agent_ready "$node_name" "$AGENT_RECOVERY_WAIT"; then
ok "${node_name} se recuperó después del reinicio."
continue
fi
docker_ip="$(
docker inspect \
-f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \
"$container_name" 2>/dev/null || true
)"
kubernetes_ip="$(
kubectl get node "$node_name" \
-o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' \
2>/dev/null || true
)"
warn "${node_name} continúa sin estar Ready. DockerIP=${docker_ip:-ausente}, KubernetesIP=${kubernetes_ip:-ausente}."
if [[ "$AUTO_REREGISTER_STALE_AGENTS" != "true" ]]; then
continue
fi
# En un lab, si el agente lleva el periodo de gracia completo sin responder,
# se elimina únicamente su objeto Node y password secret para permitir
# un registro limpio con la IP estática actual.
warn "Forzando re-registro seguro de ${node_name}."
kubectl delete node "$node_name" --ignore-not-found=true >/dev/null
kubectl delete secret \
-n kube-system \
"${node_name}.node-password.k3s" \
--ignore-not-found=true >/dev/null
docker restart "$container_name" >/dev/null
if wait_agent_ready "$node_name" "$((AGENT_RECOVERY_WAIT * 2))"; then
ok "${node_name} volvió a registrarse y está Ready."
else
docker logs --tail 180 "$container_name" || true
die "${node_name} no se recuperó después del re-registro."
fi
done
if ! docker ps --format '{{.Names}}' |
grep -Fxq "k3d-${CLUSTER_NAME}-serverlb"; then
warn "El load balancer no está activo; intentando iniciarlo."
docker start "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true
fi
}
nodes_are_ready() {
local expected_nodes=$((AGENT_COUNT + 1))
local actual_nodes ready_nodes
actual_nodes="$(kubectl get nodes --no-headers 2>/dev/null | awk 'NF {count++} END {print count+0}')"
ready_nodes="$(kubectl get nodes -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep -c '^True$' || true)"
[[ "$actual_nodes" -eq "$expected_nodes" && "$ready_nodes" -eq "$expected_nodes" ]]
}
wait_for_nodes_once() {
local deadline=$((SECONDS + WAIT_SECONDS))
while (( SECONDS < deadline )); do
if nodes_are_ready; then
return 0
fi
sleep 5
done
return 1
}
wait_for_nodes() {
log "Esperando $((AGENT_COUNT + 1)) nodos registrados y Ready"
if wait_for_nodes_once; then
kubectl get nodes -o wide
return 0
fi
recover_not_ready_agents
if wait_for_nodes_once; then
kubectl get nodes -o wide
return 0
fi
kubectl get nodes -o wide || true
docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \
--format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' || true
die "No se registraron y quedaron Ready los $((AGENT_COUNT + 1)) nodos esperados."
}
validate_runtime_after_boot() {
[[ "$CHECK_INGRESS_AFTER_BOOT" == "true" ]] || return 0
log "Validando Traefik, Argo CD y el puerto publicado"
if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then
if ! kubectl rollout status deployment/traefik \
-n kube-system \
--timeout="${AGENT_RECOVERY_WAIT}s"; then
warn "Traefik no quedó disponible; reiniciando el Deployment."
kubectl rollout restart deployment/traefik -n kube-system
kubectl rollout status deployment/traefik \
-n kube-system \
--timeout="${WAIT_SECONDS}s"
fi
fi
if kubectl get deployment/argocd-server -n argocd >/dev/null 2>&1; then
if ! kubectl rollout status deployment/argocd-server \
-n argocd \
--timeout="${AGENT_RECOVERY_WAIT}s"; then
warn "argocd-server no quedó disponible; reiniciando el Deployment."
kubectl rollout restart deployment/argocd-server -n argocd
kubectl rollout status deployment/argocd-server \
-n argocd \
--timeout="${WAIT_SECONDS}s"
fi
fi
if ! curl -sS \
--max-time 8 \
-o /dev/null \
-H "Host: ${ARGOCD_HOST}" \
"http://127.0.0.1:${HTTP_PORT}/"; then
warn "El Ingress no respondió en 127.0.0.1:${HTTP_PORT}; reiniciando serverlb y Traefik."
docker restart "k3d-${CLUSTER_NAME}-serverlb" >/dev/null || true
if kubectl get deployment/traefik -n kube-system >/dev/null 2>&1; then
kubectl rollout restart deployment/traefik -n kube-system
kubectl rollout status deployment/traefik \
-n kube-system \
--timeout="${WAIT_SECONDS}s"
fi
sleep 10
curl -sS \
--max-time 8 \
-o /dev/null \
-H "Host: ${ARGOCD_HOST}" \
"http://127.0.0.1:${HTTP_PORT}/" ||
die "El Ingress sigue sin responder en el puerto ${HTTP_PORT}."
fi
ok "Runtime, Traefik e Ingress disponibles."
}
argocd_rollout_diagnostics() {
local resource="$1"
local namespace="$2"
warn "Diagnóstico del rollout ${resource} en ${namespace}"
kubectl get "$resource" -n "$namespace" -o wide || true
kubectl get pods -n "$namespace" -o wide || true
kubectl get events -n "$namespace" \
--sort-by='.lastTimestamp' 2>/dev/null |
tail -60 || true
}
cleanup_terminating_argocd_server_pods() {
local desired available terminating_pods pod
desired="$(
kubectl get deployment/argocd-server \
-n argocd \
-o jsonpath='{.spec.replicas}' 2>/dev/null || true
)"
available="$(
kubectl get deployment/argocd-server \
-n argocd \
-o jsonpath='{.status.availableReplicas}' 2>/dev/null || true
)"
desired="${desired:-1}"
available="${available:-0}"
# Solo se fuerza la eliminación cuando la nueva réplica ya está disponible.
if (( available < desired )); then
return 1
fi
terminating_pods="$(
kubectl get pods \
-n argocd \
-l app.kubernetes.io/name=argocd-server \
-o jsonpath='{range .items[?(@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}' \
2>/dev/null || true
)"
[[ -n "$terminating_pods" ]] || return 1
while IFS= read -r pod; do
[[ -n "$pod" ]] || continue
warn "Eliminando Pod antiguo de argocd-server atascado en Terminating: ${pod}"
kubectl delete pod "$pod" \
-n argocd \
--grace-period=0 \
--force || true
done <<< "$terminating_pods"
return 0
}
wait_argocd_rollout() {
local kind="$1"
local name="$2"
local namespace="${3:-argocd}"
local resource="${kind}/${name}"
if kubectl rollout status "$resource" \
-n "$namespace" \
--timeout="${ARGOCD_ROLLOUT_TIMEOUT}s"; then
return 0
fi
argocd_rollout_diagnostics "$resource" "$namespace"
if [[ "$resource" == "deployment/argocd-server" ]] &&
cleanup_terminating_argocd_server_pods; then
log "Reintentando rollout de argocd-server después de limpiar la réplica antigua"
kubectl rollout status "$resource" \
-n "$namespace" \
--timeout="180s"
return
fi
die "El rollout ${resource} no quedó disponible."
}
install_argocd() {
log "Instalando/reconciliando Argo CD ${ARGOCD_VERSION}"
kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f -
kubectl apply \
--server-side \
--force-conflicts \
-n argocd \
-f "https://raw.githubusercontent.com/argoproj/argo-cd/${ARGOCD_VERSION}/manifests/install.yaml"
local current_insecure
current_insecure="$(
kubectl get configmap argocd-cmd-params-cm \
-n argocd \
-o jsonpath='{.data.server\.insecure}' \
2>/dev/null || true
)"
if [[ "$current_insecure" != "true" ]]; then
kubectl patch configmap argocd-cmd-params-cm \
-n argocd \
--type merge \
-p '{"data":{"server.insecure":"true"}}'
# Reiniciar únicamente cuando la configuración cambió.
kubectl rollout restart deployment/argocd-server -n argocd
else
ok "argocd-server ya está configurado con server.insecure=true; no se reinicia innecesariamente."
fi
local deployments=(
argocd-server
argocd-repo-server
argocd-redis
argocd-dex-server
argocd-applicationset-controller
argocd-notifications-controller
)
local deployment
for deployment in "${deployments[@]}"; do
if kubectl get deployment "$deployment" -n argocd >/dev/null 2>&1; then
wait_argocd_rollout deployment "$deployment" argocd
fi
done
if kubectl get statefulset argocd-application-controller -n argocd >/dev/null 2>&1; then
wait_argocd_rollout statefulset argocd-application-controller argocd
fi
}
apply_argocd_ingress() {
log "Aplicando Ingress de Argo CD"
cat <<YAML | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: argocd-server-ingress
namespace: argocd
annotations:
traefik.ingress.kubernetes.io/router.entrypoints: web
spec:
ingressClassName: traefik
rules:
- host: ${ARGOCD_HOST}
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: argocd-server
port:
number: 80
YAML
}
install_headlamp_rbac() {
[[ "$INSTALL_HEADLAMP_RBAC" == "true" ]] || return 0
log "Creando RBAC de Headlamp para laboratorio"
kubectl create namespace headlamp --dry-run=client -o yaml | kubectl apply -f -
kubectl create serviceaccount headlamp-admin -n headlamp --dry-run=client -o yaml | kubectl apply -f -
cat <<'YAML' | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: headlamp-admin-binding
subjects:
- kind: ServiceAccount
name: headlamp-admin
namespace: headlamp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
YAML
}
apply_postdeploy_rbac() {
[[ "$APPLY_POSTDEPLOY_RBAC" == "true" ]] || return 0
log "Aplicando RBAC de solo lectura para validación postdeploy"
kubectl create namespace ecommerce --dry-run=client -o yaml | kubectl apply -f -
cat <<'YAML' | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitea-postdeploy-validator
namespace: ecommerce
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitea-postdeploy-validator
namespace: ecommerce
rules:
- apiGroups: ["apps"]
resources: ["deployments", "replicasets"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gitea-postdeploy-validator
namespace: ecommerce
subjects:
- kind: ServiceAccount
name: gitea-postdeploy-validator
namespace: ecommerce
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: gitea-postdeploy-validator
YAML
}
generate_postdeploy_credentials() {
[[ "$APPLY_POSTDEPLOY_RBAC" == "true" ]] || return 0
local secret_dir="${APP_DATA_PATH}/gitea-actions-secrets"
local owner tmp_dir token ca_b64 server kubeconfig_file output_file
owner="$(id -un)"
server="https://${LAB_HOST_IP}:${API_PORT}"
tmp_dir="$(mktemp -d)"
kubeconfig_file="${tmp_dir}/postdeploy-validator.kubeconfig"
# No aplicamos chmod a un glob con archivos históricos que podrían pertenecer
# a root. Generamos todo en un directorio temporal y luego instalamos cada
# archivo con propietario y permisos explícitos.
sudo_cmd install -d -m 0700 -o "$owner" "$secret_dir"
token="$(
kubectl create token gitea-postdeploy-validator \
-n ecommerce \
--duration=8760h
)"
ca_b64="$(
kubectl config view \
--raw \
--minify \
--flatten \
-o jsonpath='{.clusters[0].cluster.certificate-authority-data}'
)"
[[ -n "$token" ]] || die "No fue posible generar el token postdeploy."
[[ -n "$ca_b64" ]] || die "No fue posible leer la CA del clúster."
printf '%s\n' "$token" > "${tmp_dir}/K8S_TOKEN.txt"
printf '%s\n' "$ca_b64" > "${tmp_dir}/K8S_CA_B64.txt"
printf '%s\n' "$server" > "${tmp_dir}/K8S_SERVER.txt"
printf '%s\n' "$LAB_HOST_IP" > "${tmp_dir}/K8S_TLS_SERVER_NAME.txt"
cat > "$kubeconfig_file" <<EOF
apiVersion: v1
kind: Config
clusters:
- name: ${CLUSTER_NAME}
cluster:
certificate-authority-data: ${ca_b64}
server: ${server}
tls-server-name: ${LAB_HOST_IP}
contexts:
- name: gitea-postdeploy-validator@${CLUSTER_NAME}
context:
cluster: ${CLUSTER_NAME}
namespace: ecommerce
user: gitea-postdeploy-validator
current-context: gitea-postdeploy-validator@${CLUSTER_NAME}
users:
- name: gitea-postdeploy-validator
user:
token: ${token}
EOF
# Compatibilidad con el workflow histórico que usa un único secret
# KUBE_CONFIG_DATA en Base64, pero ahora con privilegios mínimos.
base64 "$kubeconfig_file" |
tr -d '\n' \
> "${tmp_dir}/KUBE_CONFIG_DATA.txt"
printf '\n' >> "${tmp_dir}/KUBE_CONFIG_DATA.txt"
for output_file in \
K8S_TOKEN.txt \
K8S_CA_B64.txt \
K8S_SERVER.txt \
K8S_TLS_SERVER_NAME.txt \
KUBE_CONFIG_DATA.txt
do
sudo_cmd install \
-m 0600 \
-o "$owner" \
"${tmp_dir}/${output_file}" \
"${secret_dir}/${output_file}"
done
# Los kubeconfig históricos también contienen credenciales. Ajustamos
# únicamente propietario y permisos; no asumimos ningún grupo llamado devops.
for output_file in kubeconfig-gitea.yaml lab-cluster.kubeconfig; do
if [[ -f "${secret_dir}/${output_file}" ]]; then
sudo_cmd chown "$owner" "${secret_dir}/${output_file}"
sudo_cmd chmod 0600 "${secret_dir}/${output_file}"
fi
done
rm -rf "$tmp_dir"
warn "Credenciales postdeploy regeneradas en ${secret_dir}."
warn "Actualiza en Gitea K8S_TOKEN/K8S_CA_B64/K8S_SERVER/K8S_TLS_SERVER_NAME o el secret histórico KUBE_CONFIG_DATA."
}
read_optional_secret() {
local variable_name="$1"
local file_path="$2"
local value="${!variable_name:-}"
if [[ -n "$value" ]]; then
printf '%s' "$value"
return 0
fi
if [[ -s "$file_path" ]]; then
tr -d '\r\n' < "$file_path"
return 0
fi
return 1
}
git_with_optional_credentials() {
local username="" password="" askpass_dir="" askpass_script="" result=0
username="$(
read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true
)"
password="$(
read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true
)"
if [[ -z "$username" && -z "$password" ]]; then
GIT_TERMINAL_PROMPT=0 git "$@"
return
fi
[[ -n "$username" && -n "$password" ]] ||
die "El clon privado requiere usuario y token/password completos."
askpass_dir="$(mktemp -d)"
askpass_script="${askpass_dir}/askpass.sh"
cat > "$askpass_script" <<'ASKPASS'
#!/bin/sh
case "$1" in
*sername*) printf '%s\n' "$GIT_BOOTSTRAP_USERNAME" ;;
*assword*) printf '%s\n' "$GIT_BOOTSTRAP_PASSWORD" ;;
*) printf '\n' ;;
esac
ASKPASS
chmod 0700 "$askpass_script"
GIT_TERMINAL_PROMPT=0 \
GIT_ASKPASS="$askpass_script" \
GIT_BOOTSTRAP_USERNAME="$username" \
GIT_BOOTSTRAP_PASSWORD="$password" \
git "$@" || result=$?
rm -rf "$askpass_dir"
return "$result"
}
apps_registry_cache_valid() {
[[ -d "${APP_REGISTRY_CACHE_DIR}/.git" ]] &&
git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1
}
record_apps_registry_commit() {
local commit
commit="$(
git -C "$APP_REGISTRY_CACHE_DIR" rev-parse HEAD 2>/dev/null || true
)"
[[ -n "$commit" ]] || return 0
printf '%s\n' "$commit" > "$APP_REGISTRY_COMMIT_FILE"
chmod 0600 "$APP_REGISTRY_COMMIT_FILE"
ok "apps-registry cacheado en commit ${commit:0:12}."
}
find_application_manifest_by_name() {
local app_name="$1"
local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}"
local manifest="" manifest_name=""
while IFS= read -r manifest; do
manifest_name="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.metadata.name}' 2>/dev/null || true
)"
if [[ "$manifest_name" == "$app_name" ]]; then
printf '%s\n' "$manifest"
return 0
fi
done < <(
find "$apps_dir" -maxdepth 1 -type f \
\( -name '*.yaml' -o -name '*.yml' \) | sort
)
return 1
}
validate_monitoring_app_contract() {
local manifest="" revision="" ttl="" ingress_enabled="" ingress_host=""
local request_cpu="" request_memory="" limit_cpu="" limit_memory=""
local parameters_dump=""
manifest="$(find_application_manifest_by_name "$MONITORING_APP_NAME" || true)"
[[ -n "$manifest" ]] ||
die "No se encontró Application/${MONITORING_APP_NAME} en ${ROOT_APP_REPO_PATH}/."
revision="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
)"
[[ -n "$revision" ]] ||
die "${manifest} no contiene spec.source.targetRevision."
if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]]; then
die "${MONITORING_APP_NAME} usa ${revision}. Usa ${MONITORING_FIXED_CHART_VERSION} o superior."
fi
ttl="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished}' \
2>/dev/null || true
)"
request_cpu="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.cpu}' \
2>/dev/null || true
)"
request_memory="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.memory}' \
2>/dev/null || true
)"
limit_cpu="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.cpu}' \
2>/dev/null || true
)"
limit_memory="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.memory}' \
2>/dev/null || true
)"
parameters_dump="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{range .spec.source.helm.parameters[*]}{.name}={.value}{"\n"}{end}' \
2>/dev/null || true
)"
if [[ -z "$ttl" ]] &&
! grep -q '^prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60$' \
<<< "$parameters_dump"; then
die "${MONITORING_APP_NAME} debe configurar ttlSecondsAfterFinished=60."
fi
[[ -n "$request_cpu" && -n "$request_memory" &&
-n "$limit_cpu" && -n "$limit_memory" ]] ||
die "${MONITORING_APP_NAME} debe definir requests/limits del Job admission."
ingress_enabled="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.enabled}' \
2>/dev/null || true
)"
ingress_host="$(
kubectl create --dry-run=client -f "$manifest" \
-o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.hosts[0]}' \
2>/dev/null || true
)"
[[ "$ingress_enabled" == "true" ]] ||
die "${MONITORING_APP_NAME} debe habilitar grafana.ingress.enabled=true."
[[ "$ingress_host" == "$MONITORING_INGRESS_HOST" ]] ||
die "Grafana debe publicarse en ${MONITORING_INGRESS_HOST}; actual=${ingress_host:-vacío}."
ok "Contrato monitoring válido: chart=${revision}, TTL, recursos admission e Ingress Grafana."
}
validate_apps_registry_cache() {
apps_registry_cache_valid ||
die "El caché ${APP_REGISTRY_CACHE_DIR} no es un repositorio Git válido."
[[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]] ||
die "No existe ${APP_REGISTRY_ROOT_MANIFEST} en el repositorio apps-registry."
[[ -d "${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" ]] ||
die "No existe la ruta ${ROOT_APP_REPO_PATH}/ en el repositorio apps-registry."
kubectl apply \
--dry-run=client \
-f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \
>/dev/null
local manifest_name manifest_namespace
manifest_name="$(
kubectl create \
--dry-run=client \
-f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \
-o jsonpath='{.metadata.name}' 2>/dev/null || true
)"
manifest_namespace="$(
kubectl create \
--dry-run=client \
-f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \
-o jsonpath='{.metadata.namespace}' 2>/dev/null || true
)"
[[ "$manifest_name" == "$ROOT_APP_NAME" ]] ||
die "${APP_REGISTRY_ROOT_MANIFEST} debe crear metadata.name=${ROOT_APP_NAME}; actual=${manifest_name:-vacío}."
[[ "$manifest_namespace" == "argocd" ]] ||
die "${APP_REGISTRY_ROOT_MANIFEST} debe usar namespace argocd; actual=${manifest_namespace:-vacío}."
validate_monitoring_app_contract
}
sync_apps_registry_repo() {
local update_ok=true current_origin=""
log "Sincronizando caché local de apps-registry"
install -d -m 0750 "$(dirname "$APP_REGISTRY_CACHE_DIR")"
if apps_registry_cache_valid; then
current_origin="$(
git -C "$APP_REGISTRY_CACHE_DIR" remote get-url origin 2>/dev/null || true
)"
if [[ "$current_origin" != "$ROOT_APP_REPO_URL" ]]; then
warn "Actualizando origin: ${current_origin:-ausente} -> ${ROOT_APP_REPO_URL}"
git -C "$APP_REGISTRY_CACHE_DIR" remote set-url origin "$ROOT_APP_REPO_URL"
fi
if ! git_with_optional_credentials \
-C "$APP_REGISTRY_CACHE_DIR" \
fetch \
--prune \
--depth="$APP_REGISTRY_CLONE_DEPTH" \
origin \
"$ROOT_APP_REVISION"; then
update_ok=false
else
git -C "$APP_REGISTRY_CACHE_DIR" \
checkout -B "$ROOT_APP_REVISION" "origin/${ROOT_APP_REVISION}"
git -C "$APP_REGISTRY_CACHE_DIR" \
reset --hard "origin/${ROOT_APP_REVISION}"
git -C "$APP_REGISTRY_CACHE_DIR" clean -fd
fi
else
rm -rf "$APP_REGISTRY_CACHE_DIR"
if ! git_with_optional_credentials \
clone \
--branch "$ROOT_APP_REVISION" \
--single-branch \
--depth "$APP_REGISTRY_CLONE_DEPTH" \
"$ROOT_APP_REPO_URL" \
"$APP_REGISTRY_CACHE_DIR"; then
update_ok=false
fi
fi
if [[ "$update_ok" != "true" ]]; then
if apps_registry_cache_valid &&
[[ "$APP_REGISTRY_ALLOW_CACHE_FALLBACK" == "true" ]]; then
warn "Gitea no estuvo disponible; se utilizará el último clon válido de apps-registry."
else
die "No fue posible clonar/actualizar ${ROOT_APP_REPO_URL} y no hay un caché válido."
fi
fi
validate_apps_registry_cache
record_apps_registry_commit
}
platform_infra_cache_valid() {
[[ -d "${PLATFORM_INFRA_CACHE_DIR}/.git" ]] &&
git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1
}
record_platform_infra_commit() {
local commit
commit="$(
git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD 2>/dev/null || true
)"
[[ -n "$commit" ]] || return 0
printf '%s\n' "$commit" > "$PLATFORM_INFRA_COMMIT_FILE"
chmod 0600 "$PLATFORM_INFRA_COMMIT_FILE"
ok "platform-infra cacheado en commit ${commit:0:12}."
}
sync_platform_infra_repo() {
local update_ok=true current_origin=""
log "Sincronizando caché local de platform-infra"
install -d -m 0750 "$(dirname "$PLATFORM_INFRA_CACHE_DIR")"
if platform_infra_cache_valid; then
current_origin="$(
git -C "$PLATFORM_INFRA_CACHE_DIR" remote get-url origin 2>/dev/null || true
)"
if [[ "$current_origin" != "$PLATFORM_INFRA_REPO_URL" ]]; then
warn "Actualizando origin platform-infra: ${current_origin:-ausente} -> ${PLATFORM_INFRA_REPO_URL}"
git -C "$PLATFORM_INFRA_CACHE_DIR" remote set-url origin "$PLATFORM_INFRA_REPO_URL"
fi
if ! git_with_optional_credentials \
-C "$PLATFORM_INFRA_CACHE_DIR" \
fetch \
--prune \
--depth="$PLATFORM_INFRA_CLONE_DEPTH" \
origin \
"$PLATFORM_INFRA_REVISION"; then
update_ok=false
else
git -C "$PLATFORM_INFRA_CACHE_DIR" \
checkout -B "$PLATFORM_INFRA_REVISION" "origin/${PLATFORM_INFRA_REVISION}"
git -C "$PLATFORM_INFRA_CACHE_DIR" \
reset --hard "origin/${PLATFORM_INFRA_REVISION}"
git -C "$PLATFORM_INFRA_CACHE_DIR" clean -fd
fi
else
rm -rf "$PLATFORM_INFRA_CACHE_DIR"
if ! git_with_optional_credentials \
clone \
--branch "$PLATFORM_INFRA_REVISION" \
--single-branch \
--depth "$PLATFORM_INFRA_CLONE_DEPTH" \
"$PLATFORM_INFRA_REPO_URL" \
"$PLATFORM_INFRA_CACHE_DIR"; then
update_ok=false
fi
fi
if [[ "$update_ok" != "true" ]]; then
if platform_infra_cache_valid &&
[[ "$PLATFORM_INFRA_ALLOW_CACHE_FALLBACK" == "true" ]]; then
warn "Gitea no estuvo disponible; se utilizará el último clon válido de platform-infra."
else
die "No fue posible clonar/actualizar ${PLATFORM_INFRA_REPO_URL} y no hay un caché válido."
fi
fi
record_platform_infra_commit
}
governance_applications_tsv() {
local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}"
local manifest="" kind="" app_name="" repo_url="" source_path="" revision="" namespace=""
[[ -d "$apps_dir" ]] || die "No existe ${apps_dir}."
while IFS= read -r manifest; do
kind="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.kind}' 2>/dev/null || true
)"
[[ "$kind" == "Application" ]] || continue
repo_url="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.spec.source.repoURL}' 2>/dev/null || true
)"
[[ "$repo_url" == "$PLATFORM_INFRA_REPO_URL" ]] || continue
app_name="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.metadata.name}' 2>/dev/null || true
)"
source_path="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.spec.source.path}' 2>/dev/null || true
)"
revision="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
)"
namespace="$(
kubectl create --dry-run=client \
-f "$manifest" \
-o jsonpath='{.spec.destination.namespace}' 2>/dev/null || true
)"
[[ -n "$app_name" && -n "$source_path" ]] ||
die "Application governance inválida: ${manifest}"
printf '%s\t%s\t%s\t%s\n' \
"$app_name" \
"$source_path" \
"${revision:-$PLATFORM_INFRA_REVISION}" \
"${namespace:-default}"
done < <(
find "$apps_dir" \
-maxdepth 1 \
-type f \
\( -name '*.yaml' -o -name '*.yml' \) \
| sort
)
}
validate_monitoring_governance_contract() {
local rendered_file="${PLATFORM_INFRA_RENDER_DIR}/monitoring-governance.yaml"
[[ -f "$rendered_file" ]] || return 0
if grep -Eq '^kind:[[:space:]]*LimitRange[[:space:]]*$' "$rendered_file"; then
ok "monitoring-governance incluye LimitRange."
return 0
fi
if [[ "$ENFORCE_MONITORING_LIMITRANGE" == "true" ]]; then
die "monitoring-governance no incluye LimitRange. Agrega limitrange.yaml y decláralo en kustomization.yaml para que ResourceQuota no rechace contenedores sin requests/limits."
fi
warn "monitoring-governance no incluye LimitRange; monitoring puede fallar por ResourceQuota."
}
validate_and_render_platform_infra() {
platform_infra_cache_valid ||
die "El caché ${PLATFORM_INFRA_CACHE_DIR} no es un repositorio Git válido."
install -d -m 0750 "$PLATFORM_INFRA_RENDER_DIR"
local app_name="" source_path="" revision="" namespace=""
local source_dir="" render_file="" count=0
while IFS=$'\t' read -r app_name source_path revision namespace; do
[[ -n "$app_name" ]] || continue
count=$((count + 1))
source_dir="${PLATFORM_INFRA_CACHE_DIR}/${source_path}"
render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml"
[[ -d "$source_dir" ]] ||
die "${app_name} referencia una ruta inexistente: ${source_dir}"
if [[ "$revision" != "$PLATFORM_INFRA_REVISION" ]]; then
warn "${app_name} usa targetRevision=${revision}; el caché local está en ${PLATFORM_INFRA_REVISION}."
fi
kubectl kustomize "$source_dir" > "${render_file}.tmp"
[[ -s "${render_file}.tmp" ]] ||
die "Kustomize no generó recursos para ${app_name} (${source_path})."
{
printf '# Source: %s\n' "$PLATFORM_INFRA_REPO_URL"
printf '# Revision: %s\n' "$PLATFORM_INFRA_REVISION"
printf '# Commit: %s\n' \
"$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD)"
printf '# Application: %s\n' "$app_name"
cat "${render_file}.tmp"
} > "$render_file"
rm -f "${render_file}.tmp"
chmod 0600 "$render_file"
ok "Kustomize validado: ${app_name} -> ${source_path}"
done < <(governance_applications_tsv)
(( count > 0 )) ||
die "No se encontraron Applications que referencien ${PLATFORM_INFRA_REPO_URL}."
validate_monitoring_governance_contract
}
apply_governance_snapshots() {
[[ "$APPLY_GOVERNANCE_SNAPSHOTS" == "true" ]] || return 0
log "Aplicando snapshots governance antes de los workloads"
local app_name="" source_path="" revision="" namespace=""
local render_file=""
while IFS=$'\t' read -r app_name source_path revision namespace; do
[[ -n "$app_name" ]] || continue
kubectl create namespace "$namespace" \
--dry-run=client \
-o yaml |
kubectl apply -f - >/dev/null
render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml"
[[ -s "$render_file" ]] ||
die "No existe snapshot renderizado para ${app_name}: ${render_file}"
kubectl apply \
--server-side \
--force-conflicts \
--field-manager=deploy-k3d-lab-governance \
-f "$render_file"
done < <(governance_applications_tsv)
ok "Governance aplicada desde snapshots Kustomize."
}
sync_gitops_repositories() {
sync_apps_registry_repo
sync_platform_infra_repo
validate_and_render_platform_infra
}
root_application_source_manifest() {
local repo_manifest="${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}"
if [[ -f "$repo_manifest" ]]; then
printf '%s\n' "$repo_manifest"
else
printf '%s\n' "$ROOT_APP_MANIFEST"
fi
}
configure_single_argocd_repository() {
local secret_name="$1"
local repo_url="$2"
local repo_username="$3"
local repo_password="$4"
kubectl create secret generic "$secret_name" \
-n argocd \
--from-literal=type=git \
--from-literal=url="$repo_url" \
--from-literal=username="$repo_username" \
--from-literal=password="$repo_password" \
--dry-run=client \
-o yaml |
kubectl label \
--local -f - \
'argocd.argoproj.io/secret-type=repository' \
-o yaml |
kubectl apply -f -
}
configure_argocd_repository() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
local repo_username="" repo_password=""
repo_username="$(
read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true
)"
repo_password="$(
read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true
)"
if [[ -z "$repo_username" && -z "$repo_password" ]]; then
warn "No se configuraron credenciales HTTPS; se asumirá que apps-registry y platform-infra son públicos o que Argo CD ya tiene credenciales."
return 0
fi
[[ -n "$repo_username" && -n "$repo_password" ]] ||
die "Para repositorios privados se requieren usuario y token/password completos."
log "Registrando repositorios Git privados en Argo CD"
configure_single_argocd_repository \
"$ARGOCD_REPO_SECRET_NAME" \
"$ROOT_APP_REPO_URL" \
"$repo_username" \
"$repo_password"
configure_single_argocd_repository \
"$ARGOCD_PLATFORM_REPO_SECRET_NAME" \
"$PLATFORM_INFRA_REPO_URL" \
"$repo_username" \
"$repo_password"
ok "apps-registry y platform-infra configurados en Argo CD."
}
write_root_application_manifest() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
if [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]]; then
ok "Application raíz obtenida del clon: ${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}"
return 0
fi
warn "No se encontró ${APP_REGISTRY_ROOT_MANIFEST} en el clon; generando manifiesto fallback."
cat > "$ROOT_APP_MANIFEST" <<YAML
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: ${ROOT_APP_NAME}
namespace: argocd
labels:
app.kubernetes.io/part-of: lab-bootstrap
app.kubernetes.io/managed-by: deploy-k3d-lab
spec:
project: ${ROOT_APP_PROJECT}
source:
repoURL: '${ROOT_APP_REPO_URL}'
path: ${ROOT_APP_REPO_PATH}
targetRevision: ${ROOT_APP_REVISION}
directory:
recurse: false
destination:
server: 'https://kubernetes.default.svc'
namespace: argocd
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
YAML
chmod 0600 "$ROOT_APP_MANIFEST"
}
apply_root_application() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
local source_manifest
write_root_application_manifest
source_manifest="$(root_application_source_manifest)"
log "Aplicando ${ROOT_APP_NAME} desde ${source_manifest}"
kubectl apply -f "$source_manifest"
kubectl annotate application "$ROOT_APP_NAME" \
-n argocd \
argocd.argoproj.io/refresh=hard \
--overwrite >/dev/null
ok "Application raíz aplicada."
}
root_application_diagnostics() {
kubectl get application "$ROOT_APP_NAME" \
-n argocd \
-o wide 2>/dev/null || true
kubectl get application "$ROOT_APP_NAME" \
-n argocd \
-o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \
2>/dev/null || true
kubectl get pods -n argocd -o wide || true
}
wait_for_root_application() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
log "Esperando sincronización de ${ROOT_APP_NAME}"
local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS))
local sync_status="" health_status="" operation_phase=""
while (( SECONDS < deadline )); do
sync_status="$(
kubectl get application "$ROOT_APP_NAME" \
-n argocd \
-o jsonpath='{.status.sync.status}' 2>/dev/null || true
)"
health_status="$(
kubectl get application "$ROOT_APP_NAME" \
-n argocd \
-o jsonpath='{.status.health.status}' 2>/dev/null || true
)"
operation_phase="$(
kubectl get application "$ROOT_APP_NAME" \
-n argocd \
-o jsonpath='{.status.operationState.phase}' 2>/dev/null || true
)"
if [[ "$operation_phase" == "Error" || "$operation_phase" == "Failed" ]]; then
root_application_diagnostics
die "${ROOT_APP_NAME} terminó con operationState=${operation_phase}."
fi
if [[ "$sync_status" == "Synced" ]]; then
ok "${ROOT_APP_NAME}: Synced / ${health_status:-Unknown}."
return 0
fi
sleep 5
done
root_application_diagnostics
die "${ROOT_APP_NAME} no quedó Synced dentro de ${ROOT_APP_WAIT_SECONDS}s."
}
discover_expected_gitops_apps() {
local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}"
local manifest="" kind="" app_name=""
EXPECTED_APPS_ARRAY=()
[[ -d "$apps_dir" ]] ||
die "No existe el directorio de Applications: ${apps_dir}"
while IFS= read -r manifest; do
[[ -n "$manifest" ]] || continue
kind="$(
kubectl create \
--dry-run=client \
-f "$manifest" \
-o jsonpath='{.kind}' 2>/dev/null || true
)"
app_name="$(
kubectl create \
--dry-run=client \
-f "$manifest" \
-o jsonpath='{.metadata.name}' 2>/dev/null || true
)"
if [[ "$kind" == "Application" &&
-n "$app_name" &&
"$app_name" != "$ROOT_APP_NAME" ]]; then
EXPECTED_APPS_ARRAY+=("$app_name")
fi
done < <(
find "$apps_dir" \
-maxdepth 1 \
-type f \
\( -name '*.yaml' -o -name '*.yml' \) \
| sort
)
(( ${#EXPECTED_APPS_ARRAY[@]} > 0 )) ||
die "No se descubrieron recursos kind: Application dentro de ${apps_dir}."
}
expected_gitops_apps_array() {
local normalized=""
if [[ -n "${EXPECTED_GITOPS_APPS//[[:space:],]/}" ]]; then
normalized="${EXPECTED_GITOPS_APPS//,/ }"
read -r -a EXPECTED_APPS_ARRAY <<< "$normalized"
else
discover_expected_gitops_apps
fi
ok "Applications esperadas: ${EXPECTED_APPS_ARRAY[*]}"
}
wait_for_expected_gitops_apps() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
expected_gitops_apps_array
log "Esperando Applications hijas del repositorio"
local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS))
local app="" missing_count=0
while (( SECONDS < deadline )); do
missing_count=0
for app in "${EXPECTED_APPS_ARRAY[@]}"; do
[[ -n "$app" ]] || continue
kubectl get application "$app" -n argocd >/dev/null 2>&1 ||
missing_count=$((missing_count + 1))
done
if (( missing_count == 0 )); then
ok "Se crearon todas las Applications esperadas."
return 0
fi
sleep 5
done
warn "Applications encontradas:"
kubectl get applications.argoproj.io -n argocd || true
die "Faltan ${missing_count} Applications esperadas después de aplicar ${ROOT_APP_NAME}."
}
wait_for_gitops_apps_health() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
expected_gitops_apps_array
log "Esperando que las Applications hijas se estabilicen"
local deadline=$((SECONDS + GITOPS_APPS_WAIT_SECONDS))
local app="" sync_status="" health_status="" pending=0
while (( SECONDS < deadline )); do
pending=0
for app in "${EXPECTED_APPS_ARRAY[@]}"; do
[[ -n "$app" ]] || continue
sync_status="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.status.sync.status}' 2>/dev/null || true
)"
health_status="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.status.health.status}' 2>/dev/null || true
)"
if [[ "$sync_status" != "Synced" || "$health_status" != "Healthy" ]]; then
pending=$((pending + 1))
fi
done
if (( pending == 0 )); then
ok "Todas las Applications esperadas están Synced/Healthy."
return 0
fi
sleep 10
done
kubectl get applications.argoproj.io -n argocd || true
if [[ "$GITOPS_STRICT_HEALTH" == "true" ]]; then
die "${pending} Applications no quedaron Synced/Healthy dentro de ${GITOPS_APPS_WAIT_SECONDS}s."
fi
warn "${pending} Applications siguen Progressing, OutOfSync o Degraded; Argo CD continuará reconciliándolas."
}
monitoring_app_diagnostics() {
local app="$MONITORING_APP_NAME"
local namespace="$MONITORING_NAMESPACE"
local revision="" sync_status="" health_status="" phase="" message=""
log "Diagnóstico de ${app}"
kubectl get application "$app" -n argocd -o wide || {
die "No existe Application/${app} en argocd."
}
revision="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
)"
sync_status="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.status.sync.status}' 2>/dev/null || true
)"
health_status="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.status.health.status}' 2>/dev/null || true
)"
phase="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.status.operationState.phase}' 2>/dev/null || true
)"
message="$(
kubectl get application "$app" \
-n argocd \
-o jsonpath='{.status.operationState.message}' 2>/dev/null || true
)"
printf 'Chart revision : %s\n' "${revision:-desconocida}"
printf 'Sync : %s\n' "${sync_status:-desconocido}"
printf 'Health : %s\n' "${health_status:-desconocido}"
printf 'Operation : %s\n' "${phase:-ninguna}"
printf 'Message : %s\n' "${message:-sin mensaje}"
echo
echo "=== Jobs/Pods de admission webhooks ==="
kubectl get jobs,pods \
-n "$namespace" \
-o wide 2>/dev/null |
grep -E 'NAME|admission-(create|patch)' || true
echo
echo "=== Eventos recientes de monitoring ==="
kubectl get events \
-n "$namespace" \
--sort-by='.lastTimestamp' 2>/dev/null |
tail -60 || true
echo
echo "=== Condiciones de Argo CD ==="
kubectl get application "$app" \
-n argocd \
-o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \
2>/dev/null || true
if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]] &&
[[ "$message" == *"admission-create"* ||
"$sync_status" == "OutOfSync" ]]; then
warn "Se detectó kube-prometheus-stack ${revision}, afectado por la carrera de ttlSecondsAfterFinished: 0 con hooks de Argo CD."
warn "Actualiza apps/monitoring-app.yaml al menos a ${MONITORING_FIXED_CHART_VERSION} y configura prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60."
fi
}
argocd_core_cli() {
local temp_kubeconfig result=0
temp_kubeconfig="$(mktemp)"
cp "$KUBECONFIG_PATH" "$temp_kubeconfig"
# Argo CD --core obtiene el namespace de control desde el contexto actual
# del kubeconfig. Sin esto busca argocd-cm en "default".
KUBECONFIG="$temp_kubeconfig" \
kubectl config set-context \
--current \
--namespace=argocd \
>/dev/null
if have argocd; then
KUBECONFIG="$temp_kubeconfig" \
argocd "$@" --core || result=$?
else
docker run --rm \
--network host \
-e KUBECONFIG=/kubeconfig \
-v "${temp_kubeconfig}:/kubeconfig:ro" \
--entrypoint /usr/local/bin/argocd \
"$ARGOCD_CLI_IMAGE" \
"$@" \
--core || result=$?
fi
rm -f "$temp_kubeconfig"
return "$result"
}
monitoring_operation_fields() {
local app="$MONITORING_APP_NAME"
MONITORING_TARGET_REVISION="$(
kubectl get application "$app" -n argocd \
-o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true
)"
MONITORING_OPERATION_REVISION="$(
kubectl get application "$app" -n argocd \
-o jsonpath='{.status.operationState.syncResult.revision}' 2>/dev/null || true
)"
MONITORING_OPERATION_PHASE="$(
kubectl get application "$app" -n argocd \
-o jsonpath='{.status.operationState.phase}' 2>/dev/null || true
)"
MONITORING_OPERATION_MESSAGE="$(
kubectl get application "$app" -n argocd \
-o jsonpath='{.status.operationState.message}' 2>/dev/null || true
)"
MONITORING_SYNC_STATUS="$(
kubectl get application "$app" -n argocd \
-o jsonpath='{.status.sync.status}' 2>/dev/null || true
)"
MONITORING_HEALTH_STATUS="$(
kubectl get application "$app" -n argocd \
-o jsonpath='{.status.health.status}' 2>/dev/null || true
)"
export MONITORING_TARGET_REVISION MONITORING_OPERATION_REVISION \
MONITORING_OPERATION_PHASE MONITORING_OPERATION_MESSAGE \
MONITORING_SYNC_STATUS MONITORING_HEALTH_STATUS
}
monitoring_operation_is_stale() {
monitoring_operation_fields
[[ "$MONITORING_OPERATION_PHASE" == "Running" ]] || return 1
[[ "$MONITORING_FORCE_RECOVERY" == "true" ]] && return 0
[[ -n "$MONITORING_OPERATION_REVISION" &&
-n "$MONITORING_TARGET_REVISION" &&
"$MONITORING_OPERATION_REVISION" != "$MONITORING_TARGET_REVISION" ]]
}
wait_monitoring_operation_stopped() {
local deadline=$((SECONDS + MONITORING_OPERATION_STOP_WAIT))
while (( SECONDS < deadline )); do
monitoring_operation_fields
if [[ "$MONITORING_OPERATION_PHASE" != "Running" &&
"$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then
return 0
fi
sleep 3
done
return 1
}
cleanup_monitoring_admission_jobs() {
local job=""
while IFS= read -r job; do
[[ -n "$job" ]] || continue
kubectl delete "$job" -n "$MONITORING_NAMESPACE" --ignore-not-found=true
done < <(
kubectl get jobs -n "$MONITORING_NAMESPACE" -o name 2>/dev/null |
grep -E 'admission-(create|patch)' || true
)
}
wait_monitoring_synced_healthy() {
local deadline=$((SECONDS + MONITORING_RECOVERY_WAIT_SECONDS))
while (( SECONDS < deadline )); do
monitoring_operation_fields
if [[ "$MONITORING_SYNC_STATUS" == "Synced" &&
"$MONITORING_HEALTH_STATUS" == "Healthy" &&
"$MONITORING_OPERATION_PHASE" != "Running" &&
"$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then
ok "${MONITORING_APP_NAME}: Synced/Healthy en ${MONITORING_TARGET_REVISION}."
return 0
fi
sleep 10
done
monitoring_app_diagnostics
return 1
}
recover_monitoring_operation() {
local explicit="${1:-false}"
local refresh_deadline=0
kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 ||
die "No existe Application/${MONITORING_APP_NAME}."
kubectl get limitrange monitoring-default-resources \
-n "$MONITORING_NAMESPACE" >/dev/null 2>&1 ||
die "Falta LimitRange/monitoring-default-resources."
monitoring_operation_fields
printf 'Target revision : %s\n' "${MONITORING_TARGET_REVISION:-desconocida}"
printf 'Operation revision : %s\n' "${MONITORING_OPERATION_REVISION:-ninguna}"
printf 'Operation phase : %s\n' "${MONITORING_OPERATION_PHASE:-ninguna}"
printf 'Operation message : %s\n' "${MONITORING_OPERATION_MESSAGE:-sin mensaje}"
printf 'Sync status : %s\n' "${MONITORING_SYNC_STATUS:-desconocido}"
printf 'Health status : %s\n' "${MONITORING_HEALTH_STATUS:-desconocido}"
# No se inicia otra sincronización cuando la revisión actual ya terminó bien.
if [[ "$MONITORING_OPERATION_PHASE" == "Succeeded" &&
"$MONITORING_OPERATION_REVISION" == "$MONITORING_TARGET_REVISION" ]]; then
kubectl annotate application "$MONITORING_APP_NAME" -n argocd \
argocd.argoproj.io/refresh=hard --overwrite >/dev/null
refresh_deadline=$((SECONDS + 90))
while (( SECONDS < refresh_deadline )); do
monitoring_operation_fields
if [[ "$MONITORING_SYNC_STATUS" == "Synced" &&
"$MONITORING_HEALTH_STATUS" == "Healthy" ]]; then
ok "${MONITORING_APP_NAME} ya estaba correctamente Synced/Healthy; no se ejecutó un sync adicional."
return 0
fi
sleep 5
done
warn "La operación ${MONITORING_TARGET_REVISION} terminó, pero la Application continúa ${MONITORING_SYNC_STATUS}/${MONITORING_HEALTH_STATUS}; se solicitará una nueva reconciliación."
fi
if monitoring_operation_is_stale ||
[[ "$explicit" == "true" && "$MONITORING_OPERATION_PHASE" == "Running" ]]; then
warn "Terminando operación obsoleta/atascada de ${MONITORING_APP_NAME}."
argocd_core_cli app terminate-op "$MONITORING_APP_NAME" ||
die "No fue posible terminar la operación."
wait_monitoring_operation_stopped ||
die "La operación no terminó dentro de ${MONITORING_OPERATION_STOP_WAIT}s."
cleanup_monitoring_admission_jobs
fi
kubectl annotate application "$MONITORING_APP_NAME" -n argocd \
argocd.argoproj.io/refresh=hard --overwrite >/dev/null
# La Application tiene auto-sync. Primero damos oportunidad al controller
# para reconciliar sin crear una operación manual redundante.
refresh_deadline=$((SECONDS + 90))
while (( SECONDS < refresh_deadline )); do
monitoring_operation_fields
if [[ "$MONITORING_SYNC_STATUS" == "Synced" &&
"$MONITORING_HEALTH_STATUS" == "Healthy" &&
"$MONITORING_OPERATION_PHASE" != "Running" &&
"$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then
ok "${MONITORING_APP_NAME}: reconciliada automáticamente."
return 0
fi
if [[ "$MONITORING_OPERATION_PHASE" == "Running" ]]; then
wait_monitoring_synced_healthy ||
die "${MONITORING_APP_NAME} no quedó Synced/Healthy."
return 0
fi
sleep 5
done
log "Auto-sync no inició; solicitando sync manual de ${MONITORING_APP_NAME}"
argocd_core_cli app sync "$MONITORING_APP_NAME" \
--async --prune --assumeYes ||
die "No fue posible iniciar el sync."
wait_monitoring_synced_healthy ||
die "${MONITORING_APP_NAME} no quedó Synced/Healthy."
}
auto_recover_monitoring_if_stale() {
[[ "$AUTO_RECOVER_STALE_MONITORING_OPERATION" == "true" ]] || return 0
kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 || return 0
if monitoring_operation_is_stale; then
recover_monitoring_operation false
fi
}
ensure_root_application_present() {
[[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0
if ! kubectl get crd applications.argoproj.io >/dev/null 2>&1; then
warn "El CRD Application aún no existe; se omite la verificación de ${ROOT_APP_NAME}."
return 0
fi
if kubectl get application "$ROOT_APP_NAME" -n argocd >/dev/null 2>&1; then
return 0
fi
warn "${ROOT_APP_NAME} no existe; restaurándolo desde el clon persistente."
if [[ "$APP_REGISTRY_REFRESH_ON_ENSURE" == "true" ]]; then
sync_apps_registry_repo
elif ! apps_registry_cache_valid; then
sync_apps_registry_repo
else
validate_apps_registry_cache
fi
if [[ "$PLATFORM_INFRA_REFRESH_ON_ENSURE" == "true" ]]; then
sync_platform_infra_repo
elif ! platform_infra_cache_valid; then
sync_platform_infra_repo
fi
validate_and_render_platform_infra
apply_governance_snapshots
configure_argocd_repository
apply_root_application
}
runner_container_exists() {
docker inspect "$GITEA_RUNNER_CONTAINER" >/dev/null 2>&1
}
runner_is_running() {
[[ "$(docker inspect -f '{{.State.Running}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true)" == "true" ]]
}
runner_is_registered() {
[[ -s "${GITEA_RUNNER_DATA_DIR}/.runner" ]]
}
prepare_gitea_runner_directories() {
# El watchdog se ejecuta como devops y no dispone de TTY para sudo.
# install-autostart deja estas rutas creadas y con propietario correcto.
install -d \
-m 0700 \
"$GITEA_RUNNER_ROOT" \
"$GITEA_RUNNER_DATA_DIR" ||
die "No se pudieron preparar las rutas persistentes del Gitea runner como $(id -un)."
}
detect_gitea_runner_image() {
local detected_image=""
if [[ -n "${GITEA_RUNNER_IMAGE_OVERRIDE:-}" ]]; then
printf '%s\n' "$GITEA_RUNNER_IMAGE_OVERRIDE"
return 0
fi
if [[ -s "$GITEA_RUNNER_IMAGE_FILE" ]]; then
tr -d '\r\n' < "$GITEA_RUNNER_IMAGE_FILE"
return 0
fi
if runner_container_exists; then
detected_image="$(
docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true
)"
fi
printf '%s\n' "${detected_image:-$GITEA_RUNNER_IMAGE}"
}
adopt_existing_gitea_runner() {
runner_container_exists || return 0
local current_image data_mount=""
current_image="$(
docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER"
)"
printf '%s\n' "$current_image" > "$GITEA_RUNNER_IMAGE_FILE"
chmod 0600 "$GITEA_RUNNER_IMAGE_FILE"
data_mount="$(
docker inspect \
-f '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}' \
"$GITEA_RUNNER_CONTAINER" 2>/dev/null || true
)"
if ! runner_is_registered; then
warn "Copiando el registro actual del runner desde ${GITEA_RUNNER_CONTAINER}:/data."
docker cp \
"${GITEA_RUNNER_CONTAINER}:/data/." \
"${GITEA_RUNNER_DATA_DIR}/" 2>/dev/null || true
fi
if [[ "$data_mount" == "$GITEA_RUNNER_DATA_DIR" ]]; then
docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null
return 0
fi
warn "El runner actual no usa el volumen persistente ${GITEA_RUNNER_DATA_DIR}."
if [[ "$MIGRATE_EXISTING_RUNNER" != "true" ]]; then
warn "Se conserva el contenedor actual. Ejecuta MIGRATE_EXISTING_RUNNER=true $0 runner cuando no haya un workflow activo."
docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null
return 0
fi
runner_is_registered ||
die "No se pudo recuperar ${GITEA_RUNNER_DATA_DIR}/.runner; no se eliminará el contenedor actual."
log "Migrando ${GITEA_RUNNER_CONTAINER} al almacenamiento persistente"
docker stop "$GITEA_RUNNER_CONTAINER" >/dev/null || true
docker rm "$GITEA_RUNNER_CONTAINER" >/dev/null
}
register_gitea_runner() {
runner_is_registered && return 0
[[ -s "$GITEA_RUNNER_TOKEN_FILE" ]] ||
die "El runner no está registrado. Guarda un token de registro en ${GITEA_RUNNER_TOKEN_FILE} con permisos 0600."
local runner_image
runner_image="$(detect_gitea_runner_image)"
log "Registrando Gitea Actions runner ${GITEA_RUNNER_NAME}"
docker pull "$runner_image"
docker run --rm \
-v "${GITEA_RUNNER_DATA_DIR}:/data" \
-v "${GITEA_RUNNER_TOKEN_FILE}:/run/secrets/gitea-runner-token:ro" \
-e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \
-e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \
-e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \
--entrypoint /bin/sh \
"$runner_image" \
-ec '
cd /data
act_runner register \
--no-interactive \
--instance "$GITEA_INSTANCE_URL" \
--token "$(cat /run/secrets/gitea-runner-token)" \
--name "$GITEA_RUNNER_NAME" \
--labels "$GITEA_RUNNER_LABELS"
'
runner_is_registered ||
die "Gitea no generó ${GITEA_RUNNER_DATA_DIR}/.runner."
printf '%s\n' "$runner_image" > "$GITEA_RUNNER_IMAGE_FILE"
chmod 0600 "$GITEA_RUNNER_IMAGE_FILE"
}
create_gitea_runner_container() {
runner_container_exists && return 0
local runner_image
runner_image="$(detect_gitea_runner_image)"
register_gitea_runner
log "Creando contenedor ${GITEA_RUNNER_CONTAINER}"
docker run -d \
--name "$GITEA_RUNNER_CONTAINER" \
--restart unless-stopped \
-v "${GITEA_RUNNER_DATA_DIR}:/data" \
-v /var/run/docker.sock:/var/run/docker.sock \
-e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \
-e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \
-e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \
"$runner_image" >/dev/null
}
validate_gitea_runner() {
runner_container_exists ||
die "No existe el contenedor ${GITEA_RUNNER_CONTAINER}."
if ! runner_is_running; then
docker start "$GITEA_RUNNER_CONTAINER" >/dev/null
fi
docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null
local deadline=$((SECONDS + 60))
while (( SECONDS < deadline )); do
if runner_is_running; then
ok "Gitea runner activo: ${GITEA_RUNNER_CONTAINER}."
return 0
fi
sleep 3
done
docker logs --tail 150 "$GITEA_RUNNER_CONTAINER" || true
die "El contenedor ${GITEA_RUNNER_CONTAINER} no quedó activo."
}
ensure_gitea_runner() {
[[ "$INSTALL_GITEA_RUNNER" == "true" ]] || return 0
prepare_gitea_runner_directories
adopt_existing_gitea_runner
if ! runner_container_exists; then
if ! runner_is_registered && [[ ! -s "$GITEA_RUNNER_TOKEN_FILE" ]]; then
if [[ "$GITEA_RUNNER_REQUIRED" == "true" ]]; then
die "Falta el registro persistente del runner y el token ${GITEA_RUNNER_TOKEN_FILE}."
fi
warn "Gitea runner omitido porque no hay registro ni token."
return 0
fi
create_gitea_runner_container
fi
validate_gitea_runner
}
apply_bootstrap_manifests() {
shopt -s nullglob
local manifests=("$BOOTSTRAP_DIR"/*.yaml "$BOOTSTRAP_DIR"/*.yml)
shopt -u nullglob
if (( ${#manifests[@]} == 0 )); then
warn "No hay manifiestos GitOps en ${BOOTSTRAP_DIR}. Argo CD quedó instalado, pero las Applications deben aplicarse después."
return 0
fi
log "Aplicando manifiestos de bootstrap GitOps"
kubectl apply -f "$BOOTSTRAP_DIR"
}
print_credentials() {
local argocd_password='pendiente'
if kubectl get secret argocd-initial-admin-secret -n argocd >/dev/null 2>&1; then
argocd_password="$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 --decode)"
fi
printf '\n==========================================================\n'
printf ' LABORATORIO DISPONIBLE\n'
printf ' ---------------------------------------------------------\n'
printf ' API Kubernetes : https://%s:%s\n' "$LAB_HOST_IP" "$API_PORT"
printf ' NPM Argo CD : http://%s:%s\n' "$LAB_HOST_IP" "$HTTP_PORT"
printf ' Host Argo CD : %s\n' "$ARGOCD_HOST"
printf ' ArgoCD admin : %s\n' "$argocd_password"
printf ' Token Headlamp : kubectl create token headlamp-admin -n headlamp\n'
printf ' Kubeconfig : %s\n' "$KUBECONFIG_PATH"
printf ' Root Application: %s -> %s/%s@%s\n' \
"$ROOT_APP_NAME" "$ROOT_APP_REPO_URL" "$ROOT_APP_REPO_PATH" "$ROOT_APP_REVISION"
printf ' Gitea runner : %s\n' "$GITEA_RUNNER_CONTAINER"
printf '==========================================================\n'
}
status_report() {
require_commands
ensure_docker
log "Docker"
docker info --format 'DockerRootDir={{.DockerRootDir}} Driver={{.Driver}}'
docker ps -a --filter "label=k3d.cluster=${CLUSTER_NAME}" \
--format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
log "k3d"
k3d cluster list || true
if cluster_exists; then
write_kubeconfig
log "Kubernetes"
kubectl get nodes -o wide || true
kubectl get pods -A || true
kubectl get ingress -A || true
kubectl get applications.argoproj.io -n argocd 2>/dev/null || true
fi
log "apps-registry local"
if apps_registry_cache_valid; then
printf 'Ruta: %s
' "$APP_REGISTRY_CACHE_DIR"
printf 'Commit: %s
' "$(git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)"
else
warn "No existe un clon válido en ${APP_REGISTRY_CACHE_DIR}."
fi
log "platform-infra local"
if platform_infra_cache_valid; then
printf 'Ruta: %s
' "$PLATFORM_INFRA_CACHE_DIR"
printf 'Commit: %s
' "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)"
else
warn "No existe un clon válido en ${PLATFORM_INFRA_CACHE_DIR}."
fi
log "Gitea Actions runner"
if runner_container_exists; then
docker ps -a --filter "name=^/${GITEA_RUNNER_CONTAINER}$" --format 'table {{.Names}} {{.Image}} {{.Status}}'
else
warn "No existe ${GITEA_RUNNER_CONTAINER}."
fi
}
backup_cluster_state() {
local stamp archive
stamp="$(date '+%Y%m%d-%H%M%S')"
archive="${BACKUP_DIR}/k3d-lab-state-${stamp}.tar.gz"
log "Respaldando estado persistente en ${archive}"
if cluster_exists; then
k3d cluster stop "$CLUSTER_NAME" || true
fi
sudo_cmd tar \
--acls \
--xattrs \
-C "$APP_DATA_PATH" \
-czf "$archive" \
k3s storage secrets config bootstrap gitea-runner git rendered 2>/dev/null || \
sudo_cmd tar -C "$APP_DATA_PATH" -czf "$archive" k3s storage secrets config bootstrap gitea-runner git rendered
sudo_cmd chown "$(id -un)" "$archive"
ok "Backup creado: ${archive}"
}
reset_cluster() {
[[ "${CONFIRM_RESET:-}" == "DELETE-${CLUSTER_NAME}" ]] || \
die "Reset bloqueado. Usa CONFIRM_RESET=DELETE-${CLUSTER_NAME} $0 reset"
prepare_directories
backup_cluster_state
if cluster_exists; then
k3d cluster delete "$CLUSTER_NAME"
fi
local stamp="$(date '+%Y%m%d-%H%M%S')"
sudo_cmd mv "$K3S_SERVER_PATH" "${K3S_SERVER_PATH}.reset-${stamp}"
sudo_cmd mv "${APP_DATA_PATH}/storage" "${APP_DATA_PATH}/storage.reset-${stamp}"
sudo_cmd install -d -m 0700 "$K3S_SERVER_PATH"
sudo_cmd install -d -m 0755 "$K3S_SERVER_STORAGE" "$K3S_AGENT0_STORAGE" "$K3S_AGENT1_STORAGE"
create_cluster
}
ensure_cluster_only() {
require_commands
ensure_docker
validate_data_mount
validate_host_ip
if ! cluster_exists; then
if cluster_containers_exist; then
die "Existen contenedores del lab, pero k3d no reconoce el clúster. Se requiere diagnóstico manual; no se crearán duplicados."
fi
die "El clúster ${CLUSTER_NAME} no existe. Ejecuta bootstrap manualmente."
fi
start_cluster
set_restart_policy
write_kubeconfig
wait_for_api
wait_for_nodes
validate_runtime_after_boot
ensure_root_application_present
auto_recover_monitoring_if_stale
ensure_gitea_runner
}
resolve_service_tool() {
local command_name="$1"
local explicit_path="${2:-}"
local service_home="$3"
local candidate=""
for candidate in \
"$explicit_path" \
"$(command -v "$command_name" 2>/dev/null || true)" \
"${service_home}/.local/bin/${command_name}" \
"${service_home}/bin/${command_name}" \
"/DATA/.local/bin/${command_name}" \
"/DATA/bin/${command_name}" \
"/DATA/AppData/bin/${command_name}" \
"/usr/local/bin/${command_name}" \
"/usr/local/sbin/${command_name}" \
"/usr/bin/${command_name}" \
"/usr/sbin/${command_name}" \
"/bin/${command_name}" \
"/sbin/${command_name}"
do
[[ -n "$candidate" ]] || continue
if [[ -x "$candidate" ]]; then
readlink -f "$candidate" 2>/dev/null || printf '%s\n' "$candidate"
return 0
fi
done
return 1
}
append_unique_path_dir() {
local current_path="$1"
local new_dir="$2"
[[ -n "$new_dir" ]] || {
printf '%s\n' "$current_path"
return
}
case ":${current_path}:" in
*":${new_dir}:"*)
printf '%s\n' "$current_path"
;;
*)
if [[ -n "$current_path" ]]; then
printf '%s:%s\n' "$current_path" "$new_dir"
else
printf '%s\n' "$new_dir"
fi
;;
esac
}
install_autostart() {
[[ $(id -u) -eq 0 ]] || die "Ejecuta install-autostart con sudo."
local service_user="${SUDO_USER:-devops}"
local service_home service_group docker_root installed_script
local k3d_path kubectl_path docker_path curl_path tool_path tool_dir
service_home="$(getent passwd "$service_user" | cut -d: -f6)"
service_group="$(id -gn "$service_user")"
docker_root="$(docker info --format '{{.DockerRootDir}}')"
installed_script="${APP_DATA_PATH}/bin/deploy-k3d-lab"
[[ -n "$service_home" ]] || die "No fue posible determinar HOME de ${service_user}."
[[ -n "$service_group" ]] || die "No fue posible determinar el grupo de ${service_user}."
[[ -n "$docker_root" ]] || die "No fue posible determinar DockerRootDir."
k3d_path="$(
resolve_service_tool k3d "${K3D_BIN:-}" "$service_home"
)" || die "No se encontró k3d para systemd. Usa K3D_BIN=\$(command -v k3d) con sudo -E."
kubectl_path="$(
resolve_service_tool kubectl "${KUBECTL_BIN:-}" "$service_home"
)" || die "No se encontró kubectl para systemd."
docker_path="$(
resolve_service_tool docker "${DOCKER_BIN:-}" "$service_home"
)" || die "No se encontró docker para systemd."
curl_path="$(
resolve_service_tool curl "${CURL_BIN:-}" "$service_home"
)" || die "No se encontró curl para systemd."
tool_path="${APP_DATA_PATH}/bin"
for tool_dir in "$(dirname "$k3d_path")" "$(dirname "$kubectl_path")" "$(dirname "$docker_path")" "$(dirname "$curl_path")" /usr/local/sbin /usr/local/bin /usr/sbin /usr/bin /sbin /bin
do
tool_path="$(append_unique_path_dir "$tool_path" "$tool_dir")"
done
# ZimaOS no garantiza /usr/local/sbin y su filesystem base puede ser
# inmutable. Instalamos el ejecutable en /DATA, que es persistente.
install -d \
-m 0750 \
-o "$service_user" \
"$(dirname "$installed_script")"
install \
-m 0750 \
-o "$service_user" \
"$0" \
"$installed_script"
# El servicio se ejecutará como devops y no puede usar sudo sin TTY.
# Dejamos listas y escribibles todas las rutas que el modo ensure modifica.
install -d \
-m 0750 \
-o "$service_user" \
"$APP_DATA_PATH"
install -d \
-m 0700 \
-o "$service_user" \
"$(dirname "$KUBECONFIG_PATH")"
# El modo ensure valida y recupera el runner como usuario devops.
# Preparamos estas rutas durante la instalación privilegiada para que
# systemd nunca necesite ejecutar sudo de forma no interactiva.
install -d \
-m 0700 \
-o "$service_user" \
"$GITEA_RUNNER_ROOT" \
"$GITEA_RUNNER_DATA_DIR"
if [[ -f "$GITEA_RUNNER_IMAGE_FILE" ]]; then
chown "$service_user" "$GITEA_RUNNER_IMAGE_FILE"
chmod 0600 "$GITEA_RUNNER_IMAGE_FILE"
fi
if [[ -f "$KUBECONFIG_PATH" ]]; then
chown "$service_user" "$KUBECONFIG_PATH"
chmod 0600 "$KUBECONFIG_PATH"
fi
install -d -m 0755 \
/etc/default \
/etc/systemd/system \
/etc/systemd/system/docker.service.d
systemctl enable docker.service >/dev/null
if systemctl list-unit-files containerd.service >/dev/null 2>&1; then
systemctl enable containerd.service >/dev/null || true
fi
cat > /etc/systemd/system/docker.service.d/10-k3d-storage.conf <<UNIT
[Unit]
RequiresMountsFor=${APP_DATA_PATH} ${docker_root}
After=local-fs.target
UNIT
cat > /etc/default/k3d-lab <<ENV
CLUSTER_NAME=${CLUSTER_NAME}
APP_DATA_PATH=${APP_DATA_PATH}
KUBECONFIG_PATH=${KUBECONFIG_PATH}
LAB_HOST_IP=${LAB_HOST_IP}
EXPECTED_HOST_IP=${EXPECTED_HOST_IP}
WAIT_SECONDS=${WAIT_SECONDS}
ARGOCD_ROLLOUT_TIMEOUT=${ARGOCD_ROLLOUT_TIMEOUT}
AGENT_RECOVERY_WAIT=${AGENT_RECOVERY_WAIT}
AUTO_REREGISTER_STALE_AGENTS=${AUTO_REREGISTER_STALE_AGENTS}
CHECK_INGRESS_AFTER_BOOT=${CHECK_INGRESS_AFTER_BOOT}
K3D_BIN=${k3d_path}
KUBECTL_BIN=${kubectl_path}
DOCKER_BIN=${docker_path}
CURL_BIN=${curl_path}
LAB_TOOL_PATH=${tool_path}
BOOTSTRAP_ROOT_APP=${BOOTSTRAP_ROOT_APP}
ROOT_APP_NAME=${ROOT_APP_NAME}
ROOT_APP_REPO_URL=${ROOT_APP_REPO_URL}
ROOT_APP_REPO_PATH=${ROOT_APP_REPO_PATH}
ROOT_APP_REVISION=${ROOT_APP_REVISION}
ROOT_APP_PROJECT=${ROOT_APP_PROJECT}
ROOT_APP_WAIT_SECONDS=${ROOT_APP_WAIT_SECONDS}
GITOPS_APPS_WAIT_SECONDS=${GITOPS_APPS_WAIT_SECONDS}
GITOPS_STRICT_HEALTH=${GITOPS_STRICT_HEALTH}
EXPECTED_GITOPS_APPS=${EXPECTED_GITOPS_APPS}
ARGOCD_REPO_USERNAME_FILE=${ARGOCD_REPO_USERNAME_FILE}
ARGOCD_REPO_PASSWORD_FILE=${ARGOCD_REPO_PASSWORD_FILE}
INSTALL_GITEA_RUNNER=${INSTALL_GITEA_RUNNER}
GITEA_RUNNER_REQUIRED=${GITEA_RUNNER_REQUIRED}
GITEA_RUNNER_CONTAINER=${GITEA_RUNNER_CONTAINER}
GITEA_RUNNER_IMAGE=${GITEA_RUNNER_IMAGE}
GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}
GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}
GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}
GITEA_RUNNER_ROOT=${GITEA_RUNNER_ROOT}
GITEA_RUNNER_DATA_DIR=${GITEA_RUNNER_DATA_DIR}
GITEA_RUNNER_IMAGE_FILE=${GITEA_RUNNER_IMAGE_FILE}
GITEA_RUNNER_TOKEN_FILE=${GITEA_RUNNER_TOKEN_FILE}
MIGRATE_EXISTING_RUNNER=false
APP_REGISTRY_CACHE_DIR=${APP_REGISTRY_CACHE_DIR}
APP_REGISTRY_ROOT_MANIFEST=${APP_REGISTRY_ROOT_MANIFEST}
APP_REGISTRY_COMMIT_FILE=${APP_REGISTRY_COMMIT_FILE}
APP_REGISTRY_ALLOW_CACHE_FALLBACK=${APP_REGISTRY_ALLOW_CACHE_FALLBACK}
APP_REGISTRY_REFRESH_ON_ENSURE=${APP_REGISTRY_REFRESH_ON_ENSURE}
APP_REGISTRY_CLONE_DEPTH=${APP_REGISTRY_CLONE_DEPTH}
PLATFORM_INFRA_REPO_URL=${PLATFORM_INFRA_REPO_URL}
PLATFORM_INFRA_REVISION=${PLATFORM_INFRA_REVISION}
PLATFORM_INFRA_CACHE_DIR=${PLATFORM_INFRA_CACHE_DIR}
PLATFORM_INFRA_COMMIT_FILE=${PLATFORM_INFRA_COMMIT_FILE}
PLATFORM_INFRA_ALLOW_CACHE_FALLBACK=${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK}
PLATFORM_INFRA_REFRESH_ON_ENSURE=${PLATFORM_INFRA_REFRESH_ON_ENSURE}
PLATFORM_INFRA_CLONE_DEPTH=${PLATFORM_INFRA_CLONE_DEPTH}
PLATFORM_INFRA_RENDER_DIR=${PLATFORM_INFRA_RENDER_DIR}
APPLY_GOVERNANCE_SNAPSHOTS=${APPLY_GOVERNANCE_SNAPSHOTS}
ENFORCE_MONITORING_LIMITRANGE=${ENFORCE_MONITORING_LIMITRANGE}
MONITORING_GOVERNANCE_PATH=${MONITORING_GOVERNANCE_PATH}
ARGOCD_PLATFORM_REPO_SECRET_NAME=${ARGOCD_PLATFORM_REPO_SECRET_NAME}
MONITORING_APP_NAME=${MONITORING_APP_NAME}
MONITORING_NAMESPACE=${MONITORING_NAMESPACE}
MONITORING_FIXED_CHART_VERSION=${MONITORING_FIXED_CHART_VERSION}
MONITORING_INGRESS_HOST=${MONITORING_INGRESS_HOST}
MONITORING_RECOVERY_WAIT_SECONDS=${MONITORING_RECOVERY_WAIT_SECONDS}
MONITORING_OPERATION_STOP_WAIT=${MONITORING_OPERATION_STOP_WAIT}
AUTO_RECOVER_STALE_MONITORING_OPERATION=${AUTO_RECOVER_STALE_MONITORING_OPERATION}
MONITORING_FORCE_RECOVERY=false
ARGOCD_CLI_IMAGE=${ARGOCD_CLI_IMAGE}
ENV
cat > /etc/systemd/system/k3d-lab-ensure.service <<UNIT
[Unit]
Description=Recuperar y validar laboratorio k3d
Wants=network-online.target
After=local-fs.target network-online.target docker.service
Requires=docker.service
RequiresMountsFor=${APP_DATA_PATH} ${docker_root}
StartLimitIntervalSec=15min
StartLimitBurst=3
[Service]
Type=oneshot
User=${service_user}
Group=${service_group}
SupplementaryGroups=docker
Environment=HOME=${service_home}
Environment=KUBECONFIG=${KUBECONFIG_PATH}
Environment="PATH=${tool_path}"
EnvironmentFile=-/etc/default/k3d-lab
ExecStart=/bin/bash ${installed_script} ensure
TimeoutStartSec=900
Restart=no
StandardOutput=journal
StandardError=journal
UNIT
cat > /etc/systemd/system/k3d-lab-ensure.timer <<'UNIT'
[Unit]
Description=Watchdog del laboratorio k3d
[Timer]
OnBootSec=90s
OnUnitInactiveSec=5min
AccuracySec=20s
Persistent=true
Unit=k3d-lab-ensure.service
[Install]
WantedBy=timers.target
UNIT
systemctl daemon-reload
systemctl enable --now k3d-lab-ensure.timer
if ! systemctl start k3d-lab-ensure.service; then
systemctl status k3d-lab-ensure.service --no-pager -l || true
journalctl -u k3d-lab-ensure.service -n 200 --no-pager || true
die "La prueba inicial de k3d-lab-ensure.service falló."
fi
ok "Autostart instalado para ${service_user}."
printf 'Script persistente: %s\n' "$installed_script"
printf 'k3d para systemd: %s\n' "$k3d_path"
printf 'kubectl para systemd: %s\n' "$kubectl_path"
printf 'PATH para systemd: %s\n' "$tool_path"
printf 'DockerRootDir protegido por systemd: %s\n' "$docker_root"
printf 'Logs: journalctl -u k3d-lab-ensure.service -n 200 --no-pager\n'
printf 'Nota: el drop-in de Docker será plenamente verificable después del reinicio de prueba.\n'
}
main() {
case "$MODE" in
status)
status_report
return
;;
install-autostart)
install_autostart
return
;;
ensure)
ensure_cluster_only
return
;;
repo-sync)
require_commands
ensure_docker
validate_data_mount
prepare_directories
sync_gitops_repositories
return
;;
runner)
require_commands
ensure_docker
validate_data_mount
prepare_directories
ensure_gitea_runner
return
;;
gitops)
require_commands
ensure_docker
validate_data_mount
validate_host_ip
cluster_exists || die "No existe ${CLUSTER_NAME}; ejecuta bootstrap."
write_kubeconfig
wait_for_api
install_argocd
sync_gitops_repositories
apply_governance_snapshots
configure_argocd_repository
apply_root_application
wait_for_root_application
wait_for_expected_gitops_apps
auto_recover_monitoring_if_stale
wait_for_gitops_apps_health
return
;;
monitoring-diagnose)
require_commands
ensure_docker
validate_data_mount
cluster_exists || die "No existe ${CLUSTER_NAME}."
write_kubeconfig
wait_for_api
monitoring_app_diagnostics
return
;;
monitoring-recover)
require_commands
ensure_docker
validate_data_mount
cluster_exists || die "No existe ${CLUSTER_NAME}."
write_kubeconfig
wait_for_api
MONITORING_FORCE_RECOVERY=true recover_monitoring_operation true
return
;;
esac
require_commands
ensure_docker
validate_data_mount
validate_host_ip
prepare_directories
ensure_cluster_token
write_k3d_config
case "$MODE" in
bootstrap)
if cluster_exists; then
start_cluster
else
create_cluster
fi
;;
recover)
cluster_exists || die "No existe ${CLUSTER_NAME}; utiliza bootstrap para crearlo."
start_cluster
;;
reset)
reset_cluster
;;
*)
die "Modo no válido: ${MODE}. Usa bootstrap, recover, ensure, status, repo-sync, gitops, runner, monitoring-diagnose, monitoring-recover, install-autostart o reset."
;;
esac
set_restart_policy
write_kubeconfig
wait_for_api
wait_for_nodes
install_argocd
apply_argocd_ingress
install_headlamp_rbac
apply_postdeploy_rbac
generate_postdeploy_credentials
sync_gitops_repositories
apply_governance_snapshots
configure_argocd_repository
write_root_application_manifest
apply_bootstrap_manifests
apply_root_application
wait_for_root_application
wait_for_expected_gitops_apps
auto_recover_monitoring_if_stale
wait_for_gitops_apps_health
ensure_gitea_runner
validate_runtime_after_boot
log "Validación final"
kubectl get nodes -o wide
kubectl get pods -n argocd
kubectl get ingress -A
kubectl get applications.argoproj.io -n argocd 2>/dev/null || true
print_credentials
}
main "$@"
Reference in New Issue View Git Blame Copy Permalink