diff --git a/deploy-lab.sh b/deploy-lab.sh index d3c4f20..62df9f9 100644 --- a/deploy-lab.sh +++ b/deploy-lab.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash -# Laboratorio estable v3.8: ZimaOS + Docker + k3d + Argo CD + Traefik +# Laboratorio estable v4.4.3: ZimaOS + Docker + k3d + Argo CD + Multi-Repo + Governance + Monitoring Recovery + Runner # # Modos: # bootstrap Crea el clúster si no existe; si existe, lo inicia y reconcilia. # recover Recupera únicamente un clúster existente. # ensure Health check no destructivo para systemd; no instala Argo CD. # status Muestra diagnóstico del laboratorio. +# repo-sync Clona/actualiza apps-registry y platform-infra. +# gitops Actualiza el repo y aplica root-apps-registry. +# monitoring-diagnose Diagnostica kube-prometheus-stack/Grafana. +# monitoring-recover Finaliza una operación obsoleta y resincroniza Grafana. +# runner Adopta, crea o recupera el Gitea Actions runner. # install-autostart Instala servicio/timer systemd, orden de montajes y autorreparación. # reset Reconstrucción limpia; requiere confirmación explícita. # @@ -53,6 +58,69 @@ AGENT_RECOVERY_WAIT="${AGENT_RECOVERY_WAIT:-75}" AUTO_REREGISTER_STALE_AGENTS="${AUTO_REREGISTER_STALE_AGENTS:-true}" CHECK_INGRESS_AFTER_BOOT="${CHECK_INGRESS_AFTER_BOOT:-true}" +# Bootstrap GitOps / App-of-Apps. +BOOTSTRAP_ROOT_APP="${BOOTSTRAP_ROOT_APP:-true}" +ROOT_APP_NAME="${ROOT_APP_NAME:-root-apps-registry}" +ROOT_APP_REPO_URL="${ROOT_APP_REPO_URL:-https://gitea.cruzcloud.net/devops/apps-registry.git}" +ROOT_APP_REPO_PATH="${ROOT_APP_REPO_PATH:-apps}" +ROOT_APP_REVISION="${ROOT_APP_REVISION:-main}" +ROOT_APP_PROJECT="${ROOT_APP_PROJECT:-default}" +ROOT_APP_MANIFEST="${ROOT_APP_MANIFEST:-${BOOTSTRAP_DIR}/00-root-apps-registry.yaml}" +ROOT_APP_WAIT_SECONDS="${ROOT_APP_WAIT_SECONDS:-600}" +GITOPS_APPS_WAIT_SECONDS="${GITOPS_APPS_WAIT_SECONDS:-900}" +GITOPS_STRICT_HEALTH="${GITOPS_STRICT_HEALTH:-false}" +EXPECTED_GITOPS_APPS="${EXPECTED_GITOPS_APPS:-}" +ARGOCD_REPO_SECRET_NAME="${ARGOCD_REPO_SECRET_NAME:-apps-registry-repository}" +ARGOCD_REPO_USERNAME_FILE="${ARGOCD_REPO_USERNAME_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-username}" +ARGOCD_REPO_PASSWORD_FILE="${ARGOCD_REPO_PASSWORD_FILE:-${APP_DATA_PATH}/secrets/argocd-repo-password}" + +# Copia local persistente del repositorio GitOps. +APP_REGISTRY_CACHE_DIR="${APP_REGISTRY_CACHE_DIR:-${APP_DATA_PATH}/git/apps-registry}" +APP_REGISTRY_ROOT_MANIFEST="${APP_REGISTRY_ROOT_MANIFEST:-application.yaml}" +APP_REGISTRY_COMMIT_FILE="${APP_REGISTRY_COMMIT_FILE:-${APP_DATA_PATH}/git/apps-registry.commit}" +APP_REGISTRY_ALLOW_CACHE_FALLBACK="${APP_REGISTRY_ALLOW_CACHE_FALLBACK:-true}" +APP_REGISTRY_REFRESH_ON_ENSURE="${APP_REGISTRY_REFRESH_ON_ENSURE:-false}" +APP_REGISTRY_CLONE_DEPTH="${APP_REGISTRY_CLONE_DEPTH:-1}" + +# Repositorio de governance/Kustomize. +PLATFORM_INFRA_REPO_URL="${PLATFORM_INFRA_REPO_URL:-https://gitea.cruzcloud.net/devops/platform-infra.git}" +PLATFORM_INFRA_REVISION="${PLATFORM_INFRA_REVISION:-main}" +PLATFORM_INFRA_CACHE_DIR="${PLATFORM_INFRA_CACHE_DIR:-${APP_DATA_PATH}/git/platform-infra}" +PLATFORM_INFRA_COMMIT_FILE="${PLATFORM_INFRA_COMMIT_FILE:-${APP_DATA_PATH}/git/platform-infra.commit}" +PLATFORM_INFRA_ALLOW_CACHE_FALLBACK="${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK:-true}" +PLATFORM_INFRA_REFRESH_ON_ENSURE="${PLATFORM_INFRA_REFRESH_ON_ENSURE:-false}" +PLATFORM_INFRA_CLONE_DEPTH="${PLATFORM_INFRA_CLONE_DEPTH:-1}" +PLATFORM_INFRA_RENDER_DIR="${PLATFORM_INFRA_RENDER_DIR:-${APP_DATA_PATH}/rendered/platform-infra}" +APPLY_GOVERNANCE_SNAPSHOTS="${APPLY_GOVERNANCE_SNAPSHOTS:-true}" +ENFORCE_MONITORING_LIMITRANGE="${ENFORCE_MONITORING_LIMITRANGE:-true}" +MONITORING_GOVERNANCE_PATH="${MONITORING_GOVERNANCE_PATH:-monitoring-governance}" +ARGOCD_PLATFORM_REPO_SECRET_NAME="${ARGOCD_PLATFORM_REPO_SECRET_NAME:-platform-infra-repository}" + +# Diagnóstico de monitoring/Grafana. +MONITORING_APP_NAME="${MONITORING_APP_NAME:-monitoring-app}" +MONITORING_NAMESPACE="${MONITORING_NAMESPACE:-monitoring}" +MONITORING_FIXED_CHART_VERSION="${MONITORING_FIXED_CHART_VERSION:-58.3.3}" +MONITORING_INGRESS_HOST="${MONITORING_INGRESS_HOST:-grafana.cruzcloud.net}" +MONITORING_RECOVERY_WAIT_SECONDS="${MONITORING_RECOVERY_WAIT_SECONDS:-900}" +MONITORING_OPERATION_STOP_WAIT="${MONITORING_OPERATION_STOP_WAIT:-180}" +AUTO_RECOVER_STALE_MONITORING_OPERATION="${AUTO_RECOVER_STALE_MONITORING_OPERATION:-true}" +MONITORING_FORCE_RECOVERY="${MONITORING_FORCE_RECOVERY:-false}" +ARGOCD_CLI_IMAGE="${ARGOCD_CLI_IMAGE:-quay.io/argoproj/argocd:${ARGOCD_VERSION}}" + +# Gitea Actions runner externo al clúster. +INSTALL_GITEA_RUNNER="${INSTALL_GITEA_RUNNER:-true}" +GITEA_RUNNER_REQUIRED="${GITEA_RUNNER_REQUIRED:-true}" +GITEA_RUNNER_CONTAINER="${GITEA_RUNNER_CONTAINER:-gitea-runner}" +GITEA_RUNNER_IMAGE="${GITEA_RUNNER_IMAGE:-docker.io/gitea/act_runner:latest}" +GITEA_INSTANCE_URL="${GITEA_INSTANCE_URL:-https://gitea.cruzcloud.net}" +GITEA_RUNNER_NAME="${GITEA_RUNNER_NAME:-zimaos-lab-runner}" +GITEA_RUNNER_LABELS="${GITEA_RUNNER_LABELS:-ubuntu-latest:docker://node:20-bookworm}" +GITEA_RUNNER_ROOT="${GITEA_RUNNER_ROOT:-${APP_DATA_PATH}/gitea-runner}" +GITEA_RUNNER_DATA_DIR="${GITEA_RUNNER_DATA_DIR:-${GITEA_RUNNER_ROOT}/data}" +GITEA_RUNNER_IMAGE_FILE="${GITEA_RUNNER_IMAGE_FILE:-${GITEA_RUNNER_ROOT}/image.txt}" +GITEA_RUNNER_TOKEN_FILE="${GITEA_RUNNER_TOKEN_FILE:-${APP_DATA_PATH}/secrets/gitea-runner-registration-token}" +MIGRATE_EXISTING_RUNNER="${MIGRATE_EXISTING_RUNNER:-false}" + K3S_SERVER_PATH="${APP_DATA_PATH}/k3s/server" K3S_SERVER_STORAGE="${APP_DATA_PATH}/storage/server-0" K3S_AGENT0_STORAGE="${APP_DATA_PATH}/storage/agent-0" @@ -99,7 +167,7 @@ sudo_cmd() { require_commands() { local command_name - for command_name in docker k3d kubectl curl awk sed grep findmnt ss tar base64 tr; do + for command_name in docker k3d kubectl curl git awk sed grep findmnt ss tar base64 tr; do have "$command_name" || die "No se encontró el comando requerido: ${command_name}" done } @@ -170,6 +238,11 @@ prepare_directories() { "${APP_DATA_PATH}/secrets" \ "$BOOTSTRAP_DIR" \ "$BACKUP_DIR" \ + "$GITEA_RUNNER_ROOT" \ + "$GITEA_RUNNER_DATA_DIR" \ + "$(dirname "$APP_REGISTRY_CACHE_DIR")" \ + "$(dirname "$PLATFORM_INFRA_CACHE_DIR")" \ + "$PLATFORM_INFRA_RENDER_DIR" \ "$(dirname "$KUBECONFIG_PATH")" # K3s y los workloads escriben como root dentro de los nodos. @@ -930,6 +1003,1380 @@ EOF warn "Actualiza en Gitea K8S_TOKEN/K8S_CA_B64/K8S_SERVER/K8S_TLS_SERVER_NAME o el secret histórico KUBE_CONFIG_DATA." } + +read_optional_secret() { + local variable_name="$1" + local file_path="$2" + local value="${!variable_name:-}" + + if [[ -n "$value" ]]; then + printf '%s' "$value" + return 0 + fi + + if [[ -s "$file_path" ]]; then + tr -d '\r\n' < "$file_path" + return 0 + fi + + return 1 +} + + +git_with_optional_credentials() { + local username="" password="" askpass_dir="" askpass_script="" result=0 + + username="$( + read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true + )" + password="$( + read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true + )" + + if [[ -z "$username" && -z "$password" ]]; then + GIT_TERMINAL_PROMPT=0 git "$@" + return + fi + + [[ -n "$username" && -n "$password" ]] || + die "El clon privado requiere usuario y token/password completos." + + askpass_dir="$(mktemp -d)" + askpass_script="${askpass_dir}/askpass.sh" + + cat > "$askpass_script" <<'ASKPASS' +#!/bin/sh +case "$1" in + *sername*) printf '%s\n' "$GIT_BOOTSTRAP_USERNAME" ;; + *assword*) printf '%s\n' "$GIT_BOOTSTRAP_PASSWORD" ;; + *) printf '\n' ;; +esac +ASKPASS + chmod 0700 "$askpass_script" + + GIT_TERMINAL_PROMPT=0 \ + GIT_ASKPASS="$askpass_script" \ + GIT_BOOTSTRAP_USERNAME="$username" \ + GIT_BOOTSTRAP_PASSWORD="$password" \ + git "$@" || result=$? + + rm -rf "$askpass_dir" + return "$result" +} + +apps_registry_cache_valid() { + [[ -d "${APP_REGISTRY_CACHE_DIR}/.git" ]] && + git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 +} + +record_apps_registry_commit() { + local commit + commit="$( + git -C "$APP_REGISTRY_CACHE_DIR" rev-parse HEAD 2>/dev/null || true + )" + + [[ -n "$commit" ]] || return 0 + + printf '%s\n' "$commit" > "$APP_REGISTRY_COMMIT_FILE" + chmod 0600 "$APP_REGISTRY_COMMIT_FILE" + ok "apps-registry cacheado en commit ${commit:0:12}." +} + + +find_application_manifest_by_name() { + local app_name="$1" + local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" + local manifest="" manifest_name="" + + while IFS= read -r manifest; do + manifest_name="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.metadata.name}' 2>/dev/null || true + )" + + if [[ "$manifest_name" == "$app_name" ]]; then + printf '%s\n' "$manifest" + return 0 + fi + done < <( + find "$apps_dir" -maxdepth 1 -type f \ + \( -name '*.yaml' -o -name '*.yml' \) | sort + ) + + return 1 +} + +validate_monitoring_app_contract() { + local manifest="" revision="" ttl="" ingress_enabled="" ingress_host="" + local request_cpu="" request_memory="" limit_cpu="" limit_memory="" + local parameters_dump="" + + manifest="$(find_application_manifest_by_name "$MONITORING_APP_NAME" || true)" + [[ -n "$manifest" ]] || + die "No se encontró Application/${MONITORING_APP_NAME} en ${ROOT_APP_REPO_PATH}/." + + revision="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true + )" + + [[ -n "$revision" ]] || + die "${manifest} no contiene spec.source.targetRevision." + + if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]]; then + die "${MONITORING_APP_NAME} usa ${revision}. Usa ${MONITORING_FIXED_CHART_VERSION} o superior." + fi + + ttl="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished}' \ + 2>/dev/null || true + )" + request_cpu="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.cpu}' \ + 2>/dev/null || true + )" + request_memory="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.requests.memory}' \ + 2>/dev/null || true + )" + limit_cpu="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.cpu}' \ + 2>/dev/null || true + )" + limit_memory="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.prometheusOperator.admissionWebhooks.patch.resources.limits.memory}' \ + 2>/dev/null || true + )" + parameters_dump="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{range .spec.source.helm.parameters[*]}{.name}={.value}{"\n"}{end}' \ + 2>/dev/null || true + )" + + if [[ -z "$ttl" ]] && + ! grep -q '^prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60$' \ + <<< "$parameters_dump"; then + die "${MONITORING_APP_NAME} debe configurar ttlSecondsAfterFinished=60." + fi + + [[ -n "$request_cpu" && -n "$request_memory" && + -n "$limit_cpu" && -n "$limit_memory" ]] || + die "${MONITORING_APP_NAME} debe definir requests/limits del Job admission." + + ingress_enabled="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.enabled}' \ + 2>/dev/null || true + )" + ingress_host="$( + kubectl create --dry-run=client -f "$manifest" \ + -o jsonpath='{.spec.source.helm.valuesObject.grafana.ingress.hosts[0]}' \ + 2>/dev/null || true + )" + + [[ "$ingress_enabled" == "true" ]] || + die "${MONITORING_APP_NAME} debe habilitar grafana.ingress.enabled=true." + + [[ "$ingress_host" == "$MONITORING_INGRESS_HOST" ]] || + die "Grafana debe publicarse en ${MONITORING_INGRESS_HOST}; actual=${ingress_host:-vacío}." + + ok "Contrato monitoring válido: chart=${revision}, TTL, recursos admission e Ingress Grafana." +} + +validate_apps_registry_cache() { + apps_registry_cache_valid || + die "El caché ${APP_REGISTRY_CACHE_DIR} no es un repositorio Git válido." + + [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]] || + die "No existe ${APP_REGISTRY_ROOT_MANIFEST} en el repositorio apps-registry." + + [[ -d "${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" ]] || + die "No existe la ruta ${ROOT_APP_REPO_PATH}/ en el repositorio apps-registry." + + kubectl apply \ + --dry-run=client \ + -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \ + >/dev/null + + local manifest_name manifest_namespace + manifest_name="$( + kubectl create \ + --dry-run=client \ + -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \ + -o jsonpath='{.metadata.name}' 2>/dev/null || true + )" + manifest_namespace="$( + kubectl create \ + --dry-run=client \ + -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" \ + -o jsonpath='{.metadata.namespace}' 2>/dev/null || true + )" + + [[ "$manifest_name" == "$ROOT_APP_NAME" ]] || + die "${APP_REGISTRY_ROOT_MANIFEST} debe crear metadata.name=${ROOT_APP_NAME}; actual=${manifest_name:-vacío}." + + [[ "$manifest_namespace" == "argocd" ]] || + die "${APP_REGISTRY_ROOT_MANIFEST} debe usar namespace argocd; actual=${manifest_namespace:-vacío}." + + validate_monitoring_app_contract +} + +sync_apps_registry_repo() { + local update_ok=true current_origin="" + + log "Sincronizando caché local de apps-registry" + + install -d -m 0750 "$(dirname "$APP_REGISTRY_CACHE_DIR")" + + if apps_registry_cache_valid; then + current_origin="$( + git -C "$APP_REGISTRY_CACHE_DIR" remote get-url origin 2>/dev/null || true + )" + + if [[ "$current_origin" != "$ROOT_APP_REPO_URL" ]]; then + warn "Actualizando origin: ${current_origin:-ausente} -> ${ROOT_APP_REPO_URL}" + git -C "$APP_REGISTRY_CACHE_DIR" remote set-url origin "$ROOT_APP_REPO_URL" + fi + + if ! git_with_optional_credentials \ + -C "$APP_REGISTRY_CACHE_DIR" \ + fetch \ + --prune \ + --depth="$APP_REGISTRY_CLONE_DEPTH" \ + origin \ + "$ROOT_APP_REVISION"; then + update_ok=false + else + git -C "$APP_REGISTRY_CACHE_DIR" \ + checkout -B "$ROOT_APP_REVISION" "origin/${ROOT_APP_REVISION}" + + git -C "$APP_REGISTRY_CACHE_DIR" \ + reset --hard "origin/${ROOT_APP_REVISION}" + + git -C "$APP_REGISTRY_CACHE_DIR" clean -fd + fi + else + rm -rf "$APP_REGISTRY_CACHE_DIR" + + if ! git_with_optional_credentials \ + clone \ + --branch "$ROOT_APP_REVISION" \ + --single-branch \ + --depth "$APP_REGISTRY_CLONE_DEPTH" \ + "$ROOT_APP_REPO_URL" \ + "$APP_REGISTRY_CACHE_DIR"; then + update_ok=false + fi + fi + + if [[ "$update_ok" != "true" ]]; then + if apps_registry_cache_valid && + [[ "$APP_REGISTRY_ALLOW_CACHE_FALLBACK" == "true" ]]; then + warn "Gitea no estuvo disponible; se utilizará el último clon válido de apps-registry." + else + die "No fue posible clonar/actualizar ${ROOT_APP_REPO_URL} y no hay un caché válido." + fi + fi + + validate_apps_registry_cache + record_apps_registry_commit +} + + +platform_infra_cache_valid() { + [[ -d "${PLATFORM_INFRA_CACHE_DIR}/.git" ]] && + git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 +} + +record_platform_infra_commit() { + local commit + commit="$( + git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD 2>/dev/null || true + )" + + [[ -n "$commit" ]] || return 0 + + printf '%s\n' "$commit" > "$PLATFORM_INFRA_COMMIT_FILE" + chmod 0600 "$PLATFORM_INFRA_COMMIT_FILE" + ok "platform-infra cacheado en commit ${commit:0:12}." +} + +sync_platform_infra_repo() { + local update_ok=true current_origin="" + + log "Sincronizando caché local de platform-infra" + + install -d -m 0750 "$(dirname "$PLATFORM_INFRA_CACHE_DIR")" + + if platform_infra_cache_valid; then + current_origin="$( + git -C "$PLATFORM_INFRA_CACHE_DIR" remote get-url origin 2>/dev/null || true + )" + + if [[ "$current_origin" != "$PLATFORM_INFRA_REPO_URL" ]]; then + warn "Actualizando origin platform-infra: ${current_origin:-ausente} -> ${PLATFORM_INFRA_REPO_URL}" + git -C "$PLATFORM_INFRA_CACHE_DIR" remote set-url origin "$PLATFORM_INFRA_REPO_URL" + fi + + if ! git_with_optional_credentials \ + -C "$PLATFORM_INFRA_CACHE_DIR" \ + fetch \ + --prune \ + --depth="$PLATFORM_INFRA_CLONE_DEPTH" \ + origin \ + "$PLATFORM_INFRA_REVISION"; then + update_ok=false + else + git -C "$PLATFORM_INFRA_CACHE_DIR" \ + checkout -B "$PLATFORM_INFRA_REVISION" "origin/${PLATFORM_INFRA_REVISION}" + + git -C "$PLATFORM_INFRA_CACHE_DIR" \ + reset --hard "origin/${PLATFORM_INFRA_REVISION}" + + git -C "$PLATFORM_INFRA_CACHE_DIR" clean -fd + fi + else + rm -rf "$PLATFORM_INFRA_CACHE_DIR" + + if ! git_with_optional_credentials \ + clone \ + --branch "$PLATFORM_INFRA_REVISION" \ + --single-branch \ + --depth "$PLATFORM_INFRA_CLONE_DEPTH" \ + "$PLATFORM_INFRA_REPO_URL" \ + "$PLATFORM_INFRA_CACHE_DIR"; then + update_ok=false + fi + fi + + if [[ "$update_ok" != "true" ]]; then + if platform_infra_cache_valid && + [[ "$PLATFORM_INFRA_ALLOW_CACHE_FALLBACK" == "true" ]]; then + warn "Gitea no estuvo disponible; se utilizará el último clon válido de platform-infra." + else + die "No fue posible clonar/actualizar ${PLATFORM_INFRA_REPO_URL} y no hay un caché válido." + fi + fi + + record_platform_infra_commit +} + +governance_applications_tsv() { + local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" + local manifest="" kind="" app_name="" repo_url="" source_path="" revision="" namespace="" + + [[ -d "$apps_dir" ]] || die "No existe ${apps_dir}." + + while IFS= read -r manifest; do + kind="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.kind}' 2>/dev/null || true + )" + + [[ "$kind" == "Application" ]] || continue + + repo_url="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.spec.source.repoURL}' 2>/dev/null || true + )" + + [[ "$repo_url" == "$PLATFORM_INFRA_REPO_URL" ]] || continue + + app_name="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.metadata.name}' 2>/dev/null || true + )" + source_path="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.spec.source.path}' 2>/dev/null || true + )" + revision="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true + )" + namespace="$( + kubectl create --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.spec.destination.namespace}' 2>/dev/null || true + )" + + [[ -n "$app_name" && -n "$source_path" ]] || + die "Application governance inválida: ${manifest}" + + printf '%s\t%s\t%s\t%s\n' \ + "$app_name" \ + "$source_path" \ + "${revision:-$PLATFORM_INFRA_REVISION}" \ + "${namespace:-default}" + done < <( + find "$apps_dir" \ + -maxdepth 1 \ + -type f \ + \( -name '*.yaml' -o -name '*.yml' \) \ + | sort + ) +} + +validate_monitoring_governance_contract() { + local rendered_file="${PLATFORM_INFRA_RENDER_DIR}/monitoring-governance.yaml" + + [[ -f "$rendered_file" ]] || return 0 + + if grep -Eq '^kind:[[:space:]]*LimitRange[[:space:]]*$' "$rendered_file"; then + ok "monitoring-governance incluye LimitRange." + return 0 + fi + + if [[ "$ENFORCE_MONITORING_LIMITRANGE" == "true" ]]; then + die "monitoring-governance no incluye LimitRange. Agrega limitrange.yaml y decláralo en kustomization.yaml para que ResourceQuota no rechace contenedores sin requests/limits." + fi + + warn "monitoring-governance no incluye LimitRange; monitoring puede fallar por ResourceQuota." +} + +validate_and_render_platform_infra() { + platform_infra_cache_valid || + die "El caché ${PLATFORM_INFRA_CACHE_DIR} no es un repositorio Git válido." + + install -d -m 0750 "$PLATFORM_INFRA_RENDER_DIR" + + local app_name="" source_path="" revision="" namespace="" + local source_dir="" render_file="" count=0 + + while IFS=$'\t' read -r app_name source_path revision namespace; do + [[ -n "$app_name" ]] || continue + count=$((count + 1)) + + source_dir="${PLATFORM_INFRA_CACHE_DIR}/${source_path}" + render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml" + + [[ -d "$source_dir" ]] || + die "${app_name} referencia una ruta inexistente: ${source_dir}" + + if [[ "$revision" != "$PLATFORM_INFRA_REVISION" ]]; then + warn "${app_name} usa targetRevision=${revision}; el caché local está en ${PLATFORM_INFRA_REVISION}." + fi + + kubectl kustomize "$source_dir" > "${render_file}.tmp" + [[ -s "${render_file}.tmp" ]] || + die "Kustomize no generó recursos para ${app_name} (${source_path})." + + { + printf '# Source: %s\n' "$PLATFORM_INFRA_REPO_URL" + printf '# Revision: %s\n' "$PLATFORM_INFRA_REVISION" + printf '# Commit: %s\n' \ + "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse HEAD)" + printf '# Application: %s\n' "$app_name" + cat "${render_file}.tmp" + } > "$render_file" + + rm -f "${render_file}.tmp" + chmod 0600 "$render_file" + + ok "Kustomize validado: ${app_name} -> ${source_path}" + done < <(governance_applications_tsv) + + (( count > 0 )) || + die "No se encontraron Applications que referencien ${PLATFORM_INFRA_REPO_URL}." + + validate_monitoring_governance_contract +} + +apply_governance_snapshots() { + [[ "$APPLY_GOVERNANCE_SNAPSHOTS" == "true" ]] || return 0 + + log "Aplicando snapshots governance antes de los workloads" + + local app_name="" source_path="" revision="" namespace="" + local render_file="" + + while IFS=$'\t' read -r app_name source_path revision namespace; do + [[ -n "$app_name" ]] || continue + + kubectl create namespace "$namespace" \ + --dry-run=client \ + -o yaml | + kubectl apply -f - >/dev/null + + render_file="${PLATFORM_INFRA_RENDER_DIR}/${app_name}.yaml" + [[ -s "$render_file" ]] || + die "No existe snapshot renderizado para ${app_name}: ${render_file}" + + kubectl apply \ + --server-side \ + --force-conflicts \ + --field-manager=deploy-k3d-lab-governance \ + -f "$render_file" + done < <(governance_applications_tsv) + + ok "Governance aplicada desde snapshots Kustomize." +} + +sync_gitops_repositories() { + sync_apps_registry_repo + sync_platform_infra_repo + validate_and_render_platform_infra +} + +root_application_source_manifest() { + local repo_manifest="${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" + + if [[ -f "$repo_manifest" ]]; then + printf '%s\n' "$repo_manifest" + else + printf '%s\n' "$ROOT_APP_MANIFEST" + fi +} + +configure_single_argocd_repository() { + local secret_name="$1" + local repo_url="$2" + local repo_username="$3" + local repo_password="$4" + + kubectl create secret generic "$secret_name" \ + -n argocd \ + --from-literal=type=git \ + --from-literal=url="$repo_url" \ + --from-literal=username="$repo_username" \ + --from-literal=password="$repo_password" \ + --dry-run=client \ + -o yaml | + kubectl label \ + --local -f - \ + 'argocd.argoproj.io/secret-type=repository' \ + -o yaml | + kubectl apply -f - +} + +configure_argocd_repository() { + [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 + + local repo_username="" repo_password="" + + repo_username="$( + read_optional_secret ARGOCD_REPO_USERNAME "$ARGOCD_REPO_USERNAME_FILE" || true + )" + repo_password="$( + read_optional_secret ARGOCD_REPO_PASSWORD "$ARGOCD_REPO_PASSWORD_FILE" || true + )" + + if [[ -z "$repo_username" && -z "$repo_password" ]]; then + warn "No se configuraron credenciales HTTPS; se asumirá que apps-registry y platform-infra son públicos o que Argo CD ya tiene credenciales." + return 0 + fi + + [[ -n "$repo_username" && -n "$repo_password" ]] || + die "Para repositorios privados se requieren usuario y token/password completos." + + log "Registrando repositorios Git privados en Argo CD" + + configure_single_argocd_repository \ + "$ARGOCD_REPO_SECRET_NAME" \ + "$ROOT_APP_REPO_URL" \ + "$repo_username" \ + "$repo_password" + + configure_single_argocd_repository \ + "$ARGOCD_PLATFORM_REPO_SECRET_NAME" \ + "$PLATFORM_INFRA_REPO_URL" \ + "$repo_username" \ + "$repo_password" + + ok "apps-registry y platform-infra configurados en Argo CD." +} + +write_root_application_manifest() { + [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 + + if [[ -f "${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" ]]; then + ok "Application raíz obtenida del clon: ${APP_REGISTRY_CACHE_DIR}/${APP_REGISTRY_ROOT_MANIFEST}" + return 0 + fi + + warn "No se encontró ${APP_REGISTRY_ROOT_MANIFEST} en el clon; generando manifiesto fallback." + + cat > "$ROOT_APP_MANIFEST" </dev/null + + ok "Application raíz aplicada." +} + +root_application_diagnostics() { + kubectl get application "$ROOT_APP_NAME" \ + -n argocd \ + -o wide 2>/dev/null || true + + kubectl get application "$ROOT_APP_NAME" \ + -n argocd \ + -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \ + 2>/dev/null || true + + kubectl get pods -n argocd -o wide || true +} + +wait_for_root_application() { + [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 + + log "Esperando sincronización de ${ROOT_APP_NAME}" + + local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS)) + local sync_status="" health_status="" operation_phase="" + + while (( SECONDS < deadline )); do + sync_status="$( + kubectl get application "$ROOT_APP_NAME" \ + -n argocd \ + -o jsonpath='{.status.sync.status}' 2>/dev/null || true + )" + + health_status="$( + kubectl get application "$ROOT_APP_NAME" \ + -n argocd \ + -o jsonpath='{.status.health.status}' 2>/dev/null || true + )" + + operation_phase="$( + kubectl get application "$ROOT_APP_NAME" \ + -n argocd \ + -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true + )" + + if [[ "$operation_phase" == "Error" || "$operation_phase" == "Failed" ]]; then + root_application_diagnostics + die "${ROOT_APP_NAME} terminó con operationState=${operation_phase}." + fi + + if [[ "$sync_status" == "Synced" ]]; then + ok "${ROOT_APP_NAME}: Synced / ${health_status:-Unknown}." + return 0 + fi + + sleep 5 + done + + root_application_diagnostics + die "${ROOT_APP_NAME} no quedó Synced dentro de ${ROOT_APP_WAIT_SECONDS}s." +} + +discover_expected_gitops_apps() { + local apps_dir="${APP_REGISTRY_CACHE_DIR}/${ROOT_APP_REPO_PATH}" + local manifest="" kind="" app_name="" + + EXPECTED_APPS_ARRAY=() + + [[ -d "$apps_dir" ]] || + die "No existe el directorio de Applications: ${apps_dir}" + + while IFS= read -r manifest; do + [[ -n "$manifest" ]] || continue + + kind="$( + kubectl create \ + --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.kind}' 2>/dev/null || true + )" + + app_name="$( + kubectl create \ + --dry-run=client \ + -f "$manifest" \ + -o jsonpath='{.metadata.name}' 2>/dev/null || true + )" + + if [[ "$kind" == "Application" && + -n "$app_name" && + "$app_name" != "$ROOT_APP_NAME" ]]; then + EXPECTED_APPS_ARRAY+=("$app_name") + fi + done < <( + find "$apps_dir" \ + -maxdepth 1 \ + -type f \ + \( -name '*.yaml' -o -name '*.yml' \) \ + | sort + ) + + (( ${#EXPECTED_APPS_ARRAY[@]} > 0 )) || + die "No se descubrieron recursos kind: Application dentro de ${apps_dir}." +} + +expected_gitops_apps_array() { + local normalized="" + + if [[ -n "${EXPECTED_GITOPS_APPS//[[:space:],]/}" ]]; then + normalized="${EXPECTED_GITOPS_APPS//,/ }" + read -r -a EXPECTED_APPS_ARRAY <<< "$normalized" + else + discover_expected_gitops_apps + fi + + ok "Applications esperadas: ${EXPECTED_APPS_ARRAY[*]}" +} + +wait_for_expected_gitops_apps() { + [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 + + expected_gitops_apps_array + + log "Esperando Applications hijas del repositorio" + + local deadline=$((SECONDS + ROOT_APP_WAIT_SECONDS)) + local app="" missing_count=0 + + while (( SECONDS < deadline )); do + missing_count=0 + + for app in "${EXPECTED_APPS_ARRAY[@]}"; do + [[ -n "$app" ]] || continue + kubectl get application "$app" -n argocd >/dev/null 2>&1 || + missing_count=$((missing_count + 1)) + done + + if (( missing_count == 0 )); then + ok "Se crearon todas las Applications esperadas." + return 0 + fi + + sleep 5 + done + + warn "Applications encontradas:" + kubectl get applications.argoproj.io -n argocd || true + die "Faltan ${missing_count} Applications esperadas después de aplicar ${ROOT_APP_NAME}." +} + +wait_for_gitops_apps_health() { + [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 + + expected_gitops_apps_array + + log "Esperando que las Applications hijas se estabilicen" + + local deadline=$((SECONDS + GITOPS_APPS_WAIT_SECONDS)) + local app="" sync_status="" health_status="" pending=0 + + while (( SECONDS < deadline )); do + pending=0 + + for app in "${EXPECTED_APPS_ARRAY[@]}"; do + [[ -n "$app" ]] || continue + + sync_status="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.status.sync.status}' 2>/dev/null || true + )" + + health_status="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.status.health.status}' 2>/dev/null || true + )" + + if [[ "$sync_status" != "Synced" || "$health_status" != "Healthy" ]]; then + pending=$((pending + 1)) + fi + done + + if (( pending == 0 )); then + ok "Todas las Applications esperadas están Synced/Healthy." + return 0 + fi + + sleep 10 + done + + kubectl get applications.argoproj.io -n argocd || true + + if [[ "$GITOPS_STRICT_HEALTH" == "true" ]]; then + die "${pending} Applications no quedaron Synced/Healthy dentro de ${GITOPS_APPS_WAIT_SECONDS}s." + fi + + warn "${pending} Applications siguen Progressing, OutOfSync o Degraded; Argo CD continuará reconciliándolas." +} + + +monitoring_app_diagnostics() { + local app="$MONITORING_APP_NAME" + local namespace="$MONITORING_NAMESPACE" + local revision="" sync_status="" health_status="" phase="" message="" + + log "Diagnóstico de ${app}" + + kubectl get application "$app" -n argocd -o wide || { + die "No existe Application/${app} en argocd." + } + + revision="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true + )" + sync_status="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.status.sync.status}' 2>/dev/null || true + )" + health_status="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.status.health.status}' 2>/dev/null || true + )" + phase="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true + )" + message="$( + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{.status.operationState.message}' 2>/dev/null || true + )" + + printf 'Chart revision : %s\n' "${revision:-desconocida}" + printf 'Sync : %s\n' "${sync_status:-desconocido}" + printf 'Health : %s\n' "${health_status:-desconocido}" + printf 'Operation : %s\n' "${phase:-ninguna}" + printf 'Message : %s\n' "${message:-sin mensaje}" + + echo + echo "=== Jobs/Pods de admission webhooks ===" + kubectl get jobs,pods \ + -n "$namespace" \ + -o wide 2>/dev/null | + grep -E 'NAME|admission-(create|patch)' || true + + echo + echo "=== Eventos recientes de monitoring ===" + kubectl get events \ + -n "$namespace" \ + --sort-by='.lastTimestamp' 2>/dev/null | + tail -60 || true + + echo + echo "=== Condiciones de Argo CD ===" + kubectl get application "$app" \ + -n argocd \ + -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}' \ + 2>/dev/null || true + + if [[ "$revision" == "58.2.1" || "$revision" == "58.2.2" ]] && + [[ "$message" == *"admission-create"* || + "$sync_status" == "OutOfSync" ]]; then + warn "Se detectó kube-prometheus-stack ${revision}, afectado por la carrera de ttlSecondsAfterFinished: 0 con hooks de Argo CD." + warn "Actualiza apps/monitoring-app.yaml al menos a ${MONITORING_FIXED_CHART_VERSION} y configura prometheusOperator.admissionWebhooks.patch.ttlSecondsAfterFinished=60." + fi +} + + +argocd_core_cli() { + local temp_kubeconfig result=0 + temp_kubeconfig="$(mktemp)" + + cp "$KUBECONFIG_PATH" "$temp_kubeconfig" + + # Argo CD --core obtiene el namespace de control desde el contexto actual + # del kubeconfig. Sin esto busca argocd-cm en "default". + KUBECONFIG="$temp_kubeconfig" \ + kubectl config set-context \ + --current \ + --namespace=argocd \ + >/dev/null + + if have argocd; then + KUBECONFIG="$temp_kubeconfig" \ + argocd "$@" --core || result=$? + else + docker run --rm \ + --network host \ + -e KUBECONFIG=/kubeconfig \ + -v "${temp_kubeconfig}:/kubeconfig:ro" \ + --entrypoint /usr/local/bin/argocd \ + "$ARGOCD_CLI_IMAGE" \ + "$@" \ + --core || result=$? + fi + + rm -f "$temp_kubeconfig" + return "$result" +} + +monitoring_operation_fields() { + local app="$MONITORING_APP_NAME" + + MONITORING_TARGET_REVISION="$( + kubectl get application "$app" -n argocd \ + -o jsonpath='{.spec.source.targetRevision}' 2>/dev/null || true + )" + MONITORING_OPERATION_REVISION="$( + kubectl get application "$app" -n argocd \ + -o jsonpath='{.status.operationState.syncResult.revision}' 2>/dev/null || true + )" + MONITORING_OPERATION_PHASE="$( + kubectl get application "$app" -n argocd \ + -o jsonpath='{.status.operationState.phase}' 2>/dev/null || true + )" + MONITORING_OPERATION_MESSAGE="$( + kubectl get application "$app" -n argocd \ + -o jsonpath='{.status.operationState.message}' 2>/dev/null || true + )" + MONITORING_SYNC_STATUS="$( + kubectl get application "$app" -n argocd \ + -o jsonpath='{.status.sync.status}' 2>/dev/null || true + )" + MONITORING_HEALTH_STATUS="$( + kubectl get application "$app" -n argocd \ + -o jsonpath='{.status.health.status}' 2>/dev/null || true + )" + + export MONITORING_TARGET_REVISION MONITORING_OPERATION_REVISION \ + MONITORING_OPERATION_PHASE MONITORING_OPERATION_MESSAGE \ + MONITORING_SYNC_STATUS MONITORING_HEALTH_STATUS +} + +monitoring_operation_is_stale() { + monitoring_operation_fields + [[ "$MONITORING_OPERATION_PHASE" == "Running" ]] || return 1 + [[ "$MONITORING_FORCE_RECOVERY" == "true" ]] && return 0 + + [[ -n "$MONITORING_OPERATION_REVISION" && + -n "$MONITORING_TARGET_REVISION" && + "$MONITORING_OPERATION_REVISION" != "$MONITORING_TARGET_REVISION" ]] +} + +wait_monitoring_operation_stopped() { + local deadline=$((SECONDS + MONITORING_OPERATION_STOP_WAIT)) + + while (( SECONDS < deadline )); do + monitoring_operation_fields + if [[ "$MONITORING_OPERATION_PHASE" != "Running" && + "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then + return 0 + fi + sleep 3 + done + + return 1 +} + +cleanup_monitoring_admission_jobs() { + local job="" + + while IFS= read -r job; do + [[ -n "$job" ]] || continue + kubectl delete "$job" -n "$MONITORING_NAMESPACE" --ignore-not-found=true + done < <( + kubectl get jobs -n "$MONITORING_NAMESPACE" -o name 2>/dev/null | + grep -E 'admission-(create|patch)' || true + ) +} + +wait_monitoring_synced_healthy() { + local deadline=$((SECONDS + MONITORING_RECOVERY_WAIT_SECONDS)) + + while (( SECONDS < deadline )); do + monitoring_operation_fields + + if [[ "$MONITORING_SYNC_STATUS" == "Synced" && + "$MONITORING_HEALTH_STATUS" == "Healthy" && + "$MONITORING_OPERATION_PHASE" != "Running" && + "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then + ok "${MONITORING_APP_NAME}: Synced/Healthy en ${MONITORING_TARGET_REVISION}." + return 0 + fi + + sleep 10 + done + + monitoring_app_diagnostics + return 1 +} + +recover_monitoring_operation() { + local explicit="${1:-false}" + local refresh_deadline=0 + + kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 || + die "No existe Application/${MONITORING_APP_NAME}." + + kubectl get limitrange monitoring-default-resources \ + -n "$MONITORING_NAMESPACE" >/dev/null 2>&1 || + die "Falta LimitRange/monitoring-default-resources." + + monitoring_operation_fields + + printf 'Target revision : %s\n' "${MONITORING_TARGET_REVISION:-desconocida}" + printf 'Operation revision : %s\n' "${MONITORING_OPERATION_REVISION:-ninguna}" + printf 'Operation phase : %s\n' "${MONITORING_OPERATION_PHASE:-ninguna}" + printf 'Operation message : %s\n' "${MONITORING_OPERATION_MESSAGE:-sin mensaje}" + printf 'Sync status : %s\n' "${MONITORING_SYNC_STATUS:-desconocido}" + printf 'Health status : %s\n' "${MONITORING_HEALTH_STATUS:-desconocido}" + + # No se inicia otra sincronización cuando la revisión actual ya terminó bien. + if [[ "$MONITORING_OPERATION_PHASE" == "Succeeded" && + "$MONITORING_OPERATION_REVISION" == "$MONITORING_TARGET_REVISION" ]]; then + + kubectl annotate application "$MONITORING_APP_NAME" -n argocd \ + argocd.argoproj.io/refresh=hard --overwrite >/dev/null + + refresh_deadline=$((SECONDS + 90)) + + while (( SECONDS < refresh_deadline )); do + monitoring_operation_fields + + if [[ "$MONITORING_SYNC_STATUS" == "Synced" && + "$MONITORING_HEALTH_STATUS" == "Healthy" ]]; then + ok "${MONITORING_APP_NAME} ya estaba correctamente Synced/Healthy; no se ejecutó un sync adicional." + return 0 + fi + + sleep 5 + done + + warn "La operación ${MONITORING_TARGET_REVISION} terminó, pero la Application continúa ${MONITORING_SYNC_STATUS}/${MONITORING_HEALTH_STATUS}; se solicitará una nueva reconciliación." + fi + + if monitoring_operation_is_stale || + [[ "$explicit" == "true" && "$MONITORING_OPERATION_PHASE" == "Running" ]]; then + warn "Terminando operación obsoleta/atascada de ${MONITORING_APP_NAME}." + argocd_core_cli app terminate-op "$MONITORING_APP_NAME" || + die "No fue posible terminar la operación." + + wait_monitoring_operation_stopped || + die "La operación no terminó dentro de ${MONITORING_OPERATION_STOP_WAIT}s." + + cleanup_monitoring_admission_jobs + fi + + kubectl annotate application "$MONITORING_APP_NAME" -n argocd \ + argocd.argoproj.io/refresh=hard --overwrite >/dev/null + + # La Application tiene auto-sync. Primero damos oportunidad al controller + # para reconciliar sin crear una operación manual redundante. + refresh_deadline=$((SECONDS + 90)) + + while (( SECONDS < refresh_deadline )); do + monitoring_operation_fields + + if [[ "$MONITORING_SYNC_STATUS" == "Synced" && + "$MONITORING_HEALTH_STATUS" == "Healthy" && + "$MONITORING_OPERATION_PHASE" != "Running" && + "$MONITORING_OPERATION_PHASE" != "Terminating" ]]; then + ok "${MONITORING_APP_NAME}: reconciliada automáticamente." + return 0 + fi + + if [[ "$MONITORING_OPERATION_PHASE" == "Running" ]]; then + wait_monitoring_synced_healthy || + die "${MONITORING_APP_NAME} no quedó Synced/Healthy." + return 0 + fi + + sleep 5 + done + + log "Auto-sync no inició; solicitando sync manual de ${MONITORING_APP_NAME}" + + argocd_core_cli app sync "$MONITORING_APP_NAME" \ + --async --prune --assumeYes || + die "No fue posible iniciar el sync." + + wait_monitoring_synced_healthy || + die "${MONITORING_APP_NAME} no quedó Synced/Healthy." +} + +auto_recover_monitoring_if_stale() { + [[ "$AUTO_RECOVER_STALE_MONITORING_OPERATION" == "true" ]] || return 0 + kubectl get application "$MONITORING_APP_NAME" -n argocd >/dev/null 2>&1 || return 0 + + if monitoring_operation_is_stale; then + recover_monitoring_operation false + fi +} + +ensure_root_application_present() { + [[ "$BOOTSTRAP_ROOT_APP" == "true" ]] || return 0 + + if ! kubectl get crd applications.argoproj.io >/dev/null 2>&1; then + warn "El CRD Application aún no existe; se omite la verificación de ${ROOT_APP_NAME}." + return 0 + fi + + if kubectl get application "$ROOT_APP_NAME" -n argocd >/dev/null 2>&1; then + return 0 + fi + + warn "${ROOT_APP_NAME} no existe; restaurándolo desde el clon persistente." + + if [[ "$APP_REGISTRY_REFRESH_ON_ENSURE" == "true" ]]; then + sync_apps_registry_repo + elif ! apps_registry_cache_valid; then + sync_apps_registry_repo + else + validate_apps_registry_cache + fi + + if [[ "$PLATFORM_INFRA_REFRESH_ON_ENSURE" == "true" ]]; then + sync_platform_infra_repo + elif ! platform_infra_cache_valid; then + sync_platform_infra_repo + fi + + validate_and_render_platform_infra + apply_governance_snapshots + configure_argocd_repository + apply_root_application +} + +runner_container_exists() { + docker inspect "$GITEA_RUNNER_CONTAINER" >/dev/null 2>&1 +} + +runner_is_running() { + [[ "$(docker inspect -f '{{.State.Running}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true)" == "true" ]] +} + +runner_is_registered() { + [[ -s "${GITEA_RUNNER_DATA_DIR}/.runner" ]] +} + +prepare_gitea_runner_directories() { + # El watchdog se ejecuta como devops y no dispone de TTY para sudo. + # install-autostart deja estas rutas creadas y con propietario correcto. + install -d \ + -m 0700 \ + "$GITEA_RUNNER_ROOT" \ + "$GITEA_RUNNER_DATA_DIR" || + die "No se pudieron preparar las rutas persistentes del Gitea runner como $(id -un)." +} + +detect_gitea_runner_image() { + local detected_image="" + + if [[ -n "${GITEA_RUNNER_IMAGE_OVERRIDE:-}" ]]; then + printf '%s\n' "$GITEA_RUNNER_IMAGE_OVERRIDE" + return 0 + fi + + if [[ -s "$GITEA_RUNNER_IMAGE_FILE" ]]; then + tr -d '\r\n' < "$GITEA_RUNNER_IMAGE_FILE" + return 0 + fi + + if runner_container_exists; then + detected_image="$( + docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true + )" + fi + + printf '%s\n' "${detected_image:-$GITEA_RUNNER_IMAGE}" +} + +adopt_existing_gitea_runner() { + runner_container_exists || return 0 + + local current_image data_mount="" + current_image="$( + docker inspect -f '{{.Config.Image}}' "$GITEA_RUNNER_CONTAINER" + )" + + printf '%s\n' "$current_image" > "$GITEA_RUNNER_IMAGE_FILE" + chmod 0600 "$GITEA_RUNNER_IMAGE_FILE" + + data_mount="$( + docker inspect \ + -f '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}' \ + "$GITEA_RUNNER_CONTAINER" 2>/dev/null || true + )" + + if ! runner_is_registered; then + warn "Copiando el registro actual del runner desde ${GITEA_RUNNER_CONTAINER}:/data." + docker cp \ + "${GITEA_RUNNER_CONTAINER}:/data/." \ + "${GITEA_RUNNER_DATA_DIR}/" 2>/dev/null || true + fi + + if [[ "$data_mount" == "$GITEA_RUNNER_DATA_DIR" ]]; then + docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null + return 0 + fi + + warn "El runner actual no usa el volumen persistente ${GITEA_RUNNER_DATA_DIR}." + + if [[ "$MIGRATE_EXISTING_RUNNER" != "true" ]]; then + warn "Se conserva el contenedor actual. Ejecuta MIGRATE_EXISTING_RUNNER=true $0 runner cuando no haya un workflow activo." + docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null + return 0 + fi + + runner_is_registered || + die "No se pudo recuperar ${GITEA_RUNNER_DATA_DIR}/.runner; no se eliminará el contenedor actual." + + log "Migrando ${GITEA_RUNNER_CONTAINER} al almacenamiento persistente" + docker stop "$GITEA_RUNNER_CONTAINER" >/dev/null || true + docker rm "$GITEA_RUNNER_CONTAINER" >/dev/null +} + +register_gitea_runner() { + runner_is_registered && return 0 + + [[ -s "$GITEA_RUNNER_TOKEN_FILE" ]] || + die "El runner no está registrado. Guarda un token de registro en ${GITEA_RUNNER_TOKEN_FILE} con permisos 0600." + + local runner_image + runner_image="$(detect_gitea_runner_image)" + + log "Registrando Gitea Actions runner ${GITEA_RUNNER_NAME}" + + docker pull "$runner_image" + + docker run --rm \ + -v "${GITEA_RUNNER_DATA_DIR}:/data" \ + -v "${GITEA_RUNNER_TOKEN_FILE}:/run/secrets/gitea-runner-token:ro" \ + -e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \ + -e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \ + -e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \ + --entrypoint /bin/sh \ + "$runner_image" \ + -ec ' + cd /data + act_runner register \ + --no-interactive \ + --instance "$GITEA_INSTANCE_URL" \ + --token "$(cat /run/secrets/gitea-runner-token)" \ + --name "$GITEA_RUNNER_NAME" \ + --labels "$GITEA_RUNNER_LABELS" + ' + + runner_is_registered || + die "Gitea no generó ${GITEA_RUNNER_DATA_DIR}/.runner." + + printf '%s\n' "$runner_image" > "$GITEA_RUNNER_IMAGE_FILE" + chmod 0600 "$GITEA_RUNNER_IMAGE_FILE" +} + +create_gitea_runner_container() { + runner_container_exists && return 0 + + local runner_image + runner_image="$(detect_gitea_runner_image)" + + register_gitea_runner + + log "Creando contenedor ${GITEA_RUNNER_CONTAINER}" + + docker run -d \ + --name "$GITEA_RUNNER_CONTAINER" \ + --restart unless-stopped \ + -v "${GITEA_RUNNER_DATA_DIR}:/data" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e "GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}" \ + -e "GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}" \ + -e "GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}" \ + "$runner_image" >/dev/null +} + +validate_gitea_runner() { + runner_container_exists || + die "No existe el contenedor ${GITEA_RUNNER_CONTAINER}." + + if ! runner_is_running; then + docker start "$GITEA_RUNNER_CONTAINER" >/dev/null + fi + + docker update --restart=unless-stopped "$GITEA_RUNNER_CONTAINER" >/dev/null + + local deadline=$((SECONDS + 60)) + + while (( SECONDS < deadline )); do + if runner_is_running; then + ok "Gitea runner activo: ${GITEA_RUNNER_CONTAINER}." + return 0 + fi + sleep 3 + done + + docker logs --tail 150 "$GITEA_RUNNER_CONTAINER" || true + die "El contenedor ${GITEA_RUNNER_CONTAINER} no quedó activo." +} + +ensure_gitea_runner() { + [[ "$INSTALL_GITEA_RUNNER" == "true" ]] || return 0 + + prepare_gitea_runner_directories + adopt_existing_gitea_runner + + if ! runner_container_exists; then + if ! runner_is_registered && [[ ! -s "$GITEA_RUNNER_TOKEN_FILE" ]]; then + if [[ "$GITEA_RUNNER_REQUIRED" == "true" ]]; then + die "Falta el registro persistente del runner y el token ${GITEA_RUNNER_TOKEN_FILE}." + fi + warn "Gitea runner omitido porque no hay registro ni token." + return 0 + fi + + create_gitea_runner_container + fi + + validate_gitea_runner +} + apply_bootstrap_manifests() { shopt -s nullglob local manifests=("$BOOTSTRAP_DIR"/*.yaml "$BOOTSTRAP_DIR"/*.yml) @@ -960,6 +2407,9 @@ print_credentials() { printf ' ArgoCD admin : %s\n' "$argocd_password" printf ' Token Headlamp : kubectl create token headlamp-admin -n headlamp\n' printf ' Kubeconfig : %s\n' "$KUBECONFIG_PATH" + printf ' Root Application: %s -> %s/%s@%s\n' \ + "$ROOT_APP_NAME" "$ROOT_APP_REPO_URL" "$ROOT_APP_REPO_PATH" "$ROOT_APP_REVISION" + printf ' Gitea runner : %s\n' "$GITEA_RUNNER_CONTAINER" printf '==========================================================\n' } @@ -983,6 +2433,33 @@ status_report() { kubectl get ingress -A || true kubectl get applications.argoproj.io -n argocd 2>/dev/null || true fi + + log "apps-registry local" + if apps_registry_cache_valid; then + printf 'Ruta: %s +' "$APP_REGISTRY_CACHE_DIR" + printf 'Commit: %s +' "$(git -C "$APP_REGISTRY_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)" + else + warn "No existe un clon válido en ${APP_REGISTRY_CACHE_DIR}." + fi + + log "platform-infra local" + if platform_infra_cache_valid; then + printf 'Ruta: %s +' "$PLATFORM_INFRA_CACHE_DIR" + printf 'Commit: %s +' "$(git -C "$PLATFORM_INFRA_CACHE_DIR" rev-parse --short=12 HEAD 2>/dev/null || echo desconocido)" + else + warn "No existe un clon válido en ${PLATFORM_INFRA_CACHE_DIR}." + fi + + log "Gitea Actions runner" + if runner_container_exists; then + docker ps -a --filter "name=^/${GITEA_RUNNER_CONTAINER}$" --format 'table {{.Names}} {{.Image}} {{.Status}}' + else + warn "No existe ${GITEA_RUNNER_CONTAINER}." + fi } backup_cluster_state() { @@ -1001,8 +2478,8 @@ backup_cluster_state() { --xattrs \ -C "$APP_DATA_PATH" \ -czf "$archive" \ - k3s storage secrets config bootstrap 2>/dev/null || \ - sudo_cmd tar -C "$APP_DATA_PATH" -czf "$archive" k3s storage secrets config bootstrap + k3s storage secrets config bootstrap gitea-runner git rendered 2>/dev/null || \ + sudo_cmd tar -C "$APP_DATA_PATH" -czf "$archive" k3s storage secrets config bootstrap gitea-runner git rendered sudo_cmd chown "$(id -un)" "$archive" ok "Backup creado: ${archive}" @@ -1047,6 +2524,9 @@ ensure_cluster_only() { wait_for_api wait_for_nodes validate_runtime_after_boot + ensure_root_application_present + auto_recover_monitoring_if_stale + ensure_gitea_runner } @@ -1167,6 +2647,20 @@ install_autostart() { -o "$service_user" \ "$(dirname "$KUBECONFIG_PATH")" + # El modo ensure valida y recupera el runner como usuario devops. + # Preparamos estas rutas durante la instalación privilegiada para que + # systemd nunca necesite ejecutar sudo de forma no interactiva. + install -d \ + -m 0700 \ + -o "$service_user" \ + "$GITEA_RUNNER_ROOT" \ + "$GITEA_RUNNER_DATA_DIR" + + if [[ -f "$GITEA_RUNNER_IMAGE_FILE" ]]; then + chown "$service_user" "$GITEA_RUNNER_IMAGE_FILE" + chmod 0600 "$GITEA_RUNNER_IMAGE_FILE" + fi + if [[ -f "$KUBECONFIG_PATH" ]]; then chown "$service_user" "$KUBECONFIG_PATH" chmod 0600 "$KUBECONFIG_PATH" @@ -1204,6 +2698,57 @@ KUBECTL_BIN=${kubectl_path} DOCKER_BIN=${docker_path} CURL_BIN=${curl_path} LAB_TOOL_PATH=${tool_path} +BOOTSTRAP_ROOT_APP=${BOOTSTRAP_ROOT_APP} +ROOT_APP_NAME=${ROOT_APP_NAME} +ROOT_APP_REPO_URL=${ROOT_APP_REPO_URL} +ROOT_APP_REPO_PATH=${ROOT_APP_REPO_PATH} +ROOT_APP_REVISION=${ROOT_APP_REVISION} +ROOT_APP_PROJECT=${ROOT_APP_PROJECT} +ROOT_APP_WAIT_SECONDS=${ROOT_APP_WAIT_SECONDS} +GITOPS_APPS_WAIT_SECONDS=${GITOPS_APPS_WAIT_SECONDS} +GITOPS_STRICT_HEALTH=${GITOPS_STRICT_HEALTH} +EXPECTED_GITOPS_APPS=${EXPECTED_GITOPS_APPS} +ARGOCD_REPO_USERNAME_FILE=${ARGOCD_REPO_USERNAME_FILE} +ARGOCD_REPO_PASSWORD_FILE=${ARGOCD_REPO_PASSWORD_FILE} +INSTALL_GITEA_RUNNER=${INSTALL_GITEA_RUNNER} +GITEA_RUNNER_REQUIRED=${GITEA_RUNNER_REQUIRED} +GITEA_RUNNER_CONTAINER=${GITEA_RUNNER_CONTAINER} +GITEA_RUNNER_IMAGE=${GITEA_RUNNER_IMAGE} +GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL} +GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME} +GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS} +GITEA_RUNNER_ROOT=${GITEA_RUNNER_ROOT} +GITEA_RUNNER_DATA_DIR=${GITEA_RUNNER_DATA_DIR} +GITEA_RUNNER_IMAGE_FILE=${GITEA_RUNNER_IMAGE_FILE} +GITEA_RUNNER_TOKEN_FILE=${GITEA_RUNNER_TOKEN_FILE} +MIGRATE_EXISTING_RUNNER=false +APP_REGISTRY_CACHE_DIR=${APP_REGISTRY_CACHE_DIR} +APP_REGISTRY_ROOT_MANIFEST=${APP_REGISTRY_ROOT_MANIFEST} +APP_REGISTRY_COMMIT_FILE=${APP_REGISTRY_COMMIT_FILE} +APP_REGISTRY_ALLOW_CACHE_FALLBACK=${APP_REGISTRY_ALLOW_CACHE_FALLBACK} +APP_REGISTRY_REFRESH_ON_ENSURE=${APP_REGISTRY_REFRESH_ON_ENSURE} +APP_REGISTRY_CLONE_DEPTH=${APP_REGISTRY_CLONE_DEPTH} +PLATFORM_INFRA_REPO_URL=${PLATFORM_INFRA_REPO_URL} +PLATFORM_INFRA_REVISION=${PLATFORM_INFRA_REVISION} +PLATFORM_INFRA_CACHE_DIR=${PLATFORM_INFRA_CACHE_DIR} +PLATFORM_INFRA_COMMIT_FILE=${PLATFORM_INFRA_COMMIT_FILE} +PLATFORM_INFRA_ALLOW_CACHE_FALLBACK=${PLATFORM_INFRA_ALLOW_CACHE_FALLBACK} +PLATFORM_INFRA_REFRESH_ON_ENSURE=${PLATFORM_INFRA_REFRESH_ON_ENSURE} +PLATFORM_INFRA_CLONE_DEPTH=${PLATFORM_INFRA_CLONE_DEPTH} +PLATFORM_INFRA_RENDER_DIR=${PLATFORM_INFRA_RENDER_DIR} +APPLY_GOVERNANCE_SNAPSHOTS=${APPLY_GOVERNANCE_SNAPSHOTS} +ENFORCE_MONITORING_LIMITRANGE=${ENFORCE_MONITORING_LIMITRANGE} +MONITORING_GOVERNANCE_PATH=${MONITORING_GOVERNANCE_PATH} +ARGOCD_PLATFORM_REPO_SECRET_NAME=${ARGOCD_PLATFORM_REPO_SECRET_NAME} +MONITORING_APP_NAME=${MONITORING_APP_NAME} +MONITORING_NAMESPACE=${MONITORING_NAMESPACE} +MONITORING_FIXED_CHART_VERSION=${MONITORING_FIXED_CHART_VERSION} +MONITORING_INGRESS_HOST=${MONITORING_INGRESS_HOST} +MONITORING_RECOVERY_WAIT_SECONDS=${MONITORING_RECOVERY_WAIT_SECONDS} +MONITORING_OPERATION_STOP_WAIT=${MONITORING_OPERATION_STOP_WAIT} +AUTO_RECOVER_STALE_MONITORING_OPERATION=${AUTO_RECOVER_STALE_MONITORING_OPERATION} +MONITORING_FORCE_RECOVERY=false +ARGOCD_CLI_IMAGE=${ARGOCD_CLI_IMAGE} ENV cat > /etc/systemd/system/k3d-lab-ensure.service <