#!/usr/bin/env bash
# dokploy-cluster.sh — single-file bootstrapper for the architecture in
# GUIDE.md (Dokploy on Docker Swarm — Hetzner Cloud, Fedora 44).
#
# Two ways to drive it:
#   ./provision.sh up       interactive end-to-end run — checks dependencies,
#                           prompts for secrets, VPS types and datacenter,
#                           then executes the pipeline with guided pauses at
#                           the manual interludes
#   ./provision.sh <step>   one pipeline step at a time (see ./provision.sh)
#
# Guarantees, stated plainly: guarded re-entrancy (safe to re-run, existing
# resources are skipped), NOT convergence — this script will not reconcile
# drift or config changes. If you want managed end-state, that's Terraform
# (hcloud provider) + Ansible; this is the readable middle ground.
#
# Secrets policy: TS_AUTHKEY / CF_DNS_API_TOKEN enter via environment or an
# interactive hidden prompt only. The CF token travels to mgr-1 over ssh
# stdin and lives as a Docker secret, consumed via lego's
# CF_DNS_API_TOKEN_FILE convention — never in argv, never in the Traefik
# service spec as plaintext.

set -euo pipefail

### ── TUI ───────────────────────────────────────────────────────────────────
if [[ -t 1 && -z ${NO_COLOR:-} ]]; then
  BOLD=$'\e[1m' DIM=$'\e[2m' RED=$'\e[31m' GRN=$'\e[32m'
  YLW=$'\e[33m' BLU=$'\e[34m' CYN=$'\e[36m' RST=$'\e[0m'
else
  BOLD='' DIM='' RED='' GRN='' YLW='' BLU='' CYN='' RST=''
fi

die()  { printf '%s✗ %s%s\n' "$RED" "$*" "$RST" >&2; exit 1; }
info() { printf '%s==>%s %s\n' "$CYN" "$RST" "$*"; }
ok()   { printf '  %s✓%s %s\n' "$GRN" "$RST" "$*"; }
warn() { printf '  %s!%s %s\n' "$YLW" "$RST" "$*"; }
hdr()  { printf '\n%s%s▌ %s%s\n' "$BLU" "$BOLD" "$*" "$RST"; }

banner() {
  printf '%s' "$CYN"
  cat <<'ART'
  ╭────────────────────────────────────────────────────╮
  │  dokploy-cluster · Hetzner Cloud · Docker Swarm    │
  │  6 nodes · Traefik · Cloudflare · Tailscale SSH    │
  |         Copyright 2026 (C) LVQ Consult ApS         |
  ╰────────────────────────────────────────────────────╯
ART
  printf '%s' "$RST"
}

# confirm_yes "question"  → default Yes, returns 1 only on explicit n/N
confirm_yes() {
  local a
  read -rp "  ${YLW}?${RST} $1 ${DIM}[Y/n]${RST} " a
  [[ ! $a =~ ^[Nn] ]]
}

pause() { read -rp $'\n'"  ${YLW}⏸${RST} $1 — ${BOLD}press Enter when done${RST} " _; echo; }

# ask_plain VAR "hint"  /  ask_secret VAR "hint"  → prompt until non-empty
ask_plain() {
  local val
  while :; do
    read -rp "  ${BOLD}$1${RST} ${DIM}($2)${RST}: " val
    [[ -n $val ]] && break; warn "cannot be empty"
  done
  printf -v "$1" %s "$val"
}
ask_secret() {
  local val
  while :; do
    read -rsp "  ${BOLD}$1${RST} ${DIM}($2 — input hidden)${RST}: " val; echo
    [[ -n $val ]] && break; warn "cannot be empty"
  done
  printf -v "$1" %s "$val"
}

# menu "prompt" default_index "row" …  → sets MENU_IDX (1-based)
menu() {
  local prompt=$1 def=$2; shift 2
  local i sel
  for ((i = 1; i <= $#; i++)); do
    if (( i == def )); then
      printf '  %s%2d)%s %s %s◀%s\n' "$BOLD" "$i" "$RST" "${!i}" "$GRN" "$RST"
    else
      printf '  %s%2d)%s %s\n' "$DIM" "$i" "$RST" "${!i}"
    fi
  done
  while :; do
    read -rp "  ${YLW}?${RST} ${prompt} ${DIM}[${def}]${RST}: " sel
    sel=${sel:-$def}
    [[ $sel =~ ^[0-9]+$ ]] && (( sel >= 1 && sel <= $# )) && { MENU_IDX=$sel; return; }
    warn "enter a number between 1 and $#"
  done
}

### ── Config: env-overridable defaults; 'up' asks interactively ────────────
LOCATION=${LOCATION:-fsn1}
MGR_TYPE=${MGR_TYPE:-cx32}
WRK_TYPE=${WRK_TYPE:-cx22}
SSH_KEY_NAME=${SSH_KEY_NAME:-}
IMAGE=fedora-44
NET_NAME=dokploy-net   NET_RANGE=10.100.0.0/16
SUBNET=10.100.1.0/24                  # network zone is derived from LOCATION
FW_NAME=dokploy-fw
LB_NAME=dokploy-lb     LB_TYPE=lb11
TRAEFIK_IMAGE=traefik:v3.6.7          # check Dokploy release notes before bumping
SWARM_POOL=10.201.0.0/16              # disjoint from SUBNET by construction
MANAGERS=(dokploy-mgr-1 dokploy-mgr-2 dokploy-mgr-3)
WORKERS=(dokploy-wrk-1 dokploy-wrk-2 dokploy-wrk-3)
ALL=("${MANAGERS[@]}" "${WORKERS[@]}")
PRIMARY=${MANAGERS[0]}
SSH_OPTS=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=8)  # TOFU; tighten if you prefer
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
CLOUD_INIT=$SCRIPT_DIR/cloud-init-dokploy-node.yaml

### ── Workspace: one tmp dir, cleaned on any exit ───────────────────────────
WORK=$(mktemp -d "${TMPDIR:-/tmp}/dokploy-cluster.XXXXXX")
trap 'rm -rf "$WORK"' EXIT

### ── Helpers ───────────────────────────────────────────────────────────────
need() { command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1 — run $0 deps"; }
run()  { local h=$1; shift; ssh "${SSH_OPTS[@]}" "root@$h" "$@"; }   # Tailscale MagicDNS names
priv_ip() { hcloud server describe "$1" -o json | jq -r '.private_net[0].ip'; }

# require_secret VAR "hint": env wins; interactive prompt on a tty; else die
require_secret() {
  [[ -n ${!1:-} ]] && return 0
  [[ -t 0 ]] || die "export $1 — $2"
  ask_secret "$1" "$2"
}

### ── deps: check first, then ask before installing anything ───────────────
DEPS=(hcloud jq rsync ssh curl)

detect_pm() {
  local p
  for p in dnf apt-get pacman zypper brew; do
    command -v "$p" >/dev/null 2>&1 && { echo "$p"; return; }
  done
  echo ""
}

pkg_for() {  # dep, pm → distro package name
  case "$1:$2" in
    ssh:dnf|ssh:zypper) echo openssh-clients ;;
    ssh:apt-get)        echo openssh-client ;;
    ssh:pacman)         echo openssh ;;
    hcloud:apt-get)     echo hcloud-cli ;;
    *)                  echo "$1" ;;
  esac
}

pm_install() {  # pm, package
  local sudo=""
  [[ $1 != brew && $EUID -ne 0 ]] && sudo=sudo
  case $1 in
    dnf)     $sudo dnf -y install "$2" ;;
    apt-get) $sudo apt-get update -qq && $sudo apt-get -y install "$2" ;;
    pacman)  $sudo pacman -S --noconfirm "$2" ;;
    zypper)  $sudo zypper -n install "$2" ;;
    brew)    brew install "$2" ;;
  esac
}

install_hcloud_binary() {  # official release binary → ~/.local/bin
  local os arch
  case "$(uname -s)" in
    Linux)  os=linux ;;
    Darwin) die "install hcloud via Homebrew on macOS: brew install hcloud" ;;
    *)      die "unsupported OS for automatic hcloud install: $(uname -s)" ;;
  esac
  case "$(uname -m)" in
    x86_64)        arch=amd64 ;;
    aarch64|arm64) arch=arm64 ;;
    *)             die "unsupported architecture: $(uname -m)" ;;
  esac
  info "downloading hcloud CLI (GitHub releases, ${os}-${arch})…"
  curl -fsSL "https://github.com/hetznercloud/cli/releases/latest/download/hcloud-${os}-${arch}.tar.gz" \
    | tar -xz -C "$WORK" hcloud
  mkdir -p "$HOME/.local/bin"
  install -m 0755 "$WORK/hcloud" "$HOME/.local/bin/hcloud"
  case ":$PATH:" in
    *":$HOME/.local/bin:"*) ;;
    *) warn "~/.local/bin is not in your PATH — add it to your shell profile" ;;
  esac
  ok "hcloud installed to ~/.local/bin/hcloud"
}

cmd_deps() {
  hdr "Dependencies"
  local -a missing=()
  local d
  for d in "${DEPS[@]}"; do
    if command -v "$d" >/dev/null 2>&1; then
      ok "$d ${DIM}$(command -v "$d")${RST}"
    else
      warn "$d — not found"
      missing+=("$d")
    fi
  done
  (( ${#missing[@]} )) || { ok "all dependencies present"; return 0; }

  [[ -t 0 ]] || die "missing dependencies: ${missing[*]}"
  echo
  printf '  Missing: %s%s%s\n' "$BOLD" "${missing[*]}" "$RST"
  printf '  %sy%s) Yes, install them now\n' "$BOLD" "$RST"
  printf '  %sn%s) No, I know what I'\''m doing — I already have those installed\n' "$BOLD" "$RST"
  local a
  read -rp "  ${YLW}?${RST} Install missing dependencies? ${DIM}[Y/n]${RST} " a
  if [[ $a =~ ^[Nn] ]]; then
    warn "skipping installs — the pipeline will fail loudly if one is truly missing"
    return 0
  fi

  local pm; pm=$(detect_pm)
  for d in "${missing[@]}"; do
    info "installing $d"
    if [[ $d == hcloud ]]; then
      if [[ -n $pm ]] && pm_install "$pm" "$(pkg_for hcloud "$pm")" >/dev/null 2>&1 \
         && command -v hcloud >/dev/null 2>&1; then
        ok "hcloud ${DIM}(via $pm)${RST}"
      else
        install_hcloud_binary
      fi
    else
      [[ -n $pm ]] || die "no supported package manager found — install ${missing[*]} manually"
      pm_install "$pm" "$(pkg_for "$d" "$pm")"
      command -v "$d" >/dev/null 2>&1 || die "$d still missing after install"
      ok "$d ${DIM}(via $pm)${RST}"
    fi
  done
}

ensure_hcloud_context() {
  local ctx
  ctx=$(hcloud context active 2>/dev/null || true)
  if [[ -z $ctx ]]; then
    warn "hcloud has no active context (no API token configured)"
    [[ -t 0 ]] || die "create one first: hcloud context create dokploy"
    confirm_yes "Create one now? (hcloud will ask for a Hetzner Cloud API token)" \
      || die "aborted — create a context with: hcloud context create dokploy"
    hcloud context create dokploy
  fi
  ok "hcloud context: $(hcloud context active)"
}

### ── interactive configuration: datacenter, VPS types, SSH key ────────────
configure_cluster() {
  CONFIGURED=1
  need hcloud; need jq
  hdr "Cluster configuration"

  info "fetching Hetzner datacenter locations…"
  local loc_json; loc_json=$(hcloud location list -o json)
  local -a loc_names=() loc_rows=()
  local name city country zone def=1 i=0
  while IFS=$'\t' read -r name city country zone; do
    i=$((i + 1)); loc_names+=("$name")
    [[ $name == "$LOCATION" ]] && def=$i
    loc_rows+=("$(printf '%-6s %-18s %-3s %s%s%s' "$name" "$city" "$country" "$DIM" "$zone" "$RST")")
  done < <(jq -r '.[] | [.name, .city, .country, .network_zone] | @tsv' <<<"$loc_json")
  menu "Datacenter for all nodes" "$def" "${loc_rows[@]}"
  LOCATION=${loc_names[MENU_IDX-1]}
  ok "datacenter: $LOCATION ${DIM}($(jq -r --arg l "$LOCATION" '.[] | select(.name == $l).network_zone' <<<"$loc_json"))${RST}"

  info "fetching server types orderable in $LOCATION…"
  ST_NAMES=() ST_ROWS=()
  local arch cores mem disk price
  while IFS=$'\t' read -r name arch cores mem disk price; do
    ST_NAMES+=("$name")
    ST_ROWS+=("$(printf '%-8s %-5s %2s vCPU %5s GB RAM %5s GB disk  %s€%s/mo%s' \
      "$name" "$arch" "$cores" "$mem" "$disk" "$DIM" "$price" "$RST")")
  done < <(jq -r --arg loc "$LOCATION" '
      [ .[]
        | select((.deprecated // false) | not)
        | . + {price: ([.prices[] | select(.location == $loc) | .price_monthly.gross] | first)}
        | select(.price != null) ]
      | sort_by(.architecture, .cores, .memory)[]
      | [.name, .architecture, (.cores | tostring), (.memory | floor | tostring),
         (.disk | tostring), ((.price | tonumber * 100 | floor) / 100 | tostring)]
      | @tsv' <<<"$(hcloud server-type list -o json)")
  (( ${#ST_NAMES[@]} )) || die "no server types orderable in $LOCATION — pick another datacenter"

  _pick_type "Manager node type (×${#MANAGERS[@]})" "$MGR_TYPE"; MGR_TYPE=$PICKED
  _pick_type "Worker node type (×${#WORKERS[@]})"  "$WRK_TYPE"; WRK_TYPE=$PICKED

  info "fetching SSH keys in this hcloud project…"
  local keys_json; keys_json=$(hcloud ssh-key list -o json)
  local -a key_names=() key_rows=()
  local fp; def=1; i=0
  while IFS=$'\t' read -r name fp; do
    i=$((i + 1)); key_names+=("$name")
    [[ $name == "$SSH_KEY_NAME" ]] && def=$i
    key_rows+=("$(printf '%-20s %s%s%s' "$name" "$DIM" "$fp" "$RST")")
  done < <(jq -r '.[] | [.name, .fingerprint] | @tsv' <<<"$keys_json")
  (( ${#key_names[@]} )) || die "no SSH keys in this hcloud project — add one first:
  hcloud ssh-key create --name me --public-key-from-file ~/.ssh/id_ed25519.pub"
  menu "SSH key for root access to the nodes" "$def" "${key_rows[@]}"
  SSH_KEY_NAME=${key_names[MENU_IDX-1]}

  hdr "Cluster plan"
  printf '  %-12s %s\n' "datacenter" "$LOCATION" \
                        "managers"   "${#MANAGERS[@]} × $MGR_TYPE" \
                        "workers"    "${#WORKERS[@]} × $WRK_TYPE" \
                        "ssh key"    "$SSH_KEY_NAME" \
                        "image"      "$IMAGE" \
                        "lb"         "$LB_TYPE ($LB_NAME)"
  confirm_yes "Proceed with this configuration?" || die "aborted — nothing was created"
}

_pick_type() {  # prompt, current-default-name → sets PICKED
  local def=1 i
  for ((i = 0; i < ${#ST_NAMES[@]}; i++)); do
    [[ ${ST_NAMES[i]} == "$2" ]] && def=$((i + 1))
  done
  menu "$1" "$def" "${ST_ROWS[@]}"
  PICKED=${ST_NAMES[MENU_IDX-1]}
  ok "$1: $PICKED"
}

collect_secrets() {
  hdr "Secrets"
  printf '  %sSecrets stay in this process'\''s memory (or your environment) only;\n' "$DIM"
  printf '  the Cloudflare token reaches mgr-1 via ssh stdin as a Docker secret.%s\n\n' "$RST"
  if [[ -n ${TS_AUTHKEY:-} ]]; then
    ok "TS_AUTHKEY ${DIM}(from environment)${RST}"
  else
    ask_secret TS_AUTHKEY "tagged, reusable Tailscale auth key — GUIDE §2-B"
    [[ $TS_AUTHKEY == tskey-* ]] || warn "that doesn't look like a Tailscale auth key (expected tskey-…)"
  fi
  if [[ -n ${ACME_EMAIL:-} ]]; then
    ok "ACME_EMAIL ${DIM}(from environment)${RST}"
  else
    while :; do
      ask_plain ACME_EMAIL "email for Let's Encrypt expiry notices"
      [[ $ACME_EMAIL == *@* ]] && break; warn "that doesn't look like an email address"
    done
  fi
  if [[ -n ${CF_DNS_API_TOKEN:-} ]]; then
    ok "CF_DNS_API_TOKEN ${DIM}(from environment)${RST}"
  else
    ask_secret CF_DNS_API_TOKEN "Cloudflare API token with Zone:Read + DNS:Edit — GUIDE §9-A"
  fi
}

### ── provision (§2): network, firewall, 6 servers with cloud-init ─────────
cmd_provision() {
  need hcloud; need jq
  hdr "Provision — network, firewall, ${#ALL[@]} servers (§2)"
  [[ -t 0 && -z ${CONFIGURED:-} ]] && configure_cluster
  require_secret TS_AUTHKEY "tskey-auth-… (tagged, reusable — §2-B)"
  [[ -n $SSH_KEY_NAME ]] || die "set SSH_KEY_NAME (an SSH key name from: hcloud ssh-key list)"
  [[ -f $CLOUD_INIT ]] || die "$CLOUD_INIT not found next to this script (§2-B)"
  local net_zone
  net_zone=$(hcloud location describe "$LOCATION" -o json | jq -r .network_zone)

  hcloud network describe "$NET_NAME" >/dev/null 2>&1 || {
    info "network $NET_NAME"
    hcloud network create --name "$NET_NAME" --ip-range "$NET_RANGE"
    hcloud network add-subnet "$NET_NAME" --type cloud --network-zone "$net_zone" --ip-range "$SUBNET"
  }
  hcloud firewall describe "$FW_NAME" >/dev/null 2>&1 || {
    info "firewall $FW_NAME (public: 22/tcp, 41641/udp, icmp — nothing else, per §2)"
    hcloud firewall create --name "$FW_NAME"
    hcloud firewall add-rule "$FW_NAME" --direction in --protocol tcp  --port 22    --source-ips 0.0.0.0/0 --source-ips ::/0 --description "public ssh"
    hcloud firewall add-rule "$FW_NAME" --direction in --protocol udp  --port 41641 --source-ips 0.0.0.0/0 --source-ips ::/0 --description "tailscale"
    hcloud firewall add-rule "$FW_NAME" --direction in --protocol icmp              --source-ips 0.0.0.0/0 --source-ips ::/0 --description "icmp"
  }

  local rendered=$WORK/cloud-init-rendered.yaml
  sed "s|__TS_AUTHKEY__|$TS_AUTHKEY|" "$CLOUD_INIT" > "$rendered"

  local s
  for s in "${MANAGERS[@]}"; do
    hcloud server describe "$s" >/dev/null 2>&1 && { ok "$s exists, skipping"; continue; }
    info "creating $s ($MGR_TYPE in $LOCATION)"
    hcloud server create --name "$s" --type "$MGR_TYPE" --image "$IMAGE" --location "$LOCATION" \
      --ssh-key "$SSH_KEY_NAME" --network "$NET_NAME" --firewall "$FW_NAME" \
      --user-data-from-file "$rendered" --label cluster=dokploy --label role=manager
  done
  for s in "${WORKERS[@]}"; do
    hcloud server describe "$s" >/dev/null 2>&1 && { ok "$s exists, skipping"; continue; }
    info "creating $s ($WRK_TYPE in $LOCATION)"
    hcloud server create --name "$s" --type "$WRK_TYPE" --image "$IMAGE" --location "$LOCATION" \
      --ssh-key "$SSH_KEY_NAME" --network "$NET_NAME" --firewall "$FW_NAME" \
      --user-data-from-file "$rendered" --label cluster=dokploy --label role=worker
  done

  info "IP map:"
  for s in "${ALL[@]}"; do
    printf '  %-14s public=%-15s private=%s\n' "$s" \
      "$(hcloud server describe "$s" -o json | jq -r '.public_net.ipv4.ip')" "$(priv_ip "$s")"
  done
  warn "revoke the auth key in the Tailscale console once all ${#ALL[@]} nodes appear, then: $0 wait"
}

### ── wait (§2-B): block until cloud-init finished on every node ───────────
cmd_wait() {
  hdr "Waiting for cloud-init on all ${#ALL[@]} nodes (§2-B)"
  info "first boot = full upgrade + reboot; several minutes per node is normal"
  local s
  for s in "${ALL[@]}"; do
    _wait_node "$s" || die "$s not ready after 10 min — check /var/log/cloud-init-output.log on the node"
  done
  ok "all nodes bootstrapped"
}

_wait_node() {
  local s=$1 frames='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏' t=0
  while (( t < 600 )); do
    if (( t % 10 == 0 )) && run "$s" 'cloud-init status 2>/dev/null' 2>/dev/null | grep -q 'status: done'; then
      printf '\r  %s✓%s %-14s ready %s(%dm%02ds)%s          \n' \
        "$GRN" "$RST" "$s" "$DIM" $((t / 60)) $((t % 60)) "$RST"
      return 0
    fi
    printf '\r  %s%s%s %-14s waiting… %s%dm%02ds%s ' \
      "$CYN" "${frames:t % 10:1}" "$RST" "$s" "$DIM" $((t / 60)) $((t % 60)) "$RST"
    sleep 1; t=$((t + 1))
  done
  printf '\n'; return 1
}

### ── swarm-init (§7): init on PRIMARY + MTU-1400 ingress ──────────────────
cmd_swarm_init() {
  need hcloud; need jq
  hdr "Swarm init on $PRIMARY (§7)"
  local ip; ip=$(priv_ip "$PRIMARY")
  [[ $ip == 10.100.1.* ]] || die "unexpected private IP for $PRIMARY: $ip"
  if run "$PRIMARY" "docker info --format '{{.Swarm.LocalNodeState}}'" | grep -q '^active'; then
    ok "swarm already active on $PRIMARY"; return 0
  fi
  run "$PRIMARY" "docker swarm init --advertise-addr $ip --data-path-addr $ip \
    --default-addr-pool $SWARM_POOL --default-addr-pool-mask-length 24"
  info "recreating ingress at MTU 1400 (§7 — Verified item 6)"
  run "$PRIMARY" "echo y | docker network rm ingress; sleep 3; \
    docker network create --driver overlay --ingress --opt com.docker.network.driver.mtu=1400 ingress"
  ok "swarm initialized"
}

### ── dokploy-install (§8): network, secrets, postgres, dokploy ────────────
cmd_dokploy_install() {
  hdr "Dokploy install on $PRIMARY (§8)"
  if run "$PRIMARY" "docker service inspect dokploy >/dev/null 2>&1"; then
    ok "dokploy service already exists"; return 0
  fi
  run "$PRIMARY" bash -s <<'REMOTE'
set -euo pipefail
docker network inspect dokploy-network >/dev/null 2>&1 || \
  docker network create --driver overlay --attachable \
    --opt com.docker.network.driver.mtu=1400 dokploy-network
mkdir -p /etc/dokploy && chmod 777 /etc/dokploy
docker secret inspect dokploy_postgres_password >/dev/null 2>&1 || \
  openssl rand -base64 32 | tr -d "=+/" | cut -c1-32 | docker secret create dokploy_postgres_password -
docker secret inspect dokploy_auth_secret >/dev/null 2>&1 || \
  openssl rand -hex 32 | docker secret create dokploy_auth_secret -

docker service create \
  --name dokploy-postgres \
  --constraint "node.hostname == $(hostname)" \
  --network dokploy-network \
  --env POSTGRES_USER=dokploy --env POSTGRES_DB=dokploy \
  --secret source=dokploy_postgres_password,target=/run/secrets/postgres_password \
  --env POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password \
  --mount type=volume,source=dokploy-postgres,target=/var/lib/postgresql/data \
  postgres:16

docker service create \
  --name dokploy --replicas 1 \
  --network dokploy-network \
  --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
  --mount type=bind,source=/etc/dokploy,target=/etc/dokploy \
  --mount type=volume,source=dokploy,target=/root/.docker \
  --secret source=dokploy_postgres_password,target=/run/secrets/postgres_password \
  --secret source=dokploy_auth_secret,target=/run/secrets/dokploy_auth_secret \
  --publish published=3000,target=3000,mode=host \
  --update-parallelism 1 --update-order stop-first \
  --constraint "node.hostname == $(hostname)" \
  -e POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password \
  -e BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret \
  dokploy/dokploy:latest

echo -n "waiting for Traefik config generation "
for i in $(seq 1 60); do
  [[ -f /etc/dokploy/traefik/traefik.yml ]] && { echo ok; break; }
  echo -n .; sleep 2
done
[[ -f /etc/dokploy/traefik/traefik.yml ]] || { echo "traefik.yml never appeared"; exit 1; }
if docker ps -q --filter name=dokploy-traefik | grep -q .; then
  echo "removing stray standalone traefik container (we deploy the service variant)"
  docker rm -f dokploy-traefik
fi
REMOTE
  ok "Dokploy up"
  info "✋ Create the admin account now: http://${PRIMARY}.<tailnet>.ts.net:3000, then: $0 join"
}

### ── join (§9): managers then workers, IPs supplied from hcloud ───────────
cmd_join() {
  need hcloud; need jq
  hdr "Joining remaining nodes (§9)"
  local p1 mtok wtok s ip
  p1=$(priv_ip "$PRIMARY")
  mtok=$(run "$PRIMARY" docker swarm join-token -q manager)
  wtok=$(run "$PRIMARY" docker swarm join-token -q worker)
  for s in "${MANAGERS[@]:1}" "${WORKERS[@]}"; do
    if run "$s" "docker info --format '{{.Swarm.LocalNodeState}}'" | grep -q '^active'; then
      ok "$s already joined"; continue
    fi
    ip=$(priv_ip "$s")
    local tok=$wtok; [[ " ${MANAGERS[*]} " == *" $s "* ]] && tok=$mtok
    info "joining $s ($ip)"
    run "$s" "docker swarm join --advertise-addr $ip --data-path-addr $ip --token $tok ${p1}:2377"
  done
  run "$PRIMARY" docker node ls
}

### ── traefik (§9-A): config edits, distribution, global service ───────────
cmd_traefik() {
  need jq; need rsync; need hcloud
  hdr "Traefik — global service, CF DNS-01 certificates (§9-A)"
  require_secret ACME_EMAIL "email for Let's Encrypt expiry notices"
  require_secret CF_DNS_API_TOKEN "Cloudflare token, Zone:Read + DNS:Edit — §9-A"

  info "editing traefik.yml structurally (backup kept; key mismatch fails loud, file untouched)"
  run "$PRIMARY" "rpm -q python3-pyyaml >/dev/null 2>&1 || dnf -y install python3-pyyaml"
  run "$PRIMARY" "cp -n /etc/dokploy/traefik/traefik.yml /etc/dokploy/traefik/traefik.yml.orig; \
                  ACME_EMAIL='$ACME_EMAIL' python3 -" <<'PY'
import os, yaml
p = '/etc/dokploy/traefik/traefik.yml'
c = yaml.safe_load(open(p))
for ep in ('web', 'websecure'):
    c['entryPoints'][ep]['proxyProtocol'] = {'trustedIPs': ['10.100.1.0/24']}
c['entryPoints']['websecure'].pop('http3', None)   # no UDP through Hetzner LB (Verified item 8)
c['ping'] = {}                                     # GET /ping on :8080 (api.insecure already opens it)
acme = c['certificatesResolvers']['letsencrypt']['acme']
acme['email'] = os.environ['ACME_EMAIL']
acme.pop('httpChallenge', None)
acme['dnsChallenge'] = {'provider': 'cloudflare', 'resolvers': ['1.1.1.1:53', '8.8.8.8:53']}
yaml.safe_dump(c, open(p, 'w'), sort_keys=False)
print('traefik.yml rewritten')
PY

  info "storing Cloudflare token as a Docker secret (stdin only — never argv)"
  if ! run "$PRIMARY" "docker secret inspect cf_dns_token >/dev/null 2>&1"; then
    printf %s "$CF_DNS_API_TOKEN" | run "$PRIMARY" "docker secret create cf_dns_token -"
  fi

  _sync_traefik_conf

  if run "$PRIMARY" "docker service inspect dokploy-traefik >/dev/null 2>&1"; then
    info "dokploy-traefik service exists — forcing redeploy with current config"
    run "$PRIMARY" "docker service update --force dokploy-traefik" >/dev/null
  else
    run "$PRIMARY" "docker service create \
      --name dokploy-traefik --mode global --constraint 'node.role==manager' \
      --network dokploy-network \
      --secret source=cf_dns_token,target=cf_dns_token \
      --env CF_DNS_API_TOKEN_FILE=/run/secrets/cf_dns_token \
      --mount type=bind,source=/etc/dokploy/traefik/traefik.yml,target=/etc/traefik/traefik.yml \
      --mount type=bind,source=/etc/dokploy/traefik/dynamic,target=/etc/dokploy/traefik/dynamic \
      --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock,readonly \
      --publish mode=host,published=80,target=80 \
      --publish mode=host,published=443,target=443 \
      --publish mode=host,published=8080,target=8080 \
      $TRAEFIK_IMAGE"
  fi
  _ping_managers
}

_sync_traefik_conf() {   # workstation-mediated; acme.json stays per-node (§9-A)
  local t=$WORK/traefik-sync m
  rm -rf "$t"; mkdir -p "$t"
  rsync -a --exclude acme.json -e "ssh ${SSH_OPTS[*]}" "root@$PRIMARY:/etc/dokploy/traefik/" "$t/"
  for m in "${MANAGERS[@]:1}"; do
    run "$m" "mkdir -p /etc/dokploy/traefik/dynamic"
    rsync -a --exclude acme.json -e "ssh ${SSH_OPTS[*]}" "$t/" "root@$m:/etc/dokploy/traefik/"
  done
}

_ping_managers() {
  local m ip
  for m in "${MANAGERS[@]}"; do
    ip=$(priv_ip "$m")
    printf '  %s (%s) /ping -> ' "$m" "$ip"
    run "$PRIMARY" "curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://$ip:8080/ping" || true
    echo
  done
}

### ── lb (§10): create, attach, target managers, 80/443 + /ping health ─────
cmd_lb() {
  need hcloud; need jq
  hdr "Hetzner Load Balancer (§10)"
  hcloud load-balancer describe "$LB_NAME" >/dev/null 2>&1 || {
    hcloud load-balancer create --name "$LB_NAME" --type "$LB_TYPE" --location "$LOCATION"
    hcloud load-balancer attach-to-network "$LB_NAME" --network "$NET_NAME"
    hcloud load-balancer add-target "$LB_NAME" --label-selector role=manager --use-private-ip
    local p
    for p in 80 443; do   # ⚠ Unverified item B: confirm flags via `hcloud load-balancer add-service --help`
      hcloud load-balancer add-service "$LB_NAME" \
        --protocol tcp --listen-port "$p" --destination-port "$p" --proxy-protocol \
        --health-check-protocol http --health-check-port 8080 --health-check-http-path /ping \
        --health-check-interval 5s --health-check-timeout 3s --health-check-retries 3
    done
  }
  ok "LB public IPv4: $(hcloud load-balancer describe "$LB_NAME" -o json | jq -r '.public_net.ipv4.ip')"
  info "✋ Cloudflare next, IN ORDER (§10-B): SSL mode Full (strict) → records to the IP above, Proxied → then: $0 cf-allowlist"
}

### ── cf-allowlist (§10-B steps 3-5) ────────────────────────────────────────
cmd_cf_allowlist() {
  hdr "Cloudflare origin allowlist (§10-B)"
  if [[ -z ${CF_CONFIRMED:-} ]]; then
    cat >&2 <<'WARN'
!! Ordering check (§10-B): run this ONLY after (1) zone SSL mode = Full (strict)
!! and (2) the app records are Proxied. Running it while records are DNS-only
!! serves 403 to your real users. Ctrl-C now if that isn't done.
WARN
    sleep 5
  fi
  run "$PRIMARY" "python3 -" <<'PY'
import urllib.request, yaml
get = lambda u: urllib.request.urlopen(u, timeout=10).read().decode().split()
ranges = get('https://www.cloudflare.com/ips-v4') + get('https://www.cloudflare.com/ips-v6')
with open('/etc/dokploy/traefik/dynamic/cf-allowlist.yml', 'w') as f:
    yaml.safe_dump({'http': {'middlewares': {'cf-only': {'ipAllowList': {'sourceRange': ranges}}}}}, f)
p = '/etc/dokploy/traefik/traefik.yml'
c = yaml.safe_load(open(p))
for ep in ('web', 'websecure'):
    e = c['entryPoints'][ep]
    e['forwardedHeaders'] = {'trustedIPs': ranges}                 # trust CF's X-Forwarded-For
    e.setdefault('http', {})['middlewares'] = ['cf-only@file']     # reject non-Cloudflare
yaml.safe_dump(c, open(p, 'w'), sort_keys=False)
print(f'{len(ranges)} Cloudflare ranges applied (middleware + forwardedHeaders)')
PY
  _sync_traefik_conf
  run "$PRIMARY" "docker service update --force dokploy-traefik" >/dev/null
  info "re-run this same command whenever Cloudflare's published ranges change"
}

### ── verify (§12-ish): cluster, ingress, LB health ─────────────────────────
cmd_verify() {
  need hcloud; need jq
  hdr "Verify — cluster, Traefik, LB health (§12)"
  run "$PRIMARY" docker node ls
  run "$PRIMARY" "docker service ps dokploy-traefik --format '{{.Node}} {{.CurrentState}}'"
  _ping_managers
  hcloud load-balancer describe "$LB_NAME" -o json | \
    jq -r '.targets[].health_status[]? | "LB target \(.listen_port): \(.status)"' 2>/dev/null || \
    hcloud load-balancer describe "$LB_NAME"
}

### ── up: the whole pipeline, interactively ─────────────────────────────────
cmd_up() {
  [[ -t 0 ]] || die "'up' is interactive — run it from a terminal"
  banner
  cmd_deps
  need hcloud; need jq; need rsync; need ssh; need curl
  ensure_hcloud_context

  hdr "Pre-flight — Tailscale (GUIDE §2-B)"
  cat <<EOF
  The tailnet must be prepared before any server exists:
    1. ACL: ${BOLD}tagOwners${RST} entry for ${BOLD}tag:dokploy${RST}
    2. ACL: ssh ${BOLD}accept${RST} rule for members → tag:dokploy as root
    3. Auth key: ${BOLD}reusable${RST}, tagged ${BOLD}tag:dokploy${RST}, short expiry, NOT ephemeral
EOF
  confirm_yes "Tailnet prepared?" || die "prepare the tailnet first (GUIDE §2-B), then re-run: $0 up"

  collect_secrets
  configure_cluster

  cmd_provision
  cmd_wait
  warn "revoke the Tailscale auth key in the admin console now — all nodes have joined"
  cmd_swarm_init
  cmd_dokploy_install
  pause "Create the Dokploy admin account: http://${PRIMARY}.<your-tailnet>.ts.net:3000"
  cmd_join
  cmd_traefik
  cmd_lb
  hdr "Manual interlude — Cloudflare (GUIDE §10-B)"
  cat <<EOF
  In the Cloudflare dashboard, ${BOLD}in this order${RST}:
    1. Zone SSL/TLS mode: ${BOLD}Full (strict)${RST}
    2. DNS records for your apps → the LB IPv4 printed above, ${BOLD}Proxied${RST} (orange cloud)
EOF
  pause "Confirm both are done (cf-allowlist would 403 real users otherwise)"
  CF_CONFIRMED=1
  cmd_cf_allowlist
  cmd_verify

  hdr "Done"
  ok "cluster is up"
  info "remaining manual step: Dokploy UI → GitLab registry credentials (GUIDE §11)"
}

### ── dispatch ──────────────────────────────────────────────────────────────
usage() {
  cat <<EOF
${BOLD}dokploy-cluster${RST} — bootstrap a ${#ALL[@]}-node Dokploy swarm on Hetzner Cloud

${BOLD}USAGE${RST}
  $0 ${GRN}up${RST}          interactive end-to-end run: checks dependencies, prompts for
              secrets, VPS types and datacenter, then executes the pipeline
  $0 ${GRN}deps${RST}        check workstation dependencies, offer to install missing ones
  $0 ${GRN}<step>${RST}      run a single pipeline step (for re-runs and repairs)

${BOLD}PIPELINE STEPS${RST}  ${DIM}(✋ = manual interlude; § = section in GUIDE.md)${RST}
  ✋ Tailscale console: tagOwners + ssh rule, tagged auth key       §2-B
  ${GRN}provision${RST}        network, firewall, servers w/ cloud-init  §2    ${DIM}TS_AUTHKEY${RST}
  ${GRN}wait${RST}             block until cloud-init done everywhere    §2-B
  ${GRN}swarm-init${RST}       swarm on $PRIMARY, MTU-1400 ingress   §7
  ${GRN}dokploy-install${RST}  postgres + dokploy services               §8
  ✋ Dokploy UI via Tailscale :3000 — create admin account          §8
  ${GRN}join${RST}             join remaining managers and workers       §9
  ${GRN}traefik${RST}          global Traefik, CF DNS-01 certs           §9-A  ${DIM}ACME_EMAIL CF_DNS_API_TOKEN${RST}
  ${GRN}lb${RST}               Hetzner LB with /ping health checks       §10
  ✋ Cloudflare: SSL Full (strict), DNS → LB IP, Proxied            §10-B
  ${GRN}cf-allowlist${RST}     restrict origins to Cloudflare ranges     §10-B
  ${GRN}verify${RST}           cluster, Traefik and LB health            §12

Safe to re-run: existing resources are skipped (guarded re-entrancy, not
convergence). Secrets enter via prompt or environment — never argv.
EOF
  exit 1
}

case "${1:-}" in
  up)               cmd_up ;;
  deps)             cmd_deps ;;
  provision)        cmd_provision ;;
  wait)             cmd_wait ;;
  swarm-init)       cmd_swarm_init ;;
  dokploy-install)  cmd_dokploy_install ;;
  join)             cmd_join ;;
  traefik)          cmd_traefik ;;
  lb)               cmd_lb ;;
  cf-allowlist)     cmd_cf_allowlist ;;
  verify)           cmd_verify ;;
  *)                usage ;;
esac