#!/usr/bin/env bash
# deploy.et server agent — installer
#
#   curl -fsSL https://deploy.et/agent.sh | sudo bash -s -- <TOKEN>
#   sudo deploy-et-agent add-deploy <name> <dir> "<command>" [user]
#   sudo deploy-et-agent uninstall
#
# Installs a small agent that reports this server's CPU, memory, disk, load,
# services, PM2 processes, Docker containers, web server, certificates and
# pending updates to deploy.et every 30 seconds, and carries out the actions
# you request in the app.
#
# Those actions are a fixed list, written out in run_action below: restart,
# stop or read the logs of something this server reports; test and reload
# the web server; update a Compose project from its own files; clean up
# Docker; renew certificates; reboot; or run a deploy command that YOU added
# on this server with add-deploy. It never runs a command sent by deploy.et.
# It only makes outbound HTTPS requests; no port is opened. Needs systemd
# and curl.

set -euo pipefail

API_URL="https://api.deploy.et"
BIN=/usr/local/bin/deploy-et-agent
CONF_DIR=/etc/deploy-et
STATE_DIR=/var/lib/deploy-et-agent
UNIT=/etc/systemd/system/deploy-et-agent.service

fail() { echo "deploy.et agent: $*" >&2; exit 1; }

[ "$(id -u)" -eq 0 ] || fail "run this with sudo."
command -v systemctl >/dev/null 2>&1 || fail "this server doesn't use systemd, which the agent needs."
command -v curl >/dev/null 2>&1 || fail "curl is required. Install it and run this again."

TOKEN="${1:-}"
[[ "$TOKEN" =~ ^dpa_[A-Za-z0-9_-]{20,}$ ]] || fail "missing or malformed token. Copy the whole command from the app."

mkdir -p "$CONF_DIR/deploys" "$STATE_DIR"
chmod 700 "$CONF_DIR" "$CONF_DIR/deploys" "$STATE_DIR"

# Root-only, and read by curl from the file, so the token never shows up in
# the process list.
( umask 077; printf 'Authorization: Bearer %s\n' "$TOKEN" > "$CONF_DIR/auth.header" )

if [ ! -f "$CONF_DIR/agent.env" ]; then
  cat > "$CONF_DIR/agent.env" <<EOF
DEPLOY_ET_API=$API_URL
# Extra systemd services to watch, space-separated, e.g. "myapp worker"
DEPLOY_ET_SERVICES=""
EOF
  chmod 600 "$CONF_DIR/agent.env"
fi

cat > "$BIN" <<'AGENT'
#!/usr/bin/env bash
# deploy.et server agent. Installed by https://deploy.et/agent.sh.
set -uo pipefail

AGENT_VERSION="1.2.0"
CONF=/etc/deploy-et/agent.env
AUTH=/etc/deploy-et/auth.header
DEPLOYS=/etc/deploy-et/deploys
STATE=/var/lib/deploy-et-agent
RESULTS="$STATE/results"
LOCK="$STATE/results.lock"

# Results sent per report, and how much output each may carry. Logs get
# more room; everything else only needs the tail.
RESULT_BATCH=5
OUT_MAX=4000
LOG_MAX=12000

# Deploys, Compose updates, cleanups and renewals run in the background so
# reports keep flowing while they work. At most this many at once.
MAX_JOBS=3
LONG_KINDS=" DEPLOY COMPOSE_UP DOCKER_PRUNE CERT_RENEW "

TARGET_RE='^[A-Za-z0-9@._:-]{1,160}$'
NAME_RE='^[A-Za-z0-9._-]{1,64}$'

# Filled once per report, read by the PM2 actions that follow it.
PM2_JSON=null

# Watched when installed. Add your own with DEPLOY_ET_SERVICES in $CONF.
KNOWN_UNITS=(nginx.service apache2.service httpd.service caddy.service
  docker.service containerd.service postgresql.service mysql.service
  mariadb.service redis-server.service redis.service mongod.service
  ssh.service sshd.service cron.service crond.service
  'php*-fpm.service' 'pm2-*.service')

log() { echo "$*" >&2; }

json_str() {
  local s=$1
  s=${s//\\/\\\\}
  s=${s//\"/\\\"}
  s=${s//$'\n'/\\n}
  s=${s//$'\t'/\\t}
  s=$(printf '%s' "$s" | tr -d '\000-\037')
  printf '"%s"' "$s"
}

ok_of() { [ "$1" -eq 0 ] && echo true || echo false; }

# ---------- system ----------

cpu_pct() {
  local c u1 n1 s1 i1 w1 q1 sq1 st1 u2 n2 s2 i2 w2 q2 sq2 st2
  read -r c u1 n1 s1 i1 w1 q1 sq1 st1 _ < /proc/stat
  sleep 1
  read -r c u2 n2 s2 i2 w2 q2 sq2 st2 _ < /proc/stat
  local idle=$(( (i2 + w2) - (i1 + w1) ))
  local total=$(( (u2+n2+s2+i2+w2+q2+sq2+st2) - (u1+n1+s1+i1+w1+q1+sq1+st1) ))
  [ "$total" -gt 0 ] || { echo 0; return; }
  awk -v i="$idle" -v t="$total" 'BEGIN { printf "%.1f", (1 - i / t) * 100 }'
}

mem_mb() {
  awk '/^MemTotal:/ {t=$2} /^MemAvailable:/ {a=$2}
       END { printf "%d %d", (t - a) / 1024, t / 1024 }' /proc/meminfo
}

disk_gb() {
  df -P -k / | awk 'NR == 2 { printf "%.2f %.2f", $3 / 1048576, $2 / 1048576 }'
}

os_name() { ( . /etc/os-release 2>/dev/null; echo "${PRETTY_NAME:-Linux}" ); }

# Every 6 hours is plenty; apt-check takes a few seconds.
updates_json() {
  local cache="$STATE/updates" pending=null security=null reboot=false r
  local re='^([0-9]+);([0-9]+)$'
  if [ ! -f "$cache" ] || [ $(( $(date +%s) - $(stat -c %Y "$cache") )) -gt 21600 ]; then
    if [ -x /usr/lib/update-notifier/apt-check ]; then
      r=$(/usr/lib/update-notifier/apt-check 2>&1 >/dev/null)
      [[ $r =~ $re ]] && pending=${BASH_REMATCH[1]} security=${BASH_REMATCH[2]}
    elif command -v apt-get >/dev/null 2>&1; then
      r=$(apt-get -s -o Debug::NoLocking=1 upgrade 2>/dev/null | grep '^Inst ')
      pending=$(printf '%s' "$r" | grep -c .)
      security=$(printf '%s' "$r" | grep -ci security)
    fi
    printf '%s %s\n' "$pending" "$security" > "$cache"
  fi
  read -r pending security < "$cache"
  [ -f /var/run/reboot-required ] && reboot=true
  printf '{"pending":%s,"security":%s,"rebootRequired":%s}' \
    "${pending:-null}" "${security:-null}" "$reboot"
}

# ---------- systemd ----------

watched_units() {
  local extra=() unit
  for unit in ${DEPLOY_ET_SERVICES:-}; do
    [[ $unit == *.* ]] && extra+=("$unit") || extra+=("$unit.service")
  done
  systemctl list-unit-files --type=service --no-legend --no-pager \
    "${KNOWN_UNITS[@]}" "${extra[@]}" 2>/dev/null \
    | awk '$2 != "masked" && $1 !~ /@/ { print $1 }' | head -n 50
}

is_watched() { watched_units | grep -qxF -- "$1"; }

services_json() {
  local first=1 unit active sub
  printf '['
  while read -r unit; do
    [[ $unit =~ ^[A-Za-z0-9@._-]{1,128}$ ]] || continue
    active=$(systemctl show -p ActiveState --value -- "$unit" 2>/dev/null)
    sub=$(systemctl show -p SubState --value -- "$unit" 2>/dev/null)
    [ "$first" -eq 1 ] || printf ','
    first=0
    printf '{"name":%s,"active":%s,"sub":%s}' "$(json_str "$unit")" \
      "$(json_str "${active:-unknown}")" "$(json_str "${sub:-unknown}")"
  done < <(watched_units)
  printf ']'
}

# ---------- PM2 ----------

# PM2 runs one daemon per user. Each is found by its process title, and
# spoken to as its own user with the node binary it runs on, so nvm
# installs work without anything on root's PATH.
pm2_daemons() { # -> "user uid gid pm2_home node pm2" per daemon
  local pid args home uid gid user node bin
  while read -r pid args; do
    home=$(printf '%s' "$args" | sed -n 's/.*God Daemon (\(.*\))$/\1/p')
    [ -n "$home" ] && [ -d "$home" ] || continue
    read -r uid gid < <(stat -c '%u %g' "/proc/$pid" 2>/dev/null) || continue
    user=$(id -nu "$uid" 2>/dev/null) || continue
    [[ $user =~ $NAME_RE ]] || continue
    node=$(readlink "/proc/$pid/exe" 2>/dev/null) || continue
    bin="$(dirname "$node")/pm2"
    [ -x "$bin" ] || bin=$(command -v pm2 2>/dev/null) || continue
    printf '%s %s %s %s %s %s\n' "$user" "$uid" "$gid" "$home" "$node" "$bin"
  done < <(ps -eo pid=,args= 2>/dev/null | grep 'God Daemon (' | grep -v grep)
}

pm2_line_for() { pm2_daemons | awk -v u="$1" '$1 == u { print; exit }'; }

pm2_exec() { # user uid gid pm2_home node pm2 -- then pm2 arguments
  local uid=$2 gid=$3 home=$4 node=$5 bin=$6 userhome
  shift 6
  userhome=$(getent passwd "$uid" | cut -d: -f6)
  timeout 60 setpriv --reuid="$uid" --regid="$gid" --init-groups \
    env -i HOME="${userhome:-/}" PM2_HOME="$home" \
    PATH="$(dirname "$node"):/usr/local/bin:/usr/bin:/bin" "$bin" "$@"
}

# jlist, boiled down to what the app shows. Cluster instances of one app
# are folded into a single row.
PM2_SUMMARY='let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{let a=[];try{a=JSON.parse(s.slice(s.indexOf("[")))}catch(e){}const u=process.env.U,g=new Map();for(const p of Array.isArray(a)?a:[]){const e=p.pm2_env||{},m=p.monit||{},n=String(p.name||"").slice(0,64),st=String(e.status||"unknown");let c=g.get(n);if(!c){c={id:u+":"+n,user:u,name:n,status:st,cpu:0,memMb:0,restarts:0,uptimeSec:st==="online"&&e.pm_uptime?Math.max(0,Math.round((Date.now()-e.pm_uptime)/1000)):0,mode:e.exec_mode==="cluster_mode"?"cluster":"fork",instances:0};g.set(n,c)}c.instances++;c.cpu+=Number(m.cpu)||0;c.memMb+=Math.round((Number(m.memory)||0)/1048576);c.restarts+=Number(e.restart_time)||0;if(st!=="online")c.status=st}process.stdout.write(JSON.stringify([...g.values()].slice(0,50)))})'

pm2_json() {
  command -v setpriv >/dev/null 2>&1 || { printf 'null'; return; }
  local line out all="" found=0
  local -a d
  while read -r line; do
    read -r -a d <<< "$line"
    found=1
    out=$(pm2_exec "${d[@]}" jlist 2>/dev/null | U="${d[0]}" "${d[4]}" -e "$PM2_SUMMARY" 2>/dev/null)
    [[ $out == \[*\] ]] || continue
    out=${out#\[}
    out=${out%\]}
    [ -n "$out" ] && all+="${all:+,}$out"
  done < <(pm2_daemons)
  if [ "$found" -eq 1 ]; then printf '[%s]' "$all"; else printf 'null'; fi
}

pm2_has() { printf '%s' "$PM2_JSON" | grep -qF "\"id\":$(json_str "$1")"; }

# ---------- Docker ----------

DOCKER_FMT='{{.Names}}\t{{.Image}}\t{{.State}}\t{{.Status}}\t{{.Label "com.docker.compose.project"}}'

docker_json() {
  command -v docker >/dev/null 2>&1 || { printf 'null'; return; }
  local out first=1 name image state status project
  out=$(timeout 10 docker ps -a --format "$DOCKER_FMT" 2>/dev/null) || { printf 'null'; return; }
  printf '['
  while IFS=$'\t' read -r name image state status project; do
    [[ $name =~ $TARGET_RE ]] || continue
    [ "$first" -eq 1 ] || printf ','
    first=0
    printf '{"name":%s,"image":%s,"state":%s,"status":%s,"compose":%s}' \
      "$(json_str "$name")" "$(json_str "${image:0:300}")" \
      "$(json_str "$state")" "$(json_str "${status:0:120}")" \
      "$([[ $project =~ $TARGET_RE ]] && json_str "$project" || printf 'null')"
  done < <(printf '%s\n' "$out" | head -n 150)
  printf ']'
}

container_exists() {
  docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qxF -- "$1"
}

# A Compose project's directory and files come from Docker's own labels on
# its containers, so this only ever runs the owner's compose file.
compose_cmd() { # project -> two lines: working dir, then the command
  local info dir files f args compose
  info=$(docker ps -a --filter "label=com.docker.compose.project=$1" \
    --format '{{.Label "com.docker.compose.project.working_dir"}}|{{.Label "com.docker.compose.project.config_files"}}' \
    2>/dev/null | head -n 1)
  dir=${info%%|*}
  files=${info#*|}
  [ -n "$dir" ] && [ -d "$dir" ] || return 1
  if docker compose version >/dev/null 2>&1; then
    compose="docker compose"
  elif command -v docker-compose >/dev/null 2>&1; then
    compose="docker-compose"
  else
    return 1
  fi
  args="-p $(printf '%q' "$1") --project-directory $(printf '%q' "$dir")"
  local IFS=','
  for f in $files; do
    [ -f "$f" ] && args+=" -f $(printf '%q' "$f")"
  done
  printf '%s\n%s\n' "$dir" \
    "$compose $args pull --ignore-pull-failures && $compose $args up -d --build"
}

# ---------- web servers and certificates ----------

web_servers() {
  command -v nginx >/dev/null 2>&1 && echo nginx
  { command -v apache2ctl || command -v apachectl; } >/dev/null 2>&1 && echo apache
  command -v caddy >/dev/null 2>&1 && echo caddy
  return 0
}

web_json() {
  local first=1 w
  printf '['
  while read -r w; do
    [ -n "$w" ] || continue
    [ "$first" -eq 1 ] || printf ','
    first=0
    printf '"%s"' "$w"
  done < <(web_servers)
  printf ']'
}

web_unit() {
  case $1 in
    nginx) echo nginx.service ;;
    caddy) echo caddy.service ;;
    apache)
      if systemctl cat apache2.service >/dev/null 2>&1; then echo apache2.service
      else echo httpd.service; fi ;;
  esac
}

web_test() {
  case $1 in
    nginx) nginx -t 2>&1 ;;
    apache)
      if command -v apache2ctl >/dev/null 2>&1; then apache2ctl configtest 2>&1
      else apachectl configtest 2>&1; fi ;;
    caddy) caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile 2>&1 ;;
  esac
}

web_logs() {
  case $1 in
    nginx) tail -n 200 /var/log/nginx/error.log 2>&1 ;;
    apache)
      if [ -f /var/log/apache2/error.log ]; then tail -n 200 /var/log/apache2/error.log 2>&1
      else tail -n 200 /var/log/httpd/error_log 2>&1; fi ;;
    caddy) journalctl -u caddy -n 200 --no-pager -o short-iso 2>&1 ;;
  esac
}

certs_json() {
  local first=1 dir name end
  printf '['
  if command -v openssl >/dev/null 2>&1; then
    for dir in /etc/letsencrypt/live/*/; do
      [ -f "${dir}cert.pem" ] || continue
      name=$(basename "$dir")
      [[ $name =~ $TARGET_RE ]] || continue
      end=$(openssl x509 -enddate -noout -in "${dir}cert.pem" 2>/dev/null) || continue
      end=$(date -d "${end#notAfter=}" +%s 2>/dev/null) || continue
      [ "$first" -eq 1 ] || printf ','
      first=0
      printf '{"name":%s,"expiresAt":%s}' "$(json_str "$name")" "$end"
    done
  fi
  printf ']'
}

# ---------- deploy commands ----------

# One file per deploy in $DEPLOYS, written only by add-deploy (root). The
# API can name a deploy; it can never supply or change its command.
deploy_get() { sed -n "s/^$2=//p" "$DEPLOYS/$1.conf" 2>/dev/null | head -n 1; }

deploys_json() {
  local first=1 f name
  printf '['
  for f in "$DEPLOYS"/*.conf; do
    [ -f "$f" ] || continue
    name=$(basename "$f" .conf)
    [[ $name =~ $NAME_RE ]] || continue
    [ "$first" -eq 1 ] || printf ','
    first=0
    printf '{"name":%s,"dir":%s,"user":%s}' "$(json_str "$name")" \
      "$(json_str "$(deploy_get "$name" DIR | cut -c1-300)")" \
      "$(json_str "$(deploy_get "$name" USER)")"
  done
  printf ']'
}

# Runs outside this agent's sandbox as a transient systemd unit, so a deploy
# behaves as it would from the owner's own shell: their user, their
# directory, write access to their home. Survives an agent restart.
run_detached() { # user dir max_seconds command
  systemd-run --quiet --wait --pipe --collect \
    -p User="$1" -p WorkingDirectory="$2" -p RuntimeMaxSec="$3" \
    /bin/bash -lc "$4" < /dev/null 2>&1
}

runtime_json() {
  printf '{"pm2":%s,"docker":%s,"web":%s,"certs":%s,"certbot":%s,"deploys":%s,"updates":%s}' \
    "$PM2_JSON" "$(docker_json)" "$(web_json)" "$(certs_json)" \
    "$(command -v certbot >/dev/null 2>&1 && echo true || echo false)" \
    "$(deploys_json)" "$(updates_json)"
}

# ---------- actions ----------

record_result() { # id ok output [max]
  local out max=${4:-$OUT_MAX} line
  out=$(printf '%s' "$3" | sed 's/\x1b\[[0-9;?]*[A-Za-z]//g')
  [ "${#out}" -gt "$max" ] && out=${out: -$max}
  line=$(printf '{"id":%s,"ok":%s,"output":%s}' "$(json_str "$1")" "$2" "$(json_str "$out")")
  # Background jobs append here too.
  { flock 9; printf '%s\n' "$line" >> "$RESULTS"; } 9>>"$LOCK"
}

fail_result() { record_result "$1" false "$2"; }

# The complete list of things this agent will ever do on request. Every
# target is checked against what this server itself reports.
run_action() { # id kind target
  local id=$1 kind=$2 target=$3 out rc dir cmd user unit line name
  local -a d

  if [ -n "$target" ] && [[ ! $target =~ $TARGET_RE ]]; then
    fail_result "$id" "Not a valid target."
    return
  fi

  case $kind in
    RESTART_SERVICE|SERVICE_LOGS)
      if ! is_watched "$target"; then
        fail_result "$id" "Not a service this agent watches."
        return
      fi
      if [ "$kind" = SERVICE_LOGS ]; then
        record_result "$id" true \
          "$(journalctl -u "$target" -n 200 --no-pager -o short-iso 2>&1)" "$LOG_MAX"
      elif out=$(timeout 90 systemctl restart -- "$target" 2>&1); then
        record_result "$id" true "$(systemctl is-active -- "$target" 2>&1)"
      else
        record_result "$id" false "$out
$(journalctl -u "$target" -n 20 --no-pager 2>&1)"
      fi
      ;;

    REBOOT)
      # Recorded first; it's delivered on the first report after boot.
      record_result "$id" true "Rebooting."
      sync
      systemctl reboot
      ;;

    PM2_RESTART|PM2_STOP|PM2_LOGS)
      if ! pm2_has "$target"; then
        fail_result "$id" "Not a PM2 process on this server."
        return
      fi
      line=$(pm2_line_for "${target%%:*}")
      if [ -z "$line" ]; then
        fail_result "$id" "PM2 isn't running for that user any more."
        return
      fi
      read -r -a d <<< "$line"
      name=${target#*:}
      case $kind in
        PM2_LOGS)
          record_result "$id" true \
            "$(pm2_exec "${d[@]}" logs "$name" --lines 200 --nostream --raw 2>&1)" "$LOG_MAX"
          return ;;
        PM2_RESTART) out=$(pm2_exec "${d[@]}" restart "$name" 2>&1); rc=$? ;;
        PM2_STOP) out=$(pm2_exec "${d[@]}" stop "$name" 2>&1); rc=$? ;;
      esac
      record_result "$id" "$(ok_of "$rc")" "$out"
      ;;

    DOCKER_RESTART|DOCKER_STOP|DOCKER_LOGS)
      if ! container_exists "$target"; then
        fail_result "$id" "Not a container on this server."
        return
      fi
      case $kind in
        DOCKER_LOGS)
          record_result "$id" true \
            "$(timeout 30 docker logs --tail 200 --timestamps "$target" 2>&1)" "$LOG_MAX"
          return ;;
        DOCKER_RESTART) out=$(timeout 120 docker restart "$target" 2>&1); rc=$? ;;
        DOCKER_STOP) out=$(timeout 120 docker stop "$target" 2>&1); rc=$? ;;
      esac
      if [ "$rc" -eq 0 ]; then
        record_result "$id" true "$(docker inspect -f '{{.State.Status}}' "$target" 2>&1)"
      else
        fail_result "$id" "$out"
      fi
      ;;

    COMPOSE_UP)
      dir="" cmd=""
      { read -r dir; read -r cmd; } < <(compose_cmd "$target")
      if [ -z "$cmd" ]; then
        fail_result "$id" "Couldn't find that Compose project's files on this server."
        return
      fi
      out=$(run_detached root "$dir" 1800 "$cmd"); rc=$?
      record_result "$id" "$(ok_of "$rc")" "$out
[exit $rc]" "$LOG_MAX"
      ;;

    DOCKER_PRUNE)
      if ! command -v docker >/dev/null 2>&1; then
        fail_result "$id" "Docker isn't installed."
        return
      fi
      out=$(run_detached root / 900 'docker image prune -f && docker builder prune -f'); rc=$?
      record_result "$id" "$(ok_of "$rc")" "$out" "$LOG_MAX"
      ;;

    WEB_TEST|WEB_RELOAD|WEB_LOGS)
      if ! web_servers | grep -qxF -- "$target"; then
        fail_result "$id" "That web server isn't installed here."
        return
      fi
      case $kind in
        WEB_LOGS) record_result "$id" true "$(web_logs "$target")" "$LOG_MAX" ;;
        WEB_TEST) out=$(web_test "$target"); rc=$?; record_result "$id" "$(ok_of "$rc")" "$out" ;;
        WEB_RELOAD)
          # Caddy's own reload validates first and keeps the running config
          # on failure; nginx and Apache get an explicit test.
          out=""
          if [ "$target" != caddy ] && ! out=$(web_test "$target"); then
            fail_result "$id" "The config test failed, so nothing was reloaded.
$out"
            return
          fi
          unit=$(web_unit "$target")
          if cmd=$(timeout 60 systemctl reload -- "$unit" 2>&1); then
            record_result "$id" true "$out
Reloaded $unit."
          else
            fail_result "$id" "$cmd"
          fi ;;
      esac
      ;;

    CERT_RENEW)
      if ! command -v certbot >/dev/null 2>&1; then
        fail_result "$id" "certbot isn't installed."
        return
      fi
      out=$(run_detached root / 900 'certbot renew --no-random-sleep-on-renew'); rc=$?
      record_result "$id" "$(ok_of "$rc")" "$out" "$LOG_MAX"
      ;;

    DEPLOY)
      if [[ ! $target =~ $NAME_RE ]] || [ ! -f "$DEPLOYS/$target.conf" ]; then
        fail_result "$id" "There's no deploy called that on this server."
        return
      fi
      dir=$(deploy_get "$target" DIR)
      user=$(deploy_get "$target" USER)
      cmd=$(deploy_get "$target" RUN)
      if [ ! -d "$dir" ] || [ -z "$cmd" ] || ! id "${user:-root}" >/dev/null 2>&1; then
        fail_result "$id" "That deploy's settings are incomplete. Add it again with add-deploy."
        return
      fi
      out=$(run_detached "${user:-root}" "$dir" 1800 "$cmd"); rc=$?
      record_result "$id" "$(ok_of "$rc")" "$out
[exit $rc]" "$LOG_MAX"
      ;;

    *)
      fail_result "$id" "Unknown action. This agent may be older than the app."
      ;;
  esac
}

dispatch() { # id kind target
  if [[ $LONG_KINDS == *" $2 "* ]]; then
    local -a running
    mapfile -t running < <(jobs -rp)
    if [ "${#running[@]}" -ge "$MAX_JOBS" ]; then
      fail_result "$1" "$MAX_JOBS long jobs are already running on this server. Try again when one finishes."
      return
    fi
    run_action "$@" &
  else
    run_action "$@"
  fi
}

# Reads the response JSON on stdin; prints "id kind target" per action.
parse_actions() {
  grep -o '"actions":\[.*\]' | grep -o '{[^{}]*}' | while read -r obj; do
    local id kind target
    id=$(printf '%s' "$obj" | sed -n 's/.*"id":"\([0-9a-f-]\{36\}\)".*/\1/p')
    kind=$(printf '%s' "$obj" | sed -n 's/.*"kind":"\([A-Z0-9_]*\)".*/\1/p')
    target=$(printf '%s' "$obj" | sed -n 's/.*"target":"\([A-Za-z0-9@._:-]*\)".*/\1/p')
    [ -n "$id" ] && [ -n "$kind" ] && printf '%s %s %s\n' "$id" "$kind" "${target:--}"
  done
}

report_once() {
  local cpu mem_used mem_total disk_used disk_total results batch sent=0 body resp code
  cpu=$(cpu_pct)
  read -r mem_used mem_total <<< "$(mem_mb)"
  read -r disk_used disk_total <<< "$(disk_gb)"
  PM2_JSON=$(pm2_json)

  results="[]"
  if [ -s "$RESULTS" ]; then
    batch=$({ flock 9; head -n "$RESULT_BATCH" "$RESULTS"; } 9>>"$LOCK")
    sent=$(printf '%s\n' "$batch" | grep -c .)
    [ "$sent" -gt 0 ] && results="[$(printf '%s\n' "$batch" | paste -sd, -)]"
  fi

  body=$(printf '{"agentVersion":%s,"hostname":%s,"os":%s,"kernel":%s,"uptimeSec":%s,"cpuPct":%s,"memUsedMb":%s,"memTotalMb":%s,"diskUsedGb":%s,"diskTotalGb":%s,"load1":%s,"services":%s,"runtime":%s,"results":%s}' \
    "$(json_str "$AGENT_VERSION")" "$(json_str "$(hostname)")" \
    "$(json_str "$(os_name)")" "$(json_str "$(uname -r)")" \
    "$(cut -d. -f1 /proc/uptime)" "$cpu" "$mem_used" "$mem_total" \
    "$disk_used" "$disk_total" "$(cut -d' ' -f1 /proc/loadavg)" \
    "$(services_json)" "$(runtime_json)" "$results")

  resp=$(printf '%s' "$body" | curl -sS --max-time 20 -X POST \
    -H @"$AUTH" -H "Content-Type: application/json" \
    --data-binary @- -w '\n%{http_code}' "$DEPLOY_ET_API/agent/report" 2>&1) \
    || { log "report failed: $resp"; return 1; }

  code=${resp##*$'\n'}
  resp=${resp%$'\n'*}

  case $code in
    200) ;;
    401)
      log "deploy.et no longer recognises this server (removed in the app, or a new install command was issued). Stopping."
      systemctl disable deploy-et-agent >/dev/null 2>&1
      exit 0
      ;;
    *)
      log "report returned HTTP $code: ${resp:0:200}"
      return 1
      ;;
  esac

  # Delivered. Drop exactly the lines we sent: background jobs may have
  # added more since.
  if [ "$sent" -gt 0 ]; then
    { flock 9; sed -i "1,${sent}d" "$RESULTS"; } 9>>"$LOCK"
  fi

  INTERVAL=$(printf '%s' "$resp" | sed -n 's/.*"intervalSec":\([0-9]*\).*/\1/p')

  local id kind target
  while read -r id kind target; do
    [ -n "$id" ] || continue
    [ "$target" = "-" ] && target=""
    dispatch "$id" "$kind" "$target"
  done < <(printf '%s' "$resp" | parse_actions)
}

# Between reports, holds one request open to deploy.et. It returns the
# moment an action is queued for this server, so the action starts within
# a second or two instead of waiting out the report interval. Also returns
# early when a background job finishes, so its result goes out at once.
# Still outbound only: nothing on this server listens.
WAIT_MAX=25

wait_for_work() { # seconds
  local max=$1 pid code i lines interrupted=0
  [ "$max" -gt "$WAIT_MAX" ] && max=$WAIT_MAX
  lines=$(wc -l < "$RESULTS" 2>/dev/null || echo 0)

  curl -s -o /dev/null -w '%{http_code}' --max-time $((max + 10)) \
    -H @"$AUTH" "$DEPLOY_ET_API/agent/wait" > "$STATE/wait.code" 2>/dev/null &
  pid=$!

  for ((i = 0; i < max + 10; i++)); do
    kill -0 "$pid" 2>/dev/null || break
    if [ "$(wc -l < "$RESULTS" 2>/dev/null || echo 0)" -gt "$lines" ]; then
      kill "$pid" 2>/dev/null
      interrupted=1
      break
    fi
    sleep 1
  done
  wait "$pid" 2>/dev/null
  [ "$interrupted" -eq 1 ] && return

  code=$(cat "$STATE/wait.code" 2>/dev/null)
  case $code in
    # Woken, or timed out: report now. 401 is handled by the report.
    200|401) ;;
    # An API without /agent/wait, rate-limited, or unreachable: the old
    # rhythm, so this never turns into a tight loop.
    *) sleep "$max" ;;
  esac
}

main_run() {
  . "$CONF"
  [ -r "$AUTH" ] || { log "missing $AUTH — reinstall from the app."; exit 1; }
  DEPLOY_ET_API=${DEPLOY_ET_API:-https://api.deploy.et}
  mkdir -p "$STATE"
  touch "$LOCK" "$RESULTS"

  while true; do
    INTERVAL=30
    if report_once && [ -s "$RESULTS" ]; then
      # Something just finished: deliver it now, not on the next report.
      sleep 1
      continue
    fi
    [[ $INTERVAL =~ ^[0-9]+$ ]] || INTERVAL=30
    [ "$INTERVAL" -lt 15 ] && INTERVAL=15
    [ "$INTERVAL" -gt 300 ] && INTERVAL=300
    wait_for_work "$INTERVAL"
  done
}

# ---------- command line ----------

need_root() { [ "$(id -u)" -eq 0 ] || { echo "run with sudo" >&2; exit 1; }; }

add_deploy() { # name dir command [user]
  need_root
  local name=${1:-} dir=${2:-} cmd=${3:-} user=${4:-root}
  if [ -z "$name" ] || [ -z "$dir" ] || [ -z "$cmd" ]; then
    echo 'usage: sudo deploy-et-agent add-deploy <name> <dir> "<command>" [user]' >&2
    exit 2
  fi
  [[ $name =~ $NAME_RE ]] || { echo "name: up to 64 letters, numbers, dots, dashes or underscores." >&2; exit 2; }
  [[ $dir == /* ]] && [ -d "$dir" ] || { echo "dir must be an existing absolute path." >&2; exit 2; }
  [[ $cmd != *$'\n'* && $dir != *$'\n'* ]] || { echo "the command must be one line." >&2; exit 2; }
  id "$user" >/dev/null 2>&1 || { echo "no such user: $user" >&2; exit 2; }

  mkdir -p "$DEPLOYS"
  chmod 700 "$DEPLOYS"
  ( umask 077; printf 'DIR=%s\nUSER=%s\nRUN=%s\n' "$dir" "$user" "$cmd" > "$DEPLOYS/$name.conf" )

  echo "Added \"$name\". It appears under Deploy in the app within a minute."
  echo "It runs as $user in $dir, in a login shell (bash -lc)."
  echo "Using node from nvm? Start the command with: source ~/.nvm/nvm.sh &&"
}

remove_deploy() {
  need_root
  [[ ${1:-} =~ $NAME_RE ]] || { echo "usage: sudo deploy-et-agent remove-deploy <name>" >&2; exit 2; }
  rm -f "$DEPLOYS/$1.conf"
  echo "Removed \"$1\"."
}

list_deploys() {
  need_root
  local f found=0
  for f in "$DEPLOYS"/*.conf; do
    [ -f "$f" ] || continue
    found=1
    printf '%s\n  user: %s\n  dir:  %s\n  run:  %s\n' "$(basename "$f" .conf)" \
      "$(sed -n 's/^USER=//p' "$f")" "$(sed -n 's/^DIR=//p' "$f")" "$(sed -n 's/^RUN=//p' "$f")"
  done
  [ "$found" -eq 1 ] || echo "No deploys yet. Add one with: sudo deploy-et-agent add-deploy <name> <dir> \"<command>\" [user]"
}

uninstall() {
  need_root
  systemctl disable --now deploy-et-agent >/dev/null 2>&1 || true
  rm -f /etc/systemd/system/deploy-et-agent.service
  rm -rf /etc/deploy-et "$STATE"
  systemctl daemon-reload
  rm -f /usr/local/bin/deploy-et-agent
  echo "deploy.et agent removed. Remove the server in the app too."
}

case "${1:-}" in
  run) main_run ;;
  add-deploy) shift; add_deploy "$@" ;;
  remove-deploy) shift; remove_deploy "$@" ;;
  deploys) list_deploys ;;
  uninstall) uninstall ;;
  *)
    echo "usage: deploy-et-agent run | add-deploy <name> <dir> \"<command>\" [user] | remove-deploy <name> | deploys | uninstall" >&2
    exit 2 ;;
esac
AGENT
chmod 755 "$BIN"

cat > "$UNIT" <<EOF
[Unit]
Description=deploy.et server agent
After=network-online.target
Wants=network-online.target

[Service]
ExecStart=$BIN run
# Exits 0 when deploy.et no longer knows this server, and stays stopped.
Restart=on-failure
RestartSec=15
ProtectSystem=full
# Read-only rather than hidden: PM2 keeps its socket and node under the
# user's home. Deploys run outside this sandbox, through systemd-run.
ProtectHome=read-only
PrivateTmp=yes
NoNewPrivileges=yes

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable deploy-et-agent >/dev/null 2>&1
# restart, not start: a reinstall with a new token must pick it up.
systemctl restart deploy-et-agent

echo "deploy.et agent installed. This server appears in your app within a minute."
echo "Add a deploy command with: sudo deploy-et-agent add-deploy <name> <dir> \"<command>\" [user]"
echo "Remove the agent any time with: sudo deploy-et-agent uninstall"