#!/bin/bash
# =============================================================================
# runtime-verify.sh -- System Invariant Checker
# =============================================================================
# Category: runtime | Bird: cormorant | Tier: 2 | Node-aware: yes
#
# Purpose: Pre-flight verification for Phoenix decision-making. Checks all
# system invariants before engines, agents, or orchestrators take action.
#
# Actions:
#   --status        JSON pill (pass/fail/total)
#   --all           Run all invariant checks
#   --db            Database connectivity
#   --agent         s3rv1c3agent heartbeat
#   --registry      Tool registry parity
#   --vault         Vault engine availability
#   --ghost         Ghost engine availability
#   --phoenix       Phoenix orchestrator availability
#   --wireguard     WireGuard mesh status
#   --disk          Disk space thresholds
#   --services      Critical service check
#   --send          Same as --all
#   --digest        Full report
#   --flush         Truncate log
# =============================================================================

TOOL_NAME="runtime-verify"
TOOL_VERSION="3.0.0"
TOOL_TIER=2
TOOL_BIRD="cormorant"
TOOL_LOG="/opt/binary/logs/tools/${TOOL_NAME}.log"

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
if [[ -f "${SCRIPT_DIR}/common.sh" ]]; then source "${SCRIPT_DIR}/common.sh"
elif [[ -f "${SCRIPT_DIR}/../common.sh" ]]; then source "${SCRIPT_DIR}/../common.sh"
else echo "FATAL: common.sh not found" >&2; exit 1; fi
export BIRD_ID="${TOOL_BIRD}"

FLEET_ENV="/etc/binary/fleet.env"
[ -f "$FLEET_ENV" ] && source "$FLEET_ENV" 2>/dev/null

ALPHA_IP="${FLEET_ALPHA_IP:-10.69.69.1}"
BRAVO_IP="${FLEET_BRAVO_IP:-10.69.69.3}"
CHARLIE_IP="${FLEET_CHARLIE_IP:-10.69.69.4}"
DELTA_IP="${FLEET_DELTA_IP:-10.69.69.5}"
ECHO_IP="${FLEET_ECHO_IP:-10.69.69.6}"

PASS=0 FAIL=0 WARN=0
_pass() { echo "  [OK]   $1"; PASS=$((PASS+1)); }
_fail() { echo "  [FAIL] $1"; FAIL=$((FAIL+1)); }
_warn() { echo "  [WARN] $1"; WARN=$((WARN+1)); }

# --db (database connectivity)
_do_db_impl() {
    echo "--- Database ---"
    # Alpha MariaDB (s3rv1c3_admin)
    if timeout 3 bash -c "echo > /dev/tcp/${ALPHA_IP}/3306" 2>/dev/null; then
        _pass "Alpha MariaDB (${ALPHA_IP}:3306)"
    else _fail "Alpha MariaDB (${ALPHA_IP}:3306) unreachable"; fi
    # Bravo MariaDB (phoenix_birds)
    if timeout 3 bash -c "echo > /dev/tcp/${BRAVO_IP}/3306" 2>/dev/null; then
        _pass "Bravo MariaDB (${BRAVO_IP}:3306)"
    else _warn "Bravo MariaDB (${BRAVO_IP}:3306) unreachable"; fi
    # Delta Vault DB
    if timeout 3 bash -c "echo > /dev/tcp/${DELTA_IP}/3306" 2>/dev/null; then
        _pass "Delta Vault DB (${DELTA_IP}:3306)"
    else _warn "Delta Vault DB (${DELTA_IP}:3306) unreachable"; fi
}

# --agent (s3rv1c3agent heartbeat)
_do_agent_impl() {
    echo "--- s3rv1c3agent ---"
    if pgrep -f s3rv1c3agent &>/dev/null || ss -tlnp 2>/dev/null | grep -q ':8366 '; then
        _pass "s3rv1c3agent running on ${NODE_ID}"
    else _fail "s3rv1c3agent not running on ${NODE_ID}"; fi
    # Check all fleet agents
    for ip in "$ALPHA_IP" "$BRAVO_IP" "$CHARLIE_IP"; do
        if timeout 3 bash -c "echo > /dev/tcp/${ip}/8366" 2>/dev/null; then
            _pass "Agent at ${ip}:8366"
        else _warn "Agent at ${ip}:8366 unreachable"; fi
    done
}

# --registry (tool registry parity)
_do_registry_impl() {
    echo "--- Tool Registry ---"
    local registered=0 missing=0
    # Check that every registered tool has a script file
    if [ -f "${SCRIPT_DIR}/s3rv1c3agent.sh" ]; then
        local tools
        tools=$(bash "${SCRIPT_DIR}/s3rv1c3agent.sh" list 2>/dev/null | grep -E '^\S' | awk '{print $1}' | grep -v '^TOOL$' | grep -v '^-')
        for tool in $tools; do
            if [ -f "${SCRIPT_DIR}/${tool}.sh" ]; then
                registered=$((registered+1))
            else
                _fail "Registered but missing: ${tool}.sh"
                missing=$((missing+1))
            fi
        done
    fi
    [ "$missing" -eq 0 ] && _pass "Registry parity: ${registered} tools OK" || _fail "Registry: ${missing} missing"
}

# --vault (Vault engine availability)
_do_vault_impl() {
    echo "--- Vault Tool ---"
    if timeout 3 bash -c "echo > /dev/tcp/${DELTA_IP}/8370" 2>/dev/null; then
        _pass "Vault handler (${DELTA_IP}:8370)"
    else _fail "Vault handler (${DELTA_IP}:8370) unreachable"; fi
    # Check vault API health
    local health
    health=$(curl -sf --max-time 3 "http://${DELTA_IP}:8370/health" 2>/dev/null)
    if [ -n "$health" ]; then _pass "Vault API healthy"
    else _warn "Vault API health check failed"; fi
}

# --ghost (Ghost engine availability)
_do_ghost_impl() {
    echo "--- Ghost Tool ---"
    if timeout 3 bash -c "echo > /dev/tcp/${ALPHA_IP}/8369" 2>/dev/null; then
        _pass "Ghost engine (${ALPHA_IP}:8369)"
    else _fail "Ghost engine (${ALPHA_IP}:8369) unreachable"; fi
}

# --phoenix (Phoenix orchestrator availability)
_do_phoenix_impl() {
    echo "--- Phoenix Orchestrator ---"
    if timeout 3 bash -c "echo > /dev/tcp/${CHARLIE_IP}/8367" 2>/dev/null; then
        _pass "Phoenix orchestrator (${CHARLIE_IP}:8367)"
    else _fail "Phoenix orchestrator (${CHARLIE_IP}:8367) unreachable"; fi
    local health
    health=$(curl -sf --max-time 3 "http://${CHARLIE_IP}:8367/health" 2>/dev/null)
    if [ -n "$health" ]; then _pass "Phoenix API healthy"
    else _warn "Phoenix API health check failed"; fi
}

# --wireguard (WireGuard mesh)
_do_wireguard_impl() {
    echo "--- WireGuard ---"
    if command -v wg &>/dev/null; then
        local iface; iface=$(wg show interfaces 2>/dev/null)
        if [ -n "$iface" ]; then
            _pass "WireGuard interface: ${iface}"
            local peers; peers=$(wg show "$iface" peers 2>/dev/null | wc -l)
            _pass "Connected peers: ${peers}"
            # Ping fleet nodes
            for ip in "$ALPHA_IP" "$BRAVO_IP" "$CHARLIE_IP" "$DELTA_IP" "$ECHO_IP"; do
                if ping -c1 -W2 "$ip" &>/dev/null; then
                    _pass "Ping ${ip}"
                else _warn "Ping ${ip} timeout"; fi
            done
        else _fail "WireGuard: no active interface"; fi
    else _fail "WireGuard: wg command not found"; fi
}

# --disk (disk space)
_do_disk_impl() {
    echo "--- Disk ---"
    local disk_pct; disk_pct=$(df / 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
    if [ "${disk_pct:-0}" -lt 90 ] 2>/dev/null; then
        _pass "Root: ${disk_pct}%"
    elif [ "${disk_pct:-0}" -lt 95 ] 2>/dev/null; then
        _warn "Root: ${disk_pct}% (high)"
    else _fail "Root: ${disk_pct}% (critical)"; fi
    # Check /opt/binary specifically
    local opt_pct; opt_pct=$(df /opt/binary 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
    if [ "${opt_pct:-0}" -lt 90 ] 2>/dev/null; then
        _pass "/opt/binary: ${opt_pct:-?}%"
    else _warn "/opt/binary: ${opt_pct}%"; fi
}

# --services (critical services)
_do_services_impl() {
    echo "--- Critical Services ---"
    local svcs=(sshd nginx php-fpm mariadb fail2ban)
    for svc in "${svcs[@]}"; do
        if systemctl is-active "$svc" &>/dev/null; then
            _pass "${svc}: active"
        elif ! systemctl cat "${svc}.service" &>/dev/null; then
            : # Not installed on this node, skip silently
        else
            _fail "${svc}: not active"
        fi
    done
}

# --all / --send
_do_all_impl() {
    PASS=0; FAIL=0; WARN=0
    echo "=== System Invariant Check: ${NODE_ID} ==="
    echo "  Exec: ${EXEC_ID:-none}"
    echo ""
    _do_db_impl; echo ""
    _do_agent_impl; echo ""
    _do_registry_impl; echo ""
    _do_vault_impl; echo ""
    _do_ghost_impl; echo ""
    _do_phoenix_impl; echo ""
    _do_wireguard_impl; echo ""
    _do_disk_impl; echo ""
    _do_services_impl; echo ""
    echo "=== Summary ==="
    echo "  Passed: ${PASS}  Warned: ${WARN}  Failed: ${FAIL}"
    local verdict="PASS"; [ "$FAIL" -gt 0 ] && verdict="FAIL"
    echo "  Verdict: ${verdict}"
    tool_audit "verify_all" "pass=${PASS} warn=${WARN} fail=${FAIL} verdict=${verdict} node=${NODE_ID}"
    return $FAIL
}

_do_status_impl() {
    PASS=0; FAIL=0; WARN=0
    _do_db_impl &>/dev/null; _do_agent_impl &>/dev/null
    _do_disk_impl &>/dev/null; _do_services_impl &>/dev/null
    local s="healthy"
    [ "$FAIL" -gt 0 ] && s="degraded"
    [ "$FAIL" -gt 3 ] && s="down"
    echo "{\"tool\":\"${TOOL_NAME}\",\"node\":\"${NODE_ID}\",\"status\":\"${s}\",\"pass\":${PASS},\"warn\":${WARN},\"fail\":${FAIL},\"exec_id\":\"${EXEC_ID:-}\",\"checked_at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}"
}

_do_digest_impl() { _do_all_impl; }
_do_send_impl() { _do_all_impl; }
_do_flush_impl() { [ -f "$TOOL_LOG" ] && truncate -s 0 "$TOOL_LOG"; echo "Flush complete"; }

tool_help() {
    echo "${TOOL_NAME} v${TOOL_VERSION} -- System Invariant Checker"
    echo ""
    echo "Phoenix calls this BEFORE making decisions."
    echo ""
    echo "Checks:"
    echo "  --all / --send    Run all invariant checks"
    echo "  --db              Database connectivity"
    echo "  --agent           s3rv1c3agent heartbeat"
    echo "  --registry        Tool registry parity"
    echo "  --vault           Vault engine availability"
    echo "  --ghost           Ghost engine availability"
    echo "  --phoenix         Phoenix orchestrator availability"
    echo "  --wireguard       WireGuard mesh status"
    echo "  --disk            Disk space thresholds"
    echo "  --services        Critical service check"
    echo ""
    echo "Standard: --status|--digest|--flush"
}

do_send()   { tool_wrap "send"   _do_send_impl "$@"; }
do_status() { tool_wrap "status" _do_status_impl "$@"; }
do_digest() { tool_wrap "digest" _do_digest_impl "$@"; }
do_flush()  { tool_wrap "flush"  _do_flush_impl "$@"; }

tool_init; trap 'tool_complete' EXIT
case "${1:---help}" in
    --send|--all) shift; do_send "$@" ;; --status) shift; do_status "$@" ;;
    --digest) shift; do_digest "$@" ;; --flush) shift; do_flush "$@" ;;
    --db) tool_wrap "db" _do_db_impl ;;
    --agent) tool_wrap "agent" _do_agent_impl ;;
    --registry) tool_wrap "registry" _do_registry_impl ;;
    --vault) tool_wrap "vault" _do_vault_impl ;;
    --ghost) tool_wrap "ghost" _do_ghost_impl ;;
    --phoenix) tool_wrap "phoenix" _do_phoenix_impl ;;
    --wireguard) tool_wrap "wireguard" _do_wireguard_impl ;;
    --disk) tool_wrap "disk" _do_disk_impl ;;
    --services) tool_wrap "services" _do_services_impl ;;
    --help) tool_help ;; *) handle_standard_command "$@" ;;
esac
