#!/bin/bash
###############################################################################
# engine_bringup.sh — Master Orchestration: Engine Node 669427 Bringup
#
# Brings Engine fully online after a rebuild. Executes sequentially:
#   1.  Load environment (669427.env)
#   2.  Verify WireGuard tunnels (7 required + 1 optional)
#   3.  Apply firewall (firewall_engine.sh)
#   4.  Verify MariaDB + engine database
#   5.  Run 06-scls.sql migration if needed
#   6.  Start postfix-qmgr-aux (disguised daemon)
#   7.  Build + install 365engine (C++ daemon)
#   8.  Start s3rv1c3agent
#   9.  Verify keyserv relay (8302 on wg7)
#   10. Initial telemetry ping
#   11. Full validation
#   12. JSON status report
#
# Rollback on failure. Logs to /var/log/engine-bringup-<timestamp>.log
###############################################################################
set -uo pipefail

# ─── Constants ──────────────────────────────────────────────────────────────
readonly NODE_ID="669427"
readonly NODE_ROLE="ENGINE"
readonly DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)"
readonly ENV_FILE="${DEPLOY_DIR}/669427.env"
readonly BUILD_ROOT="/mnt/agents/output/build/v2"
readonly ENGINE_PATH="/opt/apache/engine"
readonly BRINGUP_TS="$(date +%Y%m%d-%H%M%S)"
readonly LOG_FILE="/var/log/engine-bringup-${BRINGUP_TS}.log"
readonly STATUS_FILE="/tmp/engine-bringup-status.json"
readonly SCRIPT_NAME="$(basename "$0")"

STEP=0
STEP_TOTAL=12
FAILED=0

declare -a ROLLBACK_ACTIONS

# ─── Logging ────────────────────────────────────────────────────────────────
log() {
    local msg="[$(date -Iseconds)] [BRINGUP] [${STEP}/${STEP_TOTAL}] $1"
    echo "$msg"
    echo "$msg" >> "$LOG_FILE" 2>/dev/null || true
}

warn() {
    local msg="[$(date -Iseconds)] [BRINGUP] [WARN] $1"
    echo "$msg" >&2
    echo "$msg" >> "$LOG_FILE" 2>/dev/null || true
}

die() {
    local msg="[$(date -Iseconds)] [BRINGUP] [FATAL] $1"
    echo "$msg" >&2
    echo "$msg" >> "$LOG_FILE" 2>/dev/null || true
    do_rollback
    write_json "error" "$1"
    exit 1
}

step() {
    STEP=$((STEP + 1))
    log ""
    log "═══════════════════════════════════════════════════════"
    log " STEP ${STEP}/${STEP_TOTAL}: $1"
    log "═══════════════════════════════════════════════════════"
}

# ─── Rollback ───────────────────────────────────────────────────────────────
record_rollback() {
    ROLLBACK_ACTIONS+=("$1")
}

do_rollback() {
    if [[ ${#ROLLBACK_ACTIONS[@]} -eq 0 ]]; then
        return
    fi
    warn "ROLLBACK: Executing ${#ROLLBACK_ACTIONS[@]} rollback actions..."
    for ((i=${#ROLLBACK_ACTIONS[@]}-1; i>=0; i--)); do
        warn "  Rollback: ${ROLLBACK_ACTIONS[$i]}"
        eval "${ROLLBACK_ACTIONS[$i]}" &>/dev/null || true
    done
    warn "Rollback complete. Log: ${LOG_FILE}"
}

# ─── JSON output ────────────────────────────────────────────────────────────
write_json() {
    local status="$1"
    local message="$2"
    local detail="${3:-{}}"

    cat > "${STATUS_FILE}" <<JSONEOF
{
  "status": "${status}",
  "node_id": "${NODE_ID}",
  "node_role": "${NODE_ROLE}",
  "step": ${STEP},
  "step_total": ${STEP_TOTAL},
  "timestamp": "$(date -Iseconds)",
  "log_file": "${LOG_FILE}",
  "message": "${message}",
  "detail": ${detail}
}
JSONEOF
    cat "${STATUS_FILE}"
}

# ─── Declare result collectors ──────────────────────────────────────────────
declare -A WG_STATUS
declare -A SVC_STATUS
declare -A PORT_STATUS
VALIDATION_PASS=0
VALIDATION_FAIL=0
VALIDATION_WARN=0

# ─── Pre-flight ─────────────────────────────────────────────────────────────
if [[ "$(id -u)" -ne 0 ]]; then
    echo "[FATAL] Must run as root" >&2
    exit 1
fi

mkdir -p "$(dirname "$LOG_FILE")"
touch "$LOG_FILE"
chmod 600 "$LOG_FILE"

log ""
log "═══════════════════════════════════════════════════════"
log "  ENGINE NODE ${NODE_ID} — BRINGUP ORCHESTRATION"
log "  Started: $(date -Iseconds)"
log "  Log: ${LOG_FILE}"
log "  Deploy dir: ${DEPLOY_DIR}"
log "═══════════════════════════════════════════════════════"

###############################################################################
# STEP 1: Load Environment
###############################################################################
step "Load Environment"

if [[ -f "${ENV_FILE}" ]]; then
    set -a
    # shellcheck source=/dev/null
    source "${ENV_FILE}"
    set +a
    log "Loaded: ${ENV_FILE}"
else
    warn "ENV file not found at ${ENV_FILE} — using defaults"
    NODE_ID="669427"
    ENGINE_PATH="/opt/apache/engine"
    ENGINE_USER="mail-svc"
    LISTEN_PORT=8301
    WG_INTERFACE="wg7"
    WG_IP="10.69.69.6"
    SHIELD_IP="10.69.69.1"
    SHIELD_HB_PORT=8302
    DB_HOST="127.0.0.1"
    DB_NAME="engine"
fi

# Export vars for sub-scripts
export NODE_ID NODE_ROLE ENGINE_PATH
export WG_IP="${WG_IP:-10.69.69.6}"
export SHIELD_IP="${SHIELD_IP:-10.69.69.1}"
export SHIELD_HB_PORT="${SHIELD_HB_PORT:-8302}"
export DB_NAME="${DB_NAME:-engine}"
export LISTEN_PORT="${LISTEN_PORT:-8301}"

log "NODE_ID=${NODE_ID}"
log "ENGINE_PATH=${ENGINE_PATH}"
log "WG_IP=${WG_IP}"
log "LISTEN_PORT=${LISTEN_PORT}"

###############################################################################
# STEP 2: Verify WireGuard Tunnels
###############################################################################
step "Verify WireGuard Tunnels"

# 7 required interfaces + 1 optional (forge)
declare -a WG_REQUIRED=(wg7 compass gravel lantern rain thread umbrella)
declare -a WG_OPTIONAL=(forge)

WG_ALL_UP=true

for iface in "${WG_REQUIRED[@]}"; do
    if ip link show "${iface}" &>/dev/null 2>&1; then
        # Check if UP
        state=$(ip -br link show "${iface}" 2>/dev/null | awk '{print $2}')
        if [[ "${state}" == *"UP"* ]] || [[ "${state}" == "UNKNOWN" ]]; then
            # WG interfaces show UNKNOWN when up — that's normal
            WG_STATUS["${iface}"]="UP"
            log "[OK] ${iface}: UP (state=${state})"

            # Check for recent handshake
            last_hs=$(wg show "${iface}" latest-handshakes 2>/dev/null | awk '{print $2}' | head -1)
            if [[ -n "${last_hs}" ]] && [[ "${last_hs}" != "0" ]]; then
                age=$(($(date +%s) - last_hs))
                if [[ ${age} -lt 300 ]]; then
                    log "  Handshake: ${age}s ago (healthy)"
                else
                    log "  Handshake: ${age}s ago (stale, may be idle)"
                fi
            else
                log "  Handshake: none yet"
            fi
        else
            WG_STATUS["${iface}"]="DOWN"
            warn "[FAIL] ${iface}: interface exists but state=${state}"
            WG_ALL_UP=false
        fi
    else
        WG_STATUS["${iface}"]="MISSING"
        warn "[FAIL] ${iface}: interface does not exist"
        WG_ALL_UP=false
    fi
done

# Report optional (forge)
for iface in "${WG_OPTIONAL[@]}"; do
    if ip link show "${iface}" &>/dev/null 2>&1; then
        state=$(ip -br link show "${iface}" 2>/dev/null | awk '{print $2}')
        WG_STATUS["${iface}"]="UP"
        log "[INFO] ${iface} (optional): UP (state=${state})"
    else
        WG_STATUS["${iface}"]="NOT_PRESENT"
        log "[INFO] ${iface} (optional): not present — skipping"
    fi
done

if [[ "${WG_ALL_UP}" != "true" ]]; then
    warn "Not all required WG interfaces are UP — continuing with degraded state"
    warn "Missing/down interfaces may cause connectivity issues for some relays"
    # Don't die here — let the operator decide. The validation step will catch it.
    FAILED=$((FAILED + 1))
fi

log "WG tunnel summary: ${#WG_REQUIRED[@]} required, $(echo "${!WG_STATUS[@]}" | tr ' ' '\n' | while read -r k; do echo "${WG_STATUS[$k]}"; done | grep -c "^UP$" || echo 0) UP"

###############################################################################
# STEP 3: Apply Firewall
###############################################################################
step "Apply Firewall Configuration"

FIREWALL_SCRIPT="${DEPLOY_DIR}/firewall_engine.sh"
if [[ ! -f "${FIREWALL_SCRIPT}" ]]; then
    die "Firewall script not found: ${FIREWALL_SCRIPT}"
fi

if [[ ! -x "${FIREWALL_SCRIPT}" ]]; then
    chmod +x "${FIREWALL_SCRIPT}"
fi

log "Executing: ${FIREWALL_SCRIPT}"
if bash "${FIREWALL_SCRIPT}" 2>&1 | tee -a "$LOG_FILE"; then
    log "Firewall configuration applied successfully"
    record_rollback "firewall-cmd --set-default-zone=public"
else
    die "Firewall configuration failed — see log for details"
fi

# Verify default zone
if [[ "$(firewall-cmd --get-default-zone 2>/dev/null)" != "drop" ]]; then
    die "CRITICAL: Default zone is NOT 'drop' after firewall script — aborting"
fi
log "Verified: default zone = drop"

###############################################################################
# STEP 4: Verify MariaDB + engine database
###############################################################################
step "Verify MariaDB"

if ! systemctl is-active --quiet mariadb; then
    log "MariaDB not running — starting..."
    systemctl enable mariadb 2>/dev/null || true
    systemctl start mariadb
    sleep 3
    if ! systemctl is-active --quiet mariadb; then
        die "Failed to start MariaDB"
    fi
    record_rollback "systemctl stop mariadb"
fi
log "MariaDB: running (PID=$(systemctl show --property=MainPID --value mariadb 2>/dev/null || echo '?'))"

# Verify root connectivity
if ! mysql -uroot -e "SELECT 1" &>/dev/null; then
    die "Cannot connect to MariaDB as root"
fi
log "MariaDB: root connection OK"

# Verify engine database exists
if mysql -uroot -e "USE ${DB_NAME}" &>/dev/null; then
    TABLE_COUNT=$(mysql -uroot -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='${DB_NAME}'" 2>/dev/null || echo 0)
    log "Database '${DB_NAME}': exists (${TABLE_COUNT} tables)"
else
    warn "Database '${DB_NAME}' does not exist — attempting to create via setup.sh"
    if [[ -f "${BUILD_ROOT}/database/setup.sh" ]]; then
        bash "${BUILD_ROOT}/database/setup.sh" 2>&1 | tee -a "$LOG_FILE" || die "Database setup failed"
        log "Database created via setup.sh"
    else
        die "Database '${DB_NAME}' missing and setup.sh not found at ${BUILD_ROOT}/database/setup.sh"
    fi
fi

###############################################################################
# STEP 5: Run 06-scls.sql migration
###############################################################################
step "Database Migration: 06-scls.sql"

SCLS_MIGRATION="${DEPLOY_DIR}/06-scls.sql"
# Also check in database directory
if [[ ! -f "${SCLS_MIGRATION}" ]]; then
    SCLS_MIGRATION="${BUILD_ROOT}/database/06-scls.sql"
fi
if [[ ! -f "${SCLS_MIGRATION}" ]]; then
    SCLS_MIGRATION="${DEPLOY_DIR}/../database/06-scls.sql"
fi

# Check if sec_audit_log table already exists
if mysql -uroot -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='${DB_NAME}' AND table_name='sec_audit_log'" 2>/dev/null | grep -q "^[1-9]"; then
    log "sec_audit_log table already exists — migration not needed"
else
    log "sec_audit_log table does not exist — running migration"
    if [[ -f "${SCLS_MIGRATION}" ]]; then
        log "Applying: ${SCLS_MIGRATION}"
        if mysql -uroot "${DB_NAME}" < "${SCLS_MIGRATION}" 2>&1 | tee -a "$LOG_FILE"; then
            log "Migration 06-scls.sql applied successfully"
            record_rollback "mysql -uroot -e 'DROP TABLE IF EXISTS ${DB_NAME}.sec_audit_log'"
        else
            die "Migration 06-scls.sql failed"
        fi
    else
        warn "06-scls.sql not found at expected paths — creating sec_audit_log inline"
        mysql -uroot "${DB_NAME}" <<'SQLEOF'
CREATE TABLE IF NOT EXISTS sec_audit_log (
    id              BIGINT AUTO_INCREMENT PRIMARY KEY,
    event_type      VARCHAR(64) NOT NULL,
    severity        ENUM('info','warning','critical','alert') NOT NULL DEFAULT 'info',
    source_node     VARCHAR(6) NOT NULL,
    source_ip       VARCHAR(45) NULL,
    actor           VARCHAR(128) NULL,
    target          VARCHAR(256) NULL,
    action          VARCHAR(128) NOT NULL,
    detail          JSON NULL,
    hmac            VARCHAR(128) NULL,
    session_id      VARCHAR(64) NULL,
    created_at      TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    INDEX idx_event_type (event_type),
    INDEX idx_severity (severity),
    INDEX idx_source_node (source_node),
    INDEX idx_created_at (created_at),
    INDEX idx_actor (actor),
    INDEX idx_session (session_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
SQLEOF
        log "sec_audit_log table created inline"
        record_rollback "mysql -uroot -e 'DROP TABLE IF EXISTS ${DB_NAME}.sec_audit_log'"
    fi
fi

###############################################################################
# STEP 6: Start postfix-qmgr-aux (disguised daemon)
###############################################################################
step "Start postfix-qmgr-aux Service"

PQMGR_SVC="postfix-qmgr-aux.service"

# Install service file if not present
if [[ ! -f "/etc/systemd/system/${PQMGR_SVC}" ]]; then
    PQMGR_SRC=""
    for candidate in \
        "${DEPLOY_DIR}/../tooling/systemd/${PQMGR_SVC}" \
        "/opt/apache/engine/tooling/systemd/${PQMGR_SVC}" \
        "${BUILD_ROOT}/engine/tooling/systemd/${PQMGR_SVC}"; do
        if [[ -f "${candidate}" ]]; then
            PQMGR_SRC="${candidate}"
            break
        fi
    done

    if [[ -n "${PQMGR_SRC}" ]]; then
        cp -f "${PQMGR_SRC}" "/etc/systemd/system/${PQMGR_SVC}"
        chmod 644 "/etc/systemd/system/${PQMGR_SVC}"
        systemctl daemon-reload
        log "Installed ${PQMGR_SVC} from ${PQMGR_SRC}"
    else
        warn "${PQMGR_SVC} unit file not found — attempting to start if already registered"
    fi
fi

if systemctl is-active --quiet "${PQMGR_SVC}" 2>/dev/null; then
    log "${PQMGR_SVC}: already running"
else
    systemctl enable "${PQMGR_SVC}" 2>/dev/null || true
    if systemctl start "${PQMGR_SVC}" 2>/dev/null; then
        log "${PQMGR_SVC}: started"
        record_rollback "systemctl stop ${PQMGR_SVC}"
    else
        warn "${PQMGR_SVC}: failed to start — may need decrypt-run.sh or vault key"
        warn "  Check: journalctl -u ${PQMGR_SVC} --no-pager -n 20"
        FAILED=$((FAILED + 1))
    fi
fi

SVC_STATUS["postfix-qmgr-aux"]="$(systemctl is-active "${PQMGR_SVC}" 2>/dev/null || echo 'inactive')"
log "${PQMGR_SVC}: ${SVC_STATUS["postfix-qmgr-aux"]}"

###############################################################################
# STEP 7: Build and Install 365engine
###############################################################################
step "Build 365engine (C++ Daemon)"

ENGINE_SRC="${BUILD_ROOT}/engine/src"
ENGINE_BUILD="${ENGINE_SRC}/build"

if [[ ! -d "${ENGINE_SRC}" ]]; then
    die "Engine source directory not found: ${ENGINE_SRC}"
fi

if [[ ! -f "${ENGINE_SRC}/CMakeLists.txt" ]]; then
    die "CMakeLists.txt not found in ${ENGINE_SRC}"
fi

# Preserve existing binary hash for comparison
OLD_HASH="none"
if [[ -f "${ENGINE_PATH}/365engine" ]]; then
    OLD_HASH=$(md5sum "${ENGINE_PATH}/365engine" 2>/dev/null | awk '{print $1}')
    # Backup current binary
    cp -f "${ENGINE_PATH}/365engine" "${ENGINE_PATH}/365engine.bak.${BRINGUP_TS}" 2>/dev/null || true
    record_rollback "cp -f '${ENGINE_PATH}/365engine.bak.${BRINGUP_TS}' '${ENGINE_PATH}/365engine' 2>/dev/null || true"
fi

log "Building from: ${ENGINE_SRC}"
log "Build dir: ${ENGINE_BUILD}"

# Stop engine if running, before replacing binary
if systemctl is-active --quiet 365engine 2>/dev/null; then
    log "Stopping 365engine before rebuild..."
    systemctl stop 365engine
    record_rollback "systemctl start 365engine"
fi

# Clean build
rm -rf "${ENGINE_BUILD}"
mkdir -p "${ENGINE_BUILD}"
cd "${ENGINE_BUILD}"

log "Running cmake..."
if ! cmake -DCMAKE_BUILD_TYPE=Release .. 2>&1 | tee -a "$LOG_FILE"; then
    die "CMake configuration failed"
fi

NPROC=$(nproc 2>/dev/null || echo 2)
log "Running make -j${NPROC}..."
if ! make -j"${NPROC}" 2>&1 | tee -a "$LOG_FILE"; then
    die "Engine build failed — make returned non-zero"
fi

# Find the built binary
ENGINE_BIN=""
for candidate in \
    "${ENGINE_BUILD}/bin/365engine" \
    "${ENGINE_BUILD}/365engine" \
    "${ENGINE_BUILD}/src/365engine"; do
    if [[ -f "${candidate}" ]]; then
        ENGINE_BIN="${candidate}"
        break
    fi
done

if [[ -z "${ENGINE_BIN}" ]]; then
    die "Build succeeded but 365engine binary not found in expected locations"
fi

# Strip and install
strip -s "${ENGINE_BIN}"
cp -f "${ENGINE_BIN}" "${ENGINE_PATH}/365engine"
chmod 755 "${ENGINE_PATH}/365engine"
chown root:root "${ENGINE_PATH}/365engine"

NEW_HASH=$(md5sum "${ENGINE_PATH}/365engine" | awk '{print $1}')
log "365engine installed: ${OLD_HASH} -> ${NEW_HASH}"
log "Binary size: $(stat -c '%s' "${ENGINE_PATH}/365engine" 2>/dev/null || echo '?') bytes"

# Start 365engine
systemctl enable 365engine 2>/dev/null || true
if systemctl start 365engine 2>/dev/null; then
    log "365engine started"
    sleep 2
    if systemctl is-active --quiet 365engine; then
        log "365engine: confirmed running"
    else
        warn "365engine: started but may have exited — check journalctl"
        FAILED=$((FAILED + 1))
    fi
else
    warn "365engine: failed to start — may need vault key or config"
    warn "  Check: journalctl -u 365engine --no-pager -n 30"
    FAILED=$((FAILED + 1))
fi

SVC_STATUS["365engine"]="$(systemctl is-active 365engine 2>/dev/null || echo 'inactive')"

###############################################################################
# STEP 8: Start s3rv1c3agent
###############################################################################
step "Start s3rv1c3agent"

# Ensure service file is installed
AGENT_SVC_SRC="${ENGINE_SRC}/s3rv1c3agent.service"
if [[ -f "${AGENT_SVC_SRC}" ]] && [[ ! -f "/etc/systemd/system/s3rv1c3agent.service" ]]; then
    cp -f "${AGENT_SVC_SRC}" /etc/systemd/system/s3rv1c3agent.service
    chmod 644 /etc/systemd/system/s3rv1c3agent.service
    systemctl daemon-reload
    log "Installed s3rv1c3agent.service"
fi

if systemctl is-active --quiet s3rv1c3agent 2>/dev/null; then
    log "s3rv1c3agent: already running"
else
    systemctl enable s3rv1c3agent 2>/dev/null || true
    if systemctl start s3rv1c3agent 2>/dev/null; then
        log "s3rv1c3agent: started"
        record_rollback "systemctl stop s3rv1c3agent"
    else
        warn "s3rv1c3agent: failed to start"
        warn "  Check: journalctl -u s3rv1c3agent --no-pager -n 20"
        FAILED=$((FAILED + 1))
    fi
fi

SVC_STATUS["s3rv1c3agent"]="$(systemctl is-active s3rv1c3agent 2>/dev/null || echo 'inactive')"
log "s3rv1c3agent: ${SVC_STATUS["s3rv1c3agent"]}"

###############################################################################
# STEP 9: Verify Keyserv Relay (8302 on wg7)
###############################################################################
step "Verify Keyserv Relay"

KEYSERV_OK=false

if ip link show wg7 &>/dev/null 2>&1; then
    # Check if 8302 is listening locally
    if ss -tlnp 2>/dev/null | grep -q ":8302\b"; then
        log "Port 8302: listening locally"
    else
        log "Port 8302: not listening locally (relay may be outbound-only)"
    fi

    # Try to reach Shield's keyserv relay via WG
    if command -v curl &>/dev/null; then
        log "Testing keyserv relay: ${SHIELD_IP}:${SHIELD_HB_PORT}..."
        KEYSERV_RESP=$(curl -s --max-time 10 \
            --connect-timeout 5 \
            "http://${SHIELD_IP}:${SHIELD_HB_PORT}/health" 2>/dev/null || echo "")

        if [[ -n "${KEYSERV_RESP}" ]]; then
            log "Keyserv relay responded: ${KEYSERV_RESP:0:80}"
            KEYSERV_OK=true
        else
            warn "Keyserv relay at ${SHIELD_IP}:${SHIELD_HB_PORT}: no response"
            warn "  This may be normal if Shield isn't fully up yet"
        fi
    elif command -v nc &>/dev/null; then
        if nc -z -w5 "${SHIELD_IP}" "${SHIELD_HB_PORT}" 2>/dev/null; then
            log "Keyserv relay: port ${SHIELD_HB_PORT} reachable on ${SHIELD_IP}"
            KEYSERV_OK=true
        else
            warn "Keyserv relay: port ${SHIELD_HB_PORT} not reachable on ${SHIELD_IP}"
        fi
    else
        warn "Neither curl nor nc available — skipping keyserv connectivity test"
    fi
else
    warn "wg7 interface not up — cannot test keyserv relay"
fi

log "Keyserv relay status: $(${KEYSERV_OK} && echo 'REACHABLE' || echo 'UNREACHABLE')"

###############################################################################
# STEP 10: Initial Telemetry Ping
###############################################################################
step "Initial Telemetry Ping"

TELEMETRY_OK=false

if command -v curl &>/dev/null; then
    # Send a bringup telemetry event to the engine's own health endpoint
    TELEM_PAYLOAD=$(cat <<TELEMEOF
{
  "event": "bringup",
  "node_id": "${NODE_ID}",
  "timestamp": "$(date -Iseconds)",
  "wg_tunnels_up": $(echo "${!WG_STATUS[@]}" | tr ' ' '\n' | while read -r k; do echo "${WG_STATUS[$k]}"; done | grep -c "^UP$" || echo 0),
  "wg_tunnels_total": $((${#WG_REQUIRED[@]} + ${#WG_OPTIONAL[@]})),
  "engine_hash": "${NEW_HASH:-unknown}",
  "uptime": "$(uptime -p 2>/dev/null || echo 'unknown')"
}
TELEMEOF
)

    TELEM_RESP=$(curl -s --max-time 10 \
        -X POST \
        -H "Content-Type: application/json" \
        -H "X-Node: ${NODE_ID}" \
        -d "${TELEM_PAYLOAD}" \
        "http://127.0.0.1:${LISTEN_PORT}/telemetry" 2>/dev/null || echo "")

    if [[ -n "${TELEM_RESP}" ]]; then
        log "Telemetry ping sent — response: ${TELEM_RESP:0:120}"
        TELEMETRY_OK=true
    else
        warn "Telemetry ping: no response from 127.0.0.1:${LISTEN_PORT}/telemetry"
        warn "  Engine may still be initializing — this is non-fatal"
    fi
else
    warn "curl not available — skipping telemetry ping"
fi

###############################################################################
# STEP 11: Full Validation
###############################################################################
step "Full Validation"

log ""
log "--- Port Checks ---"

declare -a REQUIRED_PORTS=(
    "8301|tcp|365engine API"
    "3306|tcp|MariaDB"
    "9922|tcp|SSH (WG only)"
)

declare -a OPTIONAL_PORTS=(
    "8302|tcp|Keyserv relay"
    "18082|tcp|Monero RPC"
    "9090|tcp|Tron signer"
    "11434|tcp|Ollama"
)

for pdef in "${REQUIRED_PORTS[@]}"; do
    IFS='|' read -r port proto desc <<< "${pdef}"
    if ss -tlnp 2>/dev/null | grep -q ":${port}\b"; then
        log "[PASS] Port ${port}/${proto} (${desc}): listening"
        PORT_STATUS["${port}"]="listening"
        VALIDATION_PASS=$((VALIDATION_PASS + 1))
    else
        warn "[FAIL] Port ${port}/${proto} (${desc}): NOT listening"
        PORT_STATUS["${port}"]="not_listening"
        VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
    fi
done

for pdef in "${OPTIONAL_PORTS[@]}"; do
    IFS='|' read -r port proto desc <<< "${pdef}"
    if ss -tlnp 2>/dev/null | grep -q ":${port}\b"; then
        log "[PASS] Port ${port}/${proto} (${desc}): listening"
        PORT_STATUS["${port}"]="listening"
        VALIDATION_PASS=$((VALIDATION_PASS + 1))
    else
        log "[WARN] Port ${port}/${proto} (${desc}): not listening (optional)"
        PORT_STATUS["${port}"]="not_listening"
        VALIDATION_WARN=$((VALIDATION_WARN + 1))
    fi
done

log ""
log "--- Service Checks ---"

declare -a REQUIRED_SVCS=(365engine s3rv1c3agent mariadb)
declare -a OPTIONAL_SVCS=(postfix-qmgr-aux monero-wallet-rpc nginx)

for svc in "${REQUIRED_SVCS[@]}"; do
    if systemctl is-active --quiet "${svc}" 2>/dev/null; then
        log "[PASS] Service ${svc}: active"
        SVC_STATUS["${svc}"]="active"
        VALIDATION_PASS=$((VALIDATION_PASS + 1))
    else
        warn "[FAIL] Service ${svc}: $(systemctl is-active "${svc}" 2>/dev/null || echo 'unknown')"
        SVC_STATUS["${svc}"]="$(systemctl is-active "${svc}" 2>/dev/null || echo 'inactive')"
        VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
    fi
done

for svc in "${OPTIONAL_SVCS[@]}"; do
    if systemctl is-active --quiet "${svc}" 2>/dev/null; then
        log "[PASS] Service ${svc}: active"
        SVC_STATUS["${svc}"]="active"
        VALIDATION_PASS=$((VALIDATION_PASS + 1))
    else
        log "[WARN] Service ${svc}: $(systemctl is-active "${svc}" 2>/dev/null || echo 'not present')"
        SVC_STATUS["${svc}"]="$(systemctl is-active "${svc}" 2>/dev/null || echo 'inactive')"
        VALIDATION_WARN=$((VALIDATION_WARN + 1))
    fi
done

log ""
log "--- WireGuard Handshake Checks ---"

for iface in "${WG_REQUIRED[@]}"; do
    if ip link show "${iface}" &>/dev/null 2>&1; then
        last_hs=$(wg show "${iface}" latest-handshakes 2>/dev/null | awk '{print $2}' | head -1)
        if [[ -n "${last_hs}" ]] && [[ "${last_hs}" != "0" ]]; then
            age=$(($(date +%s) - last_hs))
            if [[ ${age} -lt 300 ]]; then
                log "[PASS] ${iface}: handshake ${age}s ago"
                VALIDATION_PASS=$((VALIDATION_PASS + 1))
            else
                log "[WARN] ${iface}: handshake stale (${age}s)"
                VALIDATION_WARN=$((VALIDATION_WARN + 1))
            fi
        else
            log "[WARN] ${iface}: no handshake yet"
            VALIDATION_WARN=$((VALIDATION_WARN + 1))
        fi
    else
        warn "[FAIL] ${iface}: interface missing"
        VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
    fi
done

log ""
log "--- Firewall Verification ---"
if [[ "$(firewall-cmd --get-default-zone 2>/dev/null)" == "drop" ]]; then
    log "[PASS] Default zone: drop"
    VALIDATION_PASS=$((VALIDATION_PASS + 1))
else
    warn "[FAIL] Default zone: $(firewall-cmd --get-default-zone 2>/dev/null)"
    VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
fi

ACTIVE_ZONES=$(firewall-cmd --get-active-zones 2>/dev/null | grep -c "^[a-z]" || echo 0)
log "[INFO] Active firewall zones: ${ACTIVE_ZONES}"

log ""
log "--- Binary Checks ---"
if [[ -x "${ENGINE_PATH}/365engine" ]]; then
    log "[PASS] ${ENGINE_PATH}/365engine: executable"
    VALIDATION_PASS=$((VALIDATION_PASS + 1))
else
    warn "[FAIL] ${ENGINE_PATH}/365engine: missing or not executable"
    VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
fi

if [[ -x "${ENGINE_PATH}/s3rv1c3agent" ]]; then
    log "[PASS] ${ENGINE_PATH}/s3rv1c3agent: executable"
    VALIDATION_PASS=$((VALIDATION_PASS + 1))
else
    warn "[FAIL] ${ENGINE_PATH}/s3rv1c3agent: missing or not executable"
    VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
fi

log ""
log "--- Database Checks ---"
if mysql -uroot -e "USE ${DB_NAME}" &>/dev/null; then
    FINAL_TABLE_COUNT=$(mysql -uroot -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='${DB_NAME}'" 2>/dev/null || echo 0)
    log "[PASS] Database '${DB_NAME}': ${FINAL_TABLE_COUNT} tables"
    VALIDATION_PASS=$((VALIDATION_PASS + 1))

    if mysql -uroot -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='${DB_NAME}' AND table_name='sec_audit_log'" 2>/dev/null | grep -q "^[1-9]"; then
        log "[PASS] sec_audit_log: present"
        VALIDATION_PASS=$((VALIDATION_PASS + 1))
    else
        warn "[FAIL] sec_audit_log: missing after migration"
        VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
    fi
else
    warn "[FAIL] Database '${DB_NAME}': not accessible"
    VALIDATION_FAIL=$((VALIDATION_FAIL + 1))
fi

###############################################################################
# STEP 12: JSON Status Report
###############################################################################
step "Generate Status Report"

# Build WG status JSON
WG_JSON="["
first=true
for iface in "${WG_REQUIRED[@]}" "${WG_OPTIONAL[@]}"; do
    if [[ "${first}" == "true" ]]; then first=false; else WG_JSON+=","; fi
    status="${WG_STATUS[${iface}]:-unknown}"
    required="true"
    for opt in "${WG_OPTIONAL[@]}"; do
        if [[ "${opt}" == "${iface}" ]]; then required="false"; break; fi
    done
    WG_JSON+="{\"interface\":\"${iface}\",\"status\":\"${status}\",\"required\":${required}}"
done
WG_JSON+="]"

# Build service status JSON
SVC_JSON="{"
first=true
for svc in "${!SVC_STATUS[@]}"; do
    if [[ "${first}" == "true" ]]; then first=false; else SVC_JSON+=","; fi
    SVC_JSON+="\"${svc}\":\"${SVC_STATUS[${svc}]}\""
done
SVC_JSON+="}"

# Build port status JSON
PORT_JSON="{"
first=true
for port in "${!PORT_STATUS[@]}"; do
    if [[ "${first}" == "true" ]]; then first=false; else PORT_JSON+=","; fi
    PORT_JSON+="\"${port}\":\"${PORT_STATUS[${port}]}\""
done
PORT_JSON+="}"

# Determine overall status
OVERALL="success"
if [[ ${VALIDATION_FAIL} -gt 0 ]] || [[ ${FAILED} -gt 0 ]]; then
    if [[ ${VALIDATION_FAIL} -gt 3 ]] || [[ ${FAILED} -gt 2 ]]; then
        OVERALL="error"
    else
        OVERALL="degraded"
    fi
fi

DETAIL=$(cat <<DETAILEOF
{
    "engine_path": "${ENGINE_PATH}",
    "engine_hash": "${NEW_HASH:-unknown}",
    "wg_ip": "${WG_IP}",
    "listen_port": ${LISTEN_PORT},
    "firewall_default_zone": "$(firewall-cmd --get-default-zone 2>/dev/null || echo 'unknown')",
    "firewall_active_zones": ${ACTIVE_ZONES:-0},
    "db_name": "${DB_NAME}",
    "db_tables": ${FINAL_TABLE_COUNT:-0},
    "keyserv_reachable": $(${KEYSERV_OK} && echo 'true' || echo 'false'),
    "telemetry_ok": $(${TELEMETRY_OK} && echo 'true' || echo 'false'),
    "wg_tunnels": ${WG_JSON},
    "services": ${SVC_JSON},
    "ports": ${PORT_JSON},
    "validation": {
      "pass": ${VALIDATION_PASS},
      "fail": ${VALIDATION_FAIL},
      "warn": ${VALIDATION_WARN}
    },
    "log_file": "${LOG_FILE}",
    "bringup_duration_note": "started at ${BRINGUP_TS}"
  }
DETAILEOF
)

# Don't rollback on success
if [[ "${OVERALL}" != "error" ]]; then
    ROLLBACK_ACTIONS=()
fi

write_json "${OVERALL}" "Engine ${NODE_ID} bringup ${OVERALL}" "${DETAIL}"

log ""
log "═══════════════════════════════════════════════════════"
log "  ENGINE NODE ${NODE_ID} — BRINGUP ${OVERALL^^}"
log "═══════════════════════════════════════════════════════"
log ""
log "  Validation:  ${VALIDATION_PASS} PASS | ${VALIDATION_FAIL} FAIL | ${VALIDATION_WARN} WARN"
log "  Log:         ${LOG_FILE}"
log "  Status JSON: ${STATUS_FILE}"
log ""

if [[ "${OVERALL}" == "success" ]]; then
    log "  ✓ All systems nominal"
    log "  ✓ 365engine running"
    log "  ✓ Firewall: drop default, ${ACTIVE_ZONES:-0} active zones"
    log "  ✓ Database: ${DB_NAME} (${FINAL_TABLE_COUNT:-0} tables)"
    exit 0
elif [[ "${OVERALL}" == "degraded" ]]; then
    warn "  Engine is running in DEGRADED state"
    warn "  Review validation failures above"
    warn "  Some services or tunnels may need manual attention"
    exit 0
else
    warn "  Engine bringup FAILED with critical errors"
    warn "  Rollback has been executed"
    warn "  Review ${LOG_FILE} for details"
    exit 1
fi
