#!/usr/bin/env bash
###############################################################################
# verify_firewall_fleet.sh — Fleet-wide firewall verification
# Runs from local machine, SSHs into each of the 8 nodes via WG mesh.
# Collects firewall state and compares against expected configuration.
# Outputs colored pass/fail per node with a final summary.
###############################################################################
set -euo pipefail

readonly SCRIPT_NAME="verify_firewall_fleet"
readonly LOG_TAG="[${SCRIPT_NAME}]"
readonly TIMESTAMP_FMT="+%Y-%m-%dT%H:%M:%S%z"
readonly SSH_KEY="Y:\\keys\\new\\gabriella"
readonly SSH_PORT=9922
readonly SSH_OPTS="-i ${SSH_KEY} -p ${SSH_PORT} -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o BatchMode=yes"

# ── Colors ──────────────────────────────────────────────────────────────────
RED='\033[0;31m'
GRN='\033[0;32m'
YEL='\033[1;33m'
CYN='\033[0;36m'
BLD='\033[1m'
RST='\033[0m'

# ── Logging helpers ─────────────────────────────────────────────────────────
log()  { printf '%s %s %s\n' "$(date "${TIMESTAMP_FMT}")" "${LOG_TAG}" "$*"; }
warn() { printf '%s %s %b%s%b\n' "$(date "${TIMESTAMP_FMT}")" "${LOG_TAG}" "${YEL}" "WARN: $*" "${RST}" >&2; }
die()  { printf '%s %s %b%s%b\n' "$(date "${TIMESTAMP_FMT}")" "${LOG_TAG}" "${RED}" "FATAL: $*" "${RST}" >&2; exit 1; }

pass() { printf '  %b[PASS]%b %s\n' "${GRN}" "${RST}" "$*"; }
fail() { printf '  %b[FAIL]%b %s\n' "${RED}" "${RST}" "$*"; }
info() { printf '  %b[INFO]%b %s\n' "${CYN}" "${RST}" "$*"; }

# ── Node definitions ────────────────────────────────────────────────────────
declare -a NODE_NAMES=( "Shield" "Origin" "Proxy" "Filter" "DNS" "Engine" "Payment" "Vault" )
declare -a NODE_IDS=(   "414342" "558201" "244939" "333941" "199335" "669427" "714820" "869377" )
declare -a NODE_IPS=(   "10.69.69.1" "10.69.69.2" "10.69.69.3" "10.69.69.4" "10.69.69.5" "10.69.69.6" "10.69.69.7" "10.69.69.8" )

# ── Hello check-in commands per node (AGENTS.md compliance) ─────────────────
declare -A HELLO_CMDS
HELLO_CMDS["Shield"]='sudo /opt/apache/engine/tools/coredump-handler.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["Origin"]='sudo /opt/php-worker/tools/nginx-maint.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["Proxy"]='sudo /opt/cache-daemon/tools/networkd-diag.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["Filter"]='sudo /opt/netfilter/tools/journal-compact.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["DNS"]='sudo /opt/unbound/tools/resolved-sync.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["Engine"]='sudo /opt/apache/engine/tools/oomd-report.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["Payment"]='sudo /opt/systemd-journal/tools/journal-rotate.sh check-in "maintenance" "node" 3600'
HELLO_CMDS["Vault"]='sudo /opt/stunnel/tools/cert-renew-check.sh check-in "maintenance" "node" 3600'

# ── Goodbye commands per node ───────────────────────────────────────────────
declare -A BYE_CMDS
BYE_CMDS["Shield"]='sudo /opt/apache/engine/tools/coredump-handler.sh goodbye'
BYE_CMDS["Origin"]='sudo /opt/php-worker/tools/nginx-maint.sh goodbye'
BYE_CMDS["Proxy"]='sudo /opt/cache-daemon/tools/networkd-diag.sh goodbye'
BYE_CMDS["Filter"]='sudo /opt/netfilter/tools/journal-vacuum.sh'
BYE_CMDS["DNS"]='sudo /opt/unbound/tools/resolved-sync.sh goodbye'
BYE_CMDS["Engine"]='sudo /opt/apache/engine/tools/oomd-cleanup.sh'
BYE_CMDS["Payment"]='sudo /opt/systemd-journal/tools/journal-archive.sh'
BYE_CMDS["Vault"]='sudo /opt/stunnel/tools/cert-renew-check.sh goodbye'

# ── Expected firewall state per node ────────────────────────────────────────
# Format: "fw_type|default_zone|expected_zones|expected_public_udp|expected_mesh_tcp|extra_checks"

declare -A EXPECTED_DEFAULT_ZONE
EXPECTED_DEFAULT_ZONE["Shield"]="drop"
EXPECTED_DEFAULT_ZONE["Origin"]="drop"
EXPECTED_DEFAULT_ZONE["Proxy"]="drop"
EXPECTED_DEFAULT_ZONE["Filter"]="drop"
EXPECTED_DEFAULT_ZONE["DNS"]="ufw"        # sentinel: UFW node
EXPECTED_DEFAULT_ZONE["Engine"]="drop"
EXPECTED_DEFAULT_ZONE["Payment"]="drop"
EXPECTED_DEFAULT_ZONE["Vault"]="drop"

declare -A EXPECTED_ZONES
EXPECTED_ZONES["Shield"]="zone-mesh zone-comp zone-rain zone-umbr zone-pebl zone-tmbr wg10-zone"
EXPECTED_ZONES["Origin"]="zone-mesh zone-comp zone-fosl zone-lant zone-pebl zone-thrd zone-tmbr"
EXPECTED_ZONES["Proxy"]="zone-mesh zone-comp zone-shale"
EXPECTED_ZONES["Filter"]="zone-mesh zone-comp"
EXPECTED_ZONES["DNS"]=""
EXPECTED_ZONES["Engine"]="zone-mesh zone-comp"
EXPECTED_ZONES["Payment"]="zone-mesh zone-comp"
EXPECTED_ZONES["Vault"]="zone-mesh zone-comp"

declare -A EXPECTED_PUBLIC_UDP
EXPECTED_PUBLIC_UDP["Shield"]="41820 42783 53417 38471 51820 39847 40153"
EXPECTED_PUBLIC_UDP["Origin"]="41820 42783 43291 39471 39847 44619 40153"
EXPECTED_PUBLIC_UDP["Proxy"]="44634 42783 47417"
EXPECTED_PUBLIC_UDP["Filter"]="41820 42783"
EXPECTED_PUBLIC_UDP["DNS"]="42783 37747 43291 41652"
EXPECTED_PUBLIC_UDP["Engine"]=""
EXPECTED_PUBLIC_UDP["Payment"]="41820 42783"
EXPECTED_PUBLIC_UDP["Vault"]="41652 42783"

declare -A EXPECTED_MESH_TCP
EXPECTED_MESH_TCP["Shield"]="9922"
EXPECTED_MESH_TCP["Origin"]="9922"
EXPECTED_MESH_TCP["Proxy"]="9922"
EXPECTED_MESH_TCP["Filter"]="9922"
EXPECTED_MESH_TCP["DNS"]="9922"
EXPECTED_MESH_TCP["Engine"]="9922"
EXPECTED_MESH_TCP["Payment"]="9922"
EXPECTED_MESH_TCP["Vault"]="9922"

# Ports that should NOT exist (stale)
declare -A STALE_PORTS
STALE_PORTS["Shield"]="33022/tcp"
STALE_PORTS["Origin"]="33022/tcp"
STALE_PORTS["Proxy"]="30222/tcp 33022/tcp 39847/udp 40153/udp 49616/udp"
STALE_PORTS["Filter"]=""
STALE_PORTS["DNS"]=""
STALE_PORTS["Engine"]=""
STALE_PORTS["Payment"]=""
STALE_PORTS["Vault"]=""

# ── Counters ────────────────────────────────────────────────────────────────
TOTAL_PASS=0
TOTAL_FAIL=0
TOTAL_SKIP=0
declare -A NODE_RESULT

###############################################################################
# run_on_node — SSH to a node with hello check-in + commands + goodbye
# Usage: run_on_node <node_name> <wg_ip> <commands_string>
# Returns: stdout from the remote commands
###############################################################################
run_on_node() {
    local node_name="$1"
    local wg_ip="$2"
    local commands="$3"

    local hello="${HELLO_CMDS[${node_name}]}"
    local bye="${BYE_CMDS[${node_name}]}"

    # Combine: hello check-in → actual commands → goodbye
    local full_cmd="${hello} >/dev/null 2>&1; ${commands}; ${bye} >/dev/null 2>&1"

    # shellcheck disable=SC2086
    ssh ${SSH_OPTS} "admin@${wg_ip}" "${full_cmd}" 2>/dev/null
}

###############################################################################
# verify_firewalld_node — Verify a firewalld-based node
###############################################################################
verify_firewalld_node() {
    local idx="$1"
    local node_name="${NODE_NAMES[$idx]}"
    local node_ip="${NODE_IPS[$idx]}"
    local node_fails=0

    printf '\n%b══════════════════════════════════════════════════════════%b\n' "${BLD}" "${RST}"
    printf '%b  %s (%s) — %s%b\n' "${CYN}" "${node_name}" "${NODE_IDS[$idx]}" "${node_ip}" "${RST}"
    printf '%b══════════════════════════════════════════════════════════%b\n' "${BLD}" "${RST}"

    # Collect all state in one SSH call
    local collect_cmds
    collect_cmds=$(cat <<'REMOTE_EOF'
echo "===DEFAULT_ZONE==="
firewall-cmd --get-default-zone 2>/dev/null || echo "UNKNOWN"
echo "===ALL_ZONES==="
firewall-cmd --get-zones 2>/dev/null || echo ""
echo "===ACTIVE_ZONES==="
firewall-cmd --get-active-zones 2>/dev/null || echo ""
echo "===ZONE_MESH_PORTS==="
firewall-cmd --zone=zone-mesh --list-ports 2>/dev/null || echo ""
echo "===ZONE_MESH_RICH==="
firewall-cmd --zone=zone-mesh --list-rich-rules 2>/dev/null || echo ""
echo "===PUBLIC_PORTS==="
firewall-cmd --zone=public --list-ports 2>/dev/null || echo ""
echo "===PUBLIC_RICH==="
firewall-cmd --zone=public --list-rich-rules 2>/dev/null || echo ""
echo "===DROP_RICH==="
firewall-cmd --zone=drop --list-rich-rules 2>/dev/null || echo ""
echo "===ALL_ZONE_DETAILS==="
for z in $(firewall-cmd --get-zones 2>/dev/null); do
    echo "---ZONE:${z}---"
    firewall-cmd --zone="${z}" --list-all 2>/dev/null || true
done
echo "===LISTENING==="
ss -tlnup 2>/dev/null | grep -E 'LISTEN|udp' || true
echo "===END==="
REMOTE_EOF
    )

    local output
    if ! output=$(run_on_node "${node_name}" "${node_ip}" "${collect_cmds}" 2>&1); then
        fail "Cannot connect to ${node_name} at ${node_ip}"
        NODE_RESULT["${node_name}"]="UNREACHABLE"
        (( TOTAL_SKIP++ )) || true
        return
    fi

    # ── Parse default zone ──────────────────────────────────────────────
    local actual_default
    actual_default=$(echo "${output}" | sed -n '/===DEFAULT_ZONE===/,/===ALL_ZONES===/p' | grep -v '===' | head -1 | tr -d '[:space:]')
    local expected_default="${EXPECTED_DEFAULT_ZONE[${node_name}]}"

    if [[ "${actual_default}" == "${expected_default}" ]]; then
        pass "Default zone: ${actual_default}"
    else
        fail "Default zone: expected '${expected_default}', got '${actual_default}'"
        (( node_fails++ )) || true
    fi

    # ── Check expected zones exist ──────────────────────────────────────
    local all_zones
    all_zones=$(echo "${output}" | sed -n '/===ALL_ZONES===/,/===ACTIVE_ZONES===/p' | grep -v '===' | head -1)

    for zone in ${EXPECTED_ZONES[${node_name}]}; do
        if echo "${all_zones}" | grep -qw "${zone}"; then
            pass "Zone exists: ${zone}"
        else
            fail "Zone MISSING: ${zone}"
            (( node_fails++ )) || true
        fi
    done

    # ── Check SSH 9922 in zone-mesh ─────────────────────────────────────
    local mesh_ports
    mesh_ports=$(echo "${output}" | sed -n '/===ZONE_MESH_PORTS===/,/===ZONE_MESH_RICH===/p' | grep -v '===' | head -1)

    if echo "${mesh_ports}" | grep -q "9922/tcp"; then
        pass "SSH 9922/tcp in zone-mesh"
    else
        fail "SSH 9922/tcp NOT in zone-mesh (got: ${mesh_ports:-empty})"
        (( node_fails++ )) || true
    fi

    # ── Check expected public UDP ports ─────────────────────────────────
    local public_ports
    public_ports=$(echo "${output}" | sed -n '/===PUBLIC_PORTS===/,/===PUBLIC_RICH===/p' | grep -v '===' | head -1)

    for port in ${EXPECTED_PUBLIC_UDP[${node_name}]}; do
        if echo "${public_ports}" | grep -q "${port}/udp"; then
            pass "Public UDP: ${port}/udp"
        else
            fail "Public UDP MISSING: ${port}/udp"
            (( node_fails++ )) || true
        fi
    done

    # ── Check stale ports are absent ────────────────────────────────────
    local all_zone_details
    all_zone_details=$(echo "${output}" | sed -n '/===ALL_ZONE_DETAILS===/,/===LISTENING===/p')

    for stale in ${STALE_PORTS[${node_name}]}; do
        if echo "${all_zone_details}" | grep -q "${stale}"; then
            fail "Stale port STILL PRESENT: ${stale}"
            (( node_fails++ )) || true
        else
            pass "Stale port removed: ${stale}"
        fi
    done

    # ── Check no SSH on public/drop zone ────────────────────────────────
    if echo "${public_ports}" | grep -q "9922/tcp\|22/tcp\|33022/tcp\|30222/tcp"; then
        fail "SSH port found in PUBLIC zone — must be WG-only!"
        (( node_fails++ )) || true
    else
        pass "No SSH ports exposed on public zone"
    fi

    # ── Check listening sockets ─────────────────────────────────────────
    local listeners
    listeners=$(echo "${output}" | sed -n '/===LISTENING===/,/===END===/p' | grep -v '===')
    if echo "${listeners}" | grep -q ":9922"; then
        pass "sshd listening on port 9922"
    else
        warn "sshd NOT detected listening on 9922 (may be filtered by ss output)"
    fi

    # ── Node result ─────────────────────────────────────────────────────
    if [[ ${node_fails} -eq 0 ]]; then
        NODE_RESULT["${node_name}"]="PASS"
        (( TOTAL_PASS++ )) || true
    else
        NODE_RESULT["${node_name}"]="FAIL(${node_fails})"
        (( TOTAL_FAIL++ )) || true
    fi
}

###############################################################################
# verify_ufw_node — Verify the DNS node (Debian/UFW)
###############################################################################
verify_ufw_node() {
    local idx="$1"
    local node_name="${NODE_NAMES[$idx]}"
    local node_ip="${NODE_IPS[$idx]}"
    local node_fails=0

    printf '\n%b══════════════════════════════════════════════════════════%b\n' "${BLD}" "${RST}"
    printf '%b  %s (%s) — %s [UFW]%b\n' "${CYN}" "${node_name}" "${NODE_IDS[$idx]}" "${node_ip}" "${RST}"
    printf '%b══════════════════════════════════════════════════════════%b\n' "${BLD}" "${RST}"

    local collect_cmds
    collect_cmds=$(cat <<'REMOTE_EOF'
echo "===UFW_STATUS==="
sudo ufw status verbose 2>/dev/null || echo "UFW_ERROR"
echo "===UFW_NUMBERED==="
sudo ufw status numbered 2>/dev/null || echo "UFW_ERROR"
echo "===LISTENING==="
ss -tlnup 2>/dev/null | grep -E 'LISTEN|udp' || true
echo "===END==="
REMOTE_EOF
    )

    local output
    if ! output=$(run_on_node "${node_name}" "${node_ip}" "${collect_cmds}" 2>&1); then
        fail "Cannot connect to ${node_name} at ${node_ip}"
        NODE_RESULT["${node_name}"]="UNREACHABLE"
        (( TOTAL_SKIP++ )) || true
        return
    fi

    local ufw_status
    ufw_status=$(echo "${output}" | sed -n '/===UFW_STATUS===/,/===UFW_NUMBERED===/p' | grep -v '===')

    # ── Check UFW is active ─────────────────────────────────────────────
    if echo "${ufw_status}" | grep -qi "Status: active"; then
        pass "UFW is active"
    else
        fail "UFW is NOT active"
        (( node_fails++ )) || true
    fi

    # ── Check default incoming deny ─────────────────────────────────────
    if echo "${ufw_status}" | grep -qi "Default: deny (incoming)"; then
        pass "Default incoming: deny"
    else
        fail "Default incoming is NOT deny"
        (( node_fails++ )) || true
    fi

    # ── Check default outgoing allow ────────────────────────────────────
    if echo "${ufw_status}" | grep -qi "Default: allow (outgoing)"; then
        pass "Default outgoing: allow"
    else
        fail "Default outgoing is NOT allow"
        (( node_fails++ )) || true
    fi

    # ── Check DNS ports ─────────────────────────────────────────────────
    local ufw_numbered
    ufw_numbered=$(echo "${output}" | sed -n '/===UFW_NUMBERED===/,/===LISTENING===/p' | grep -v '===')

    for proto in tcp udp; do
        if echo "${ufw_numbered}" | grep -q "53/${proto}"; then
            pass "DNS 53/${proto} allowed"
        else
            fail "DNS 53/${proto} NOT found in rules"
            (( node_fails++ )) || true
        fi
    done

    # ── Check SSH 9922 from WG mesh only ────────────────────────────────
    if echo "${ufw_numbered}" | grep -q "9922.*10.69.69.0/24"; then
        pass "SSH 9922/tcp restricted to 10.69.69.0/24"
    else
        fail "SSH 9922/tcp rule for WG mesh not found"
        (( node_fails++ )) || true
    fi

    # ── Check WG UDP ports ──────────────────────────────────────────────
    for port in ${EXPECTED_PUBLIC_UDP["DNS"]}; do
        if echo "${ufw_numbered}" | grep -q "${port}/udp"; then
            pass "UDP ${port}/udp allowed"
        else
            fail "UDP ${port}/udp NOT found in rules"
            (( node_fails++ )) || true
        fi
    done

    # ── Check port 22 denied ────────────────────────────────────────────
    if echo "${ufw_numbered}" | grep -q "22/tcp.*DENY"; then
        pass "Legacy SSH 22/tcp denied"
    else
        warn "Legacy SSH 22/tcp deny rule not explicitly found (may be covered by default deny)"
    fi

    # ── Check listening sockets ─────────────────────────────────────────
    local listeners
    listeners=$(echo "${output}" | sed -n '/===LISTENING===/,/===END===/p' | grep -v '===')
    if echo "${listeners}" | grep -q ":9922"; then
        pass "sshd listening on port 9922"
    else
        warn "sshd NOT detected listening on 9922"
    fi

    # ── Node result ─────────────────────────────────────────────────────
    if [[ ${node_fails} -eq 0 ]]; then
        NODE_RESULT["${node_name}"]="PASS"
        (( TOTAL_PASS++ )) || true
    else
        NODE_RESULT["${node_name}"]="FAIL(${node_fails})"
        (( TOTAL_FAIL++ )) || true
    fi
}

###############################################################################
# Main
###############################################################################
log "Fleet firewall verification starting"
log "Timestamp: $(date "${TIMESTAMP_FMT}")"
printf '%b\n' "${BLD}Fleet Firewall Verification — 8 Nodes${RST}"
printf '%b\n' "═══════════════════════════════════════════════════════════════"

for idx in "${!NODE_NAMES[@]}"; do
    node_name="${NODE_NAMES[$idx]}"
    if [[ "${EXPECTED_DEFAULT_ZONE[${node_name}]}" == "ufw" ]]; then
        verify_ufw_node "${idx}"
    else
        verify_firewalld_node "${idx}"
    fi
done

###############################################################################
# Final Summary
###############################################################################
printf '\n%b══════════════════════════════════════════════════════════════%b\n' "${BLD}" "${RST}"
printf '%b  FLEET FIREWALL VERIFICATION SUMMARY%b\n' "${BLD}" "${RST}"
printf '%b══════════════════════════════════════════════════════════════%b\n' "${BLD}" "${RST}"

for idx in "${!NODE_NAMES[@]}"; do
    node_name="${NODE_NAMES[$idx]}"
    result="${NODE_RESULT[${node_name}]:-UNKNOWN}"
    case "${result}" in
        PASS)
            printf '  %b✓ %-10s%b %s\n' "${GRN}" "${node_name}" "${RST}" "${result}"
            ;;
        UNREACHABLE)
            printf '  %b⊘ %-10s%b %s\n' "${YEL}" "${node_name}" "${RST}" "${result}"
            ;;
        *)
            printf '  %b✗ %-10s%b %s\n' "${RED}" "${node_name}" "${RST}" "${result}"
            ;;
    esac
done

printf '\n'
printf '  Passed:      %b%d%b\n' "${GRN}" "${TOTAL_PASS}" "${RST}"
printf '  Failed:      %b%d%b\n' "${RED}" "${TOTAL_FAIL}" "${RST}"
printf '  Unreachable: %b%d%b\n' "${YEL}" "${TOTAL_SKIP}" "${RST}"
printf '\n'

if [[ ${TOTAL_FAIL} -gt 0 || ${TOTAL_SKIP} -gt 0 ]]; then
    printf '%b⚠  Fleet verification completed with issues.%b\n' "${RED}" "${RST}"
    exit 1
else
    printf '%b✓  All nodes passed firewall verification.%b\n' "${GRN}" "${RST}"
    exit 0
fi
