pfsense_monitor.py

#!/usr/bin/env python3
"""
pfSense Monitor
==========================
Monitors pfSense services and takes corrective actions:
1. WAN Gateway: Auto-detects and pings WAN gateway, restarts interface if needed
2. Unbound DNS: Multi-layer health checking:
   - Process state detection (SBWAIT/STOP states)
   - Sustained high CPU detection (spinning/livelock)
   - DNS responsiveness testing (catches kernel deadlocks)
3. Sends email alerts and logs all actions

Requirements:
- Python 3.6+
- Gmail app password for sending emails
- Root privileges for network and service operations
"""

import subprocess
import time
import logging
import smtplib
import socket
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from datetime import datetime
import os
import sys
import json
from pathlib import Path

class PfSenseMonitor:
    def __init__(self, config_file=None):
        """Initialize the pfSense monitor with configuration."""
        if config_file is None:
            # Use the same directory as the script for config file
            script_dir = os.path.dirname(os.path.abspath(__file__))
            config_file = os.path.join(script_dir, "pfsense_monitor_config.json")
        
        self.config_file = os.path.abspath(config_file)
        self.config = self.load_config()
        self.setup_logging()
        
    def load_config(self):
        """Load configuration from JSON file or create default config."""
        default_config = {
            "email": {
                "smtp_server": "smtp.gmail.com",
                "smtp_port": 587,
                "sender_email": "your-email@gmail.com",
                "sender_password": "your-app-password",
                "recipient_email": "alert-recipient@gmail.com",
                "fallback_dns_servers": ["8.8.8.8", "1.1.1.1"],
                "smtp_server_fallback_ips": ["142.250.27.109", "142.250.27.108", "74.125.133.109"]
            },
            "network": {
                "wan_interface": "em0",
                "gateway_override": "",
                "timeout_seconds": 5,
                "retry_attempts": 3,
                "retry_delay": 10,
                "ping_count": 3
            },
            "unbound": {
                "enabled": True,
                "process_name": "unbound",
                "bad_states": ["SBWAIT", "STOP"],
                "restart_attempts": 2,
                "restart_timeout": 30,
                "force_reboot_on_failure": True,
                "cpu_monitoring": {
                    "enabled": True,
                    "threshold_percent": 80.0,
                    "sample_count": 3,
                    "sample_interval_seconds": 5
                },
                "dns_responsiveness": {
                    "enabled": True,
                    "test_domains": ["google.com", "cloudflare.com"],
                    "timeout_seconds": 5,
                    "require_all": False
                },
                "pf_lock_contention": {
                    "wait_for_lock_release": True,
                    "max_wait_seconds": 90,
                    "poll_interval_seconds": 15
                }
            },
            "watchdog": {
                "enabled": False,
                "use_as_last_resort": True,
                "timeout_seconds": 60
            },
            "logging": {
                "log_file": "/var/log/pfsense_monitor.log",
                "max_log_size_mb": 10
            },
            "actions": {
                "restart_interface_delay": 30,
                "reboot_delay": 60
            }
        }
        
        if os.path.exists(self.config_file):
            try:
                with open(self.config_file, 'r') as f:
                    user_config = json.load(f)
                # Merge with defaults for any missing keys
                merged_config = self.merge_configs(default_config, user_config)
                return merged_config
            except Exception as e:
                print(f"Error loading config from {self.config_file}: {e}")
                print("Using default configuration.")
                return default_config
        else:
            # Create default config file
            try:
                os.makedirs(os.path.dirname(self.config_file), exist_ok=True)
                with open(self.config_file, 'w') as f:
                    json.dump(default_config, f, indent=4)
                print(f"Created default config file: {self.config_file}")
                print("Please edit the configuration file with your settings before running.")
            except Exception as e:
                print(f"Could not create config file {self.config_file}: {e}")
                print("Using default configuration in memory.")
            return default_config
    
    def merge_configs(self, default, user):
        """Recursively merge user config with default config."""
        result = default.copy()
        for key, value in user.items():
            if key in result and isinstance(value, dict) and isinstance(result[key], dict):
                result[key] = self.merge_configs(result[key], value)
            else:
                result[key] = value
        return result
    
    def setup_logging(self):
        """Configure logging to file and console."""
        log_file = self.config["logging"]["log_file"]
        
        # Ensure log directory exists
        Path(log_file).parent.mkdir(parents=True, exist_ok=True)
        
        # Clear any existing handlers
        for handler in logging.root.handlers[:]:
            logging.root.removeHandler(handler)
        
        # Configure logging
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler(log_file),
                logging.StreamHandler(sys.stdout)
            ],
            force=True
        )
        self.logger = logging.getLogger(__name__)
        self.logger.info(f"pfSense Monitor initialized")
        self.logger.info(f"Config file: {self.config_file}")
        self.logger.info(f"WAN Interface: {self.config['network']['wan_interface']}")
    
    def get_wan_gateway(self):
        """Detect the default gateway for the WAN interface."""
        wan_interface = self.config["network"]["wan_interface"]
        gateway_override = self.config["network"]["gateway_override"]
        
        # Use manual override if specified
        if gateway_override:
            self.logger.info(f"Using manual gateway override: {gateway_override}")
            return gateway_override
        
        try:
            # Method 1: Try to get default route using netstat
            result = subprocess.run(['netstat', '-rn'], capture_output=True, text=True, timeout=10)
            if result.returncode == 0:
                lines = result.stdout.split('\n')
                for line in lines:
                    # Look for default route
                    if line.strip().startswith('default') or line.strip().startswith('0.0.0.0'):
                        parts = line.split()
                        if len(parts) >= 2:
                            gateway = parts[1]
                            if self.is_valid_ip(gateway):
                                self.logger.info(f"Detected gateway from netstat: {gateway}")
                                return gateway
            
            # Method 2: Try route command if available (FreeBSD/pfSense)
            try:
                result = subprocess.run(['route', '-n', 'get', 'default'], 
                                      capture_output=True, text=True, timeout=10)
                if result.returncode == 0:
                    lines = result.stdout.split('\n')
                    for line in lines:
                        if 'gateway:' in line.lower():
                            gateway = line.split(':')[1].strip()
                            if self.is_valid_ip(gateway):
                                self.logger.info(f"Detected gateway from route command: {gateway}")
                                return gateway
            except subprocess.CalledProcessError:
                pass
            
            # Method 3: Parse /proc/net/route (Linux systems)
            try:
                with open('/proc/net/route', 'r') as f:
                    lines = f.readlines()
                    for line in lines[1:]:  # Skip header
                        fields = line.strip().split('\t')
                        if len(fields) >= 8:
                            iface, dest, gateway = fields[0], fields[1], fields[2]
                            # Check for default route (destination 00000000)
                            if dest == '00000000' and iface == wan_interface:
                                # Convert hex gateway to IP
                                gateway_ip = socket.inet_ntoa(bytes.fromhex(gateway)[::-1])
                                if self.is_valid_ip(gateway_ip):
                                    self.logger.info(f"Detected gateway from /proc/net/route: {gateway_ip}")
                                    return gateway_ip
            except FileNotFoundError:
                pass  # /proc/net/route doesn't exist on FreeBSD (pfSense)
            
            raise Exception("Could not determine default gateway using any method")
            
        except subprocess.TimeoutExpired:
            raise Exception("Timeout while trying to determine gateway")
        except Exception as e:
            error_msg = f"Failed to detect WAN gateway: {e}"
            self.logger.error(error_msg)
            raise Exception(error_msg)
    
    def is_valid_ip(self, ip):
        """Validate if string is a valid IP address."""
        try:
            socket.inet_aton(ip)
            # Additional check to ensure it's not invalid gateways
            return ip != "0.0.0.0" and not ip.startswith("127.") and ip != "255.255.255.255"
        except socket.error:
            return False
    
    def test_gateway_connectivity(self):
        """Test connectivity by pinging the WAN gateway."""
        try:
            gateway = self.get_wan_gateway()
            if not gateway:
                self.logger.error("No gateway detected, cannot test connectivity")
                return False
            
            timeout = self.config["network"]["timeout_seconds"]
            ping_count = self.config["network"]["ping_count"]
            
            self.logger.info(f"Testing connectivity to gateway: {gateway}")
            
            # Use ping command for connectivity test
            result = subprocess.run([
                'ping', '-c', str(ping_count), 
                '-W', str(timeout * 1000),  # timeout in milliseconds
                '-i', '0.5',  # 0.5 second interval between pings
                gateway
            ], capture_output=True, text=True, timeout=timeout * ping_count + 10)
            
            if result.returncode == 0:
                # Parse ping output to get success rate
                output = result.stdout
                if 'packet loss' in output:
                    # Extract packet loss info
                    loss_line = [line for line in output.split('\n') if 'packet loss' in line]
                    if loss_line:
                        loss_info = loss_line[0]
                        self.logger.info(f"Gateway ping result: {loss_info.strip()}")
                        
                        # Consider successful if less than 100% packet loss
                        if '100% packet loss' not in loss_info:
                            return True
                else:
                    self.logger.info(f"Gateway {gateway} is reachable")
                    return True
            
            self.logger.warning(f"Gateway {gateway} is not reachable")
            return False
            
        except subprocess.TimeoutExpired:
            self.logger.warning(f"Ping to gateway timed out")
            return False
        except Exception as e:
            self.logger.error(f"Error testing gateway connectivity: {e}")
            return False
    
    def restart_wan_interface(self):
        """Restart the WAN interface."""
        wan_interface = self.config["network"]["wan_interface"]
        delay = self.config["actions"]["restart_interface_delay"]
        
        try:
            self.logger.info(f"Bringing down WAN interface: {wan_interface}")
            subprocess.run(['ifconfig', wan_interface, 'down'], check=True)
            
            time.sleep(5)  # Brief pause
            
            self.logger.info(f"Bringing up WAN interface: {wan_interface}")
            subprocess.run(['ifconfig', wan_interface, 'up'], check=True)
            
            self.logger.info(f"Waiting {delay} seconds for interface to stabilize")
            time.sleep(delay)
            
            return True
            
        except subprocess.CalledProcessError as e:
            error_msg = f"Failed to restart WAN interface {wan_interface}: {e}"
            self.logger.error(error_msg)
            return False
        except Exception as e:
            error_msg = f"Unexpected error restarting interface {wan_interface}: {e}"
            self.logger.error(error_msg)
            return False
    
    def reboot_system(self):
        """Initiate system reboot."""
        delay = self.config["actions"]["reboot_delay"]
        
        try:
            self.logger.critical("Initiating system reboot due to persistent gateway connectivity issues")
            self.send_email_alert("System Reboot Initiated", 
                                "pfSense system is rebooting due to persistent WAN gateway connectivity issues.")
            
            # Schedule reboot with delay
            subprocess.run(['shutdown', '-r', f'+{delay//60}'], check=True)
            self.logger.info(f"System reboot scheduled in {delay} seconds")
            
        except subprocess.CalledProcessError as e:
            error_msg = f"Failed to initiate reboot: {e}"
            self.logger.error(error_msg)
            self.send_email_alert("Reboot Failed", error_msg)
        except Exception as e:
            error_msg = f"Unexpected error during reboot: {e}"
            self.logger.error(error_msg)
            self.send_email_alert("Reboot Error", error_msg)

    def _get_unbound_cpu_usage(self):
        """
        Get current CPU usage for the unbound process.
        Returns CPU percentage as float, or None if process not found or error occurs.
        """
        process_name = self.config["unbound"]["process_name"]

        try:
            result = subprocess.run(['ps', 'aux'], capture_output=True, text=True, timeout=10)

            if result.returncode == 0:
                lines = result.stdout.split('\n')
                for line in lines:
                    if process_name in line and not 'grep' in line:
                        # Extract command from ps output (column 10+)
                        # Check that the executable name (not args) matches process_name
                        parts = line.split()
                        if len(parts) >= 11:
                            # Command is everything from index 10 onwards
                            command = ' '.join(parts[10:])
                            # Get just the executable name (first part of command)
                            executable = command.split()[0] if command else ""
                            # Extract basename in case it's a full path
                            executable_name = executable.split('/')[-1]

                            # Only match if the actual executable is our target process
                            if executable_name == process_name or executable_name.startswith(process_name + ':'):
                                # %CPU is at index 2 in ps aux output
                                cpu_str = parts[2]
                                try:
                                    cpu_percent = float(cpu_str)
                                    return cpu_percent
                                except ValueError:
                                    self.logger.error(f"Failed to parse CPU value: {cpu_str}")
                                    return None

                # Process not found
                return None
            else:
                self.logger.error("ps command failed while checking CPU")
                return None

        except subprocess.TimeoutExpired:
            self.logger.error("Timeout checking unbound CPU usage")
            return None
        except Exception as e:
            self.logger.error(f"Error getting unbound CPU usage: {e}")
            return None

    def _test_dns_responsiveness(self):
        """
        Test if DNS is actually responding to queries.
        This catches cases where the process is running but deadlocked/unresponsive.
        Returns True if DNS is responsive, False otherwise.
        """
        dns_config = self.config["unbound"].get("dns_responsiveness", {})

        # Check if DNS responsiveness testing is enabled
        if not dns_config.get("enabled", True):
            return True  # Skip check if disabled

        test_domains = dns_config.get("test_domains", ["google.com", "cloudflare.com"])
        timeout = dns_config.get("timeout_seconds", 5)
        require_all = dns_config.get("require_all", False)

        self.logger.info(f"Testing DNS responsiveness (timeout: {timeout}s, domains: {test_domains})")

        successful = 0
        failed = 0

        for domain in test_domains:
            try:
                # Try drill first (common on FreeBSD/pfSense)
                result = subprocess.run(
                    ['drill', domain, '@127.0.0.1'],
                    capture_output=True,
                    text=True,
                    timeout=timeout
                )

                if result.returncode == 0 and 'ANSWER SECTION' in result.stdout:
                    self.logger.info(f"DNS lookup for {domain}: OK")
                    successful += 1
                else:
                    self.logger.warning(f"DNS lookup for {domain}: FAILED (no answer)")
                    failed += 1

            except subprocess.TimeoutExpired:
                self.logger.warning(f"DNS lookup for {domain}: TIMEOUT after {timeout}s")
                failed += 1
            except FileNotFoundError:
                # drill not available, try dig
                try:
                    result = subprocess.run(
                        ['dig', '+short', '+time=2', '+tries=1', domain, '@127.0.0.1'],
                        capture_output=True,
                        text=True,
                        timeout=timeout
                    )

                    if result.returncode == 0 and result.stdout.strip():
                        self.logger.info(f"DNS lookup for {domain}: OK (via dig)")
                        successful += 1
                    else:
                        self.logger.warning(f"DNS lookup for {domain}: FAILED (no answer)")
                        failed += 1

                except subprocess.TimeoutExpired:
                    self.logger.warning(f"DNS lookup for {domain}: TIMEOUT after {timeout}s")
                    failed += 1
                except FileNotFoundError:
                    self.logger.error("Neither drill nor dig available for DNS testing")
                    # Can't test without tools - assume OK to avoid false positives
                    return True
                except Exception as e:
                    self.logger.warning(f"DNS lookup for {domain}: ERROR ({e})")
                    failed += 1
            except Exception as e:
                self.logger.warning(f"DNS lookup for {domain}: ERROR ({e})")
                failed += 1

        # Evaluate results
        total = len(test_domains)

        if require_all:
            # All domains must resolve successfully
            if failed > 0:
                self.logger.error(f"DNS responsiveness check FAILED: {failed}/{total} domains failed (require_all=True)")
                return False
            self.logger.info(f"DNS responsiveness check PASSED: {successful}/{total} domains resolved")
            return True
        else:
            # At least one domain must resolve successfully
            if successful == 0:
                self.logger.error(f"DNS responsiveness check FAILED: 0/{total} domains resolved")
                return False
            self.logger.info(f"DNS responsiveness check PASSED: {successful}/{total} domains resolved")
            return True

    def _check_unbound_cpu_sustained(self):
        """
        Check if unbound CPU usage is sustained above threshold.
        Takes multiple samples over time to confirm sustained high usage.
        Returns True if CPU is OK, False if sustained high CPU detected.
        """
        cpu_config = self.config["unbound"].get("cpu_monitoring", {})

        # Check if CPU monitoring is enabled
        if not cpu_config.get("enabled", False):
            return True  # Skip check if disabled

        threshold = cpu_config.get("threshold_percent", 80.0)
        sample_count = cpu_config.get("sample_count", 3)
        sample_interval = cpu_config.get("sample_interval_seconds", 5)

        self.logger.info(f"Checking unbound CPU usage (threshold: {threshold}%, samples: {sample_count})")

        samples = []
        samples_above_threshold = 0

        for i in range(sample_count):
            if i > 0:  # Don't sleep before first sample
                time.sleep(sample_interval)

            cpu_usage = self._get_unbound_cpu_usage()

            if cpu_usage is None:
                self.logger.warning(f"CPU sample {i+1}/{sample_count}: Process not found or error")
                # If process not found, return False (will be caught by state check)
                return False

            samples.append(cpu_usage)
            self.logger.info(f"CPU sample {i+1}/{sample_count}: {cpu_usage}%")

            if cpu_usage > threshold:
                samples_above_threshold += 1

        # Check if majority of samples exceeded threshold
        avg_cpu = sum(samples) / len(samples) if samples else 0

        if samples_above_threshold >= (sample_count / 2):
            self.logger.error(
                f"Unbound CPU usage sustained above threshold: "
                f"{samples_above_threshold}/{sample_count} samples exceeded {threshold}% "
                f"(average: {avg_cpu:.1f}%)"
            )
            return False
        else:
            self.logger.info(
                f"Unbound CPU usage normal: average {avg_cpu:.1f}% "
                f"({samples_above_threshold}/{sample_count} samples above threshold)"
            )
            return True

    def _detect_pf_lock_contention(self, pids=None):
        """
        Analyze kernel stack traces to detect pf (packet filter) lock contention.

        pf lock contention is a FreeBSD issue where a process (like unbound)
        gets stuck spinning on _rm_rlock_hard while trying to pass a packet through
        the packet filter. This happens when pfBlocker or filterdns holds the pf
        write lock during a table update. The process appears "running" (Rs state)
        with high CPU but cannot respond to any requests.

        Signature: _rm_rlock_hard (or rw_rlock_hard) + pf_test in the same
        kernel stack trace.

        Returns a dict:
            detected: bool - True if pf lock contention was detected
            confidence: str - "high", "medium", or "none"
            evidence: list of str - specific findings
            stack_traces: dict - pid -> raw stack trace string
            pf_search_rate: float or None - pf state searches/sec
            suspect_processes: list of str - processes that may hold the pf write lock
        """
        detection = {
            "detected": False,
            "confidence": "none",
            "evidence": [],
            "stack_traces": {},
            "pf_search_rate": None,
            "suspect_processes": [],
        }

        # High confidence: a lock-spin frame AND a pf evaluation frame in the same trace.
        HIGH_CONFIDENCE_PAIRS = [
            ("_rm_rlock_hard", "pf_test"),
            ("_rm_rlock_hard", "pf_check"),
            ("rw_rlock_hard", "pf_test"),
            ("rw_rlock_hard", "pf_check"),
            ("_rm_rlock", "pf_test"),
            ("rw_rlock", "pf_test"),
        ]
        # Medium confidence: lock-spin frame with pf references anywhere in trace.
        MEDIUM_CONFIDENCE_LOCK_FRAMES = [
            "_rm_rlock_hard", "rw_rlock_hard", "rw_wlock_hard",
            "_rm_wlock_hard", "pf_lock",
        ]

        process_name = self.config["unbound"]["process_name"]

        # Find unbound PIDs if not provided.
        if pids is None:
            pids = []
            try:
                ps_result = subprocess.run(
                    ['ps', 'aux'], capture_output=True, text=True, timeout=10
                )
                if ps_result.returncode == 0:
                    for line in ps_result.stdout.split('\n'):
                        if process_name in line and 'grep' not in line:
                            parts = line.split()
                            if len(parts) >= 11:
                                command = ' '.join(parts[10:])
                                executable = command.split()[0].split('/')[-1] if command else ""
                                if executable == process_name or executable.startswith(process_name + ':'):
                                    pids.append(parts[1])
            except Exception as e:
                self.logger.debug(f"Error finding unbound PIDs for lock detection: {e}")

        # Analyze kernel stack traces for each unbound PID.
        for pid in pids:
            try:
                stack_result = subprocess.run(
                    ['procstat', '-kk', pid],
                    capture_output=True, text=True, timeout=5
                )
                if stack_result.returncode != 0 or not stack_result.stdout.strip():
                    continue

                trace = stack_result.stdout.strip()
                detection["stack_traces"][pid] = trace

                # Check for high confidence pf lock signature.
                for lock_frame, pf_frame in HIGH_CONFIDENCE_PAIRS:
                    if lock_frame in trace and pf_frame in trace:
                        detection["detected"] = True
                        detection["confidence"] = "high"
                        detection["evidence"].append(
                            f"PID {pid}: kernel stack contains '{lock_frame}' "
                            f"waiting on '{pf_frame}' (pf read-lock spin)"
                        )
                        break

                # Check for medium confidence if not already high.
                if detection["confidence"] != "high":
                    for frame in MEDIUM_CONFIDENCE_LOCK_FRAMES:
                        if frame in trace and "pf_" in trace:
                            detection["detected"] = True
                            if detection["confidence"] == "none":
                                detection["confidence"] = "medium"
                            detection["evidence"].append(
                                f"PID {pid}: kernel stack contains '{frame}' "
                                f"with pf references"
                            )
                            break

            except subprocess.TimeoutExpired:
                # procstat timeout on a live process can itself indicate severe lock contention.
                detection["evidence"].append(
                    f"PID {pid}: procstat -kk timed out (possible kernel deadlock)"
                )
                if detection["confidence"] == "none":
                    detection["confidence"] = "medium"
                    detection["detected"] = True
            except FileNotFoundError:
                pass  # procstat not available on this system
            except Exception as e:
                self.logger.debug(f"Error getting stack trace for PID {pid}: {e}")

        # Get pf state table search rate from pfctl -si.
        # High rates (>1500/s) indicate heavy pf table activity and elevated lock
        # contention risk, especially during pfBlocker/filterdns updates.
        try:
            pfctl_result = subprocess.run(
                ['pfctl', '-si'], capture_output=True, text=True, timeout=5
            )
            if pfctl_result.returncode == 0:
                for line in pfctl_result.stdout.split('\n'):
                    if 'searches' in line.lower() and '/s' in line:
                        for part in line.split():
                            if part.endswith('/s'):
                                try:
                                    rate = float(part[:-2])
                                    detection["pf_search_rate"] = rate
                                    if rate > 1500 and not detection["detected"]:
                                        detection["evidence"].append(
                                            f"High pf state search rate: {rate:.0f}/s "
                                            f"(elevated lock contention risk)"
                                        )
                                except ValueError:
                                    pass
                        break
        except subprocess.TimeoutExpired:
            detection["evidence"].append(
                "pfctl -si timed out (pf subsystem may be locked)"
            )
            detection["detected"] = True
            detection["confidence"] = "high"
        except Exception:
            pass

        # Identify processes that commonly hold the pf write lock.
        # pfBlocker (php_pfb) and filterdns are the most frequent culprits.
        SUSPECT_PATTERNS = [
            ("php_pfb", "pfBlocker"),
            ("filterdns", "filterdns"),
            ("pfctl", "pfctl (scripted/manual)"),
        ]
        try:
            ps_result = subprocess.run(
                ['ps', 'aux'], capture_output=True, text=True, timeout=10
            )
            if ps_result.returncode == 0:
                for line in ps_result.stdout.split('\n'):
                    if 'grep' in line:
                        continue
                    for pattern, label in SUSPECT_PATTERNS:
                        if pattern in line and label not in detection["suspect_processes"]:
                            detection["suspect_processes"].append(label)
        except Exception:
            pass

        return detection

    def _is_filterdns_running(self):
        """Check if filterdns is currently running (common pf write-lock holder)."""
        try:
            result = subprocess.run(
                ['pgrep', '-x', 'filterdns'],
                capture_output=True, text=True, timeout=5
            )
            return result.returncode == 0
        except Exception:
            return False

    def _wait_for_pf_lock_release(self, pf_detection):
        """
        When pf lock contention is detected, wait for the lock to release naturally
        before attempting service restarts.

        Restart attempts are futile while the pf write lock is held — unbound-control
        and the new unbound process will hit the same lock. Waiting for the lock holder
        (typically filterdns during a 5-minute alias refresh) to finish is more effective.

        Polls DNS responsiveness every poll_interval_seconds up to max_wait_seconds.

        Returns True if Unbound recovered on its own (no restart needed).
        Returns False if the wait timed out without recovery.
        """
        wait_config = self.config["unbound"].get("pf_lock_contention", {})
        if not wait_config.get("wait_for_lock_release", True):
            return False

        max_wait = wait_config.get("max_wait_seconds", 90)
        poll_interval = wait_config.get("poll_interval_seconds", 15)
        confidence = pf_detection.get("confidence", "unknown")
        suspect = ", ".join(pf_detection.get("suspect_processes", [])) or "filterdns/pfBlocker"

        self.logger.info(
            f"pf lock contention detected ({confidence} confidence). "
            f"Suspected lock holder: {suspect}. "
            f"Waiting up to {max_wait}s for lock to release before attempting restart."
        )

        elapsed = 0
        while elapsed < max_wait:
            time.sleep(poll_interval)
            elapsed += poll_interval

            filterdns_running = self._is_filterdns_running()
            self.logger.info(
                f"pf lock wait: {elapsed}/{max_wait}s elapsed, "
                f"filterdns running: {filterdns_running}. Testing DNS responsiveness..."
            )

            if self._test_dns_responsiveness():
                self.logger.info(
                    f"Unbound recovered after {elapsed}s wait — pf lock released. "
                    f"No restart needed."
                )
                return True

        self.logger.warning(
            f"pf lock wait timed out after {max_wait}s without recovery. "
            f"Proceeding with restart attempts."
        )
        return False

    def get_unbound_diagnostics(self):
        """
        Collect diagnostic information about unbound and system state.
        Returns a formatted string with diagnostic details for email alerts.
        """
        diagnostics = []
        process_name = self.config["unbound"]["process_name"]

        # 1. Get all unbound process details from ps aux
        try:
            result = subprocess.run(['ps', 'aux'], capture_output=True, text=True, timeout=10)
            if result.returncode == 0:
                unbound_processes = []
                for line in result.stdout.split('\n'):
                    if process_name in line and 'grep' not in line:
                        parts = line.split()
                        if len(parts) >= 11:
                            command = ' '.join(parts[10:])
                            executable = command.split()[0] if command else ""
                            executable_name = executable.split('/')[-1]
                            if executable_name == process_name or executable_name.startswith(process_name + ':'):
                                unbound_processes.append({
                                    'user': parts[0],
                                    'pid': parts[1],
                                    'cpu': parts[2],
                                    'mem': parts[3],
                                    'vsz': parts[4],
                                    'rss': parts[5],
                                    'stat': parts[7],
                                    'start': parts[8],
                                    'time': parts[9],
                                    'command': command
                                })

                if unbound_processes:
                    diagnostics.append(f"Unbound Processes Found: {len(unbound_processes)}")
                    diagnostics.append("-" * 40)
                    for proc in unbound_processes:
                        diagnostics.append(f"  PID: {proc['pid']}")
                        diagnostics.append(f"  State: {proc['stat']}")
                        diagnostics.append(f"  CPU: {proc['cpu']}%")
                        diagnostics.append(f"  Memory: {proc['mem']}% (RSS: {proc['rss']} KB)")
                        diagnostics.append(f"  Started: {proc['start']}")
                        diagnostics.append(f"  CPU Time: {proc['time']}")
                        diagnostics.append(f"  Command: {proc['command']}")
                        diagnostics.append("")

                    # Collect procstat info for each unbound process
                    for proc in unbound_processes:
                        pid = proc['pid']
                        diagnostics.append(f"Procstat for PID {pid}:")
                        diagnostics.append("-" * 40)

                        # Thread info - shows what each thread is waiting on
                        try:
                            result = subprocess.run(['procstat', '-t', pid],
                                                   capture_output=True, text=True, timeout=5)
                            if result.returncode == 0 and result.stdout.strip():
                                diagnostics.append("  Thread Status:")
                                for line in result.stdout.strip().split('\n')[:15]:
                                    diagnostics.append(f"    {line}")
                        except FileNotFoundError:
                            diagnostics.append("  procstat not available")
                        except Exception as e:
                            diagnostics.append(f"  Thread info error: {e}")

                        # Kernel stack trace - shows what the process is blocked on
                        try:
                            result = subprocess.run(['procstat', '-kk', pid],
                                                   capture_output=True, text=True, timeout=5)
                            if result.returncode == 0 and result.stdout.strip():
                                diagnostics.append("  Kernel Stack Trace:")
                                for line in result.stdout.strip().split('\n')[:20]:
                                    diagnostics.append(f"    {line}")
                        except FileNotFoundError:
                            pass  # Already reported above
                        except Exception as e:
                            diagnostics.append(f"  Stack trace error: {e}")

                        # File descriptors - may show stuck sockets
                        try:
                            result = subprocess.run(['procstat', '-f', pid],
                                                   capture_output=True, text=True, timeout=5)
                            if result.returncode == 0 and result.stdout.strip():
                                lines = result.stdout.strip().split('\n')
                                # Count fd types
                                socket_count = sum(1 for l in lines if 'socket' in l.lower())
                                pipe_count = sum(1 for l in lines if 'pipe' in l.lower())
                                file_count = sum(1 for l in lines if 'vnode' in l.lower())
                                diagnostics.append(f"  File Descriptors: {len(lines)-1} total "
                                                 f"(sockets: {socket_count}, pipes: {pipe_count}, files: {file_count})")
                                # Show first few socket entries if any
                                socket_lines = [l for l in lines if 'socket' in l.lower()][:5]
                                if socket_lines:
                                    diagnostics.append("  Recent sockets:")
                                    for line in socket_lines:
                                        diagnostics.append(f"    {line[:100]}")
                        except FileNotFoundError:
                            pass
                        except Exception as e:
                            diagnostics.append(f"  FD info error: {e}")

                        diagnostics.append("")
                else:
                    diagnostics.append("Unbound Processes Found: 0 (NOT RUNNING)")
        except Exception as e:
            diagnostics.append(f"Error getting process info: {e}")

        # 2. Get system load average
        try:
            with open('/proc/loadavg', 'r') as f:
                load = f.read().strip()
                diagnostics.append(f"System Load: {load}")
        except FileNotFoundError:
            # FreeBSD doesn't have /proc/loadavg, try sysctl
            try:
                result = subprocess.run(['sysctl', '-n', 'vm.loadavg'],
                                       capture_output=True, text=True, timeout=5)
                if result.returncode == 0:
                    diagnostics.append(f"System Load: {result.stdout.strip()}")
            except Exception:
                pass
        except Exception as e:
            diagnostics.append(f"Error getting load average: {e}")

        # 3. Get memory usage
        try:
            result = subprocess.run(['sysctl', '-n', 'hw.physmem', 'hw.usermem'],
                                   capture_output=True, text=True, timeout=5)
            if result.returncode == 0:
                lines = result.stdout.strip().split('\n')
                if len(lines) >= 2:
                    physmem = int(lines[0]) / (1024 * 1024 * 1024)
                    usermem = int(lines[1]) / (1024 * 1024 * 1024)
                    diagnostics.append(f"Physical Memory: {physmem:.2f} GB")
                    diagnostics.append(f"User Memory: {usermem:.2f} GB")
        except Exception:
            pass

        # 4. Get top memory-consuming processes
        try:
            result = subprocess.run(['ps', 'aux', '--sort=-rss'],
                                   capture_output=True, text=True, timeout=10)
            if result.returncode != 0:
                # FreeBSD ps doesn't support --sort, use different approach
                result = subprocess.run(['ps', 'auxww'], capture_output=True, text=True, timeout=10)

            if result.returncode == 0:
                lines = result.stdout.strip().split('\n')
                if len(lines) > 1:
                    diagnostics.append("\nTop Memory Consumers:")
                    diagnostics.append("-" * 40)
                    # Parse and sort by RSS (column 5)
                    procs = []
                    for line in lines[1:]:  # Skip header
                        parts = line.split()
                        if len(parts) >= 11:
                            try:
                                rss = int(parts[5])
                                procs.append((rss, parts[0], parts[1], parts[2], parts[3], ' '.join(parts[10:])[:40]))
                            except ValueError:
                                continue
                    procs.sort(reverse=True)
                    for rss, _user, pid, cpu, mem, cmd in procs[:5]:
                        diagnostics.append(f"  {pid:>6} {cpu:>5}% CPU {mem:>5}% MEM  {cmd}")
        except Exception as e:
            diagnostics.append(f"Error getting top processes: {e}")

        # 5. Check recent unbound log entries
        unbound_log_paths = [
            '/var/log/resolver.log',
            '/var/log/unbound.log',
            '/var/log/messages'
        ]
        for log_path in unbound_log_paths:
            try:
                result = subprocess.run(['tail', '-20', log_path],
                                       capture_output=True, text=True, timeout=5)
                if result.returncode == 0 and result.stdout.strip():
                    # Filter for unbound-related entries
                    unbound_lines = [line for line in result.stdout.split('\n')
                                    if 'unbound' in line.lower() or 'dns' in line.lower()]
                    if unbound_lines:
                        diagnostics.append(f"\nRecent DNS Log Entries ({log_path}):")
                        diagnostics.append("-" * 40)
                        for line in unbound_lines[-10:]:  # Last 10 relevant lines
                            diagnostics.append(f"  {line[:100]}")
                        break
            except Exception:
                continue

        # 6. Check DNS resolution capability
        diagnostics.append("\nDNS Resolution Test:")
        diagnostics.append("-" * 40)
        test_domains = ['google.com', 'cloudflare.com']
        for domain in test_domains:
            try:
                result = subprocess.run(['drill', domain, '@127.0.0.1'],
                                       capture_output=True, text=True, timeout=5)
                if result.returncode == 0 and 'ANSWER SECTION' in result.stdout:
                    diagnostics.append(f"  {domain}: OK")
                else:
                    diagnostics.append(f"  {domain}: FAILED (no answer)")
            except subprocess.TimeoutExpired:
                diagnostics.append(f"  {domain}: TIMEOUT")
            except FileNotFoundError:
                # Try dig if drill not available
                try:
                    result = subprocess.run(['dig', '+short', domain, '@127.0.0.1'],
                                           capture_output=True, text=True, timeout=5)
                    if result.returncode == 0 and result.stdout.strip():
                        diagnostics.append(f"  {domain}: OK")
                    else:
                        diagnostics.append(f"  {domain}: FAILED")
                except Exception:
                    diagnostics.append(f"  {domain}: Unable to test (no drill/dig)")
            except Exception as e:
                diagnostics.append(f"  {domain}: ERROR ({e})")

        # 7. Check unbound service status
        try:
            result = subprocess.run(['service', 'unbound', 'status'],
                                   capture_output=True, text=True, timeout=10)
            diagnostics.append(f"\nService Status:")
            diagnostics.append("-" * 40)
            diagnostics.append(f"  {result.stdout.strip()}")
            if result.stderr.strip():
                diagnostics.append(f"  stderr: {result.stderr.strip()}")
        except Exception as e:
            diagnostics.append(f"\nService Status: Error checking ({e})")

        # 8. Check unbound-control status (direct control interface query)
        try:
            result = subprocess.run(['unbound-control', 'status'],
                                   capture_output=True, text=True, timeout=5)
            diagnostics.append("\nUnbound Control Status:")
            diagnostics.append("-" * 40)
            if result.returncode == 0:
                for line in result.stdout.strip().split('\n')[:10]:
                    diagnostics.append(f"  {line}")
            else:
                diagnostics.append(f"  Failed: {result.stderr.strip()[:100]}")
        except subprocess.TimeoutExpired:
            diagnostics.append("\nUnbound Control Status:")
            diagnostics.append("-" * 40)
            diagnostics.append("  TIMEOUT - unbound-control not responding (indicates hung service)")
        except FileNotFoundError:
            pass  # unbound-control not available
        except Exception as e:
            diagnostics.append(f"\nUnbound Control Status: Error ({e})")

        # 9. pf (packet filter) statistics - critical for detecting pf lock issues
        try:
            result = subprocess.run(['pfctl', '-si'],
                                   capture_output=True, text=True, timeout=5)
            if result.returncode == 0:
                diagnostics.append("\npf State Table Info:")
                diagnostics.append("-" * 40)
                for line in result.stdout.strip().split('\n'):
                    # Show key metrics
                    if any(key in line.lower() for key in ['state', 'insert', 'search', 'removal', 'match']):
                        diagnostics.append(f"  {line.strip()}")
        except subprocess.TimeoutExpired:
            diagnostics.append("\npf State Table Info:")
            diagnostics.append("-" * 40)
            diagnostics.append("  TIMEOUT - pfctl not responding (possible pf lock contention)")
        except Exception:
            pass

        # 10. System uptime
        try:
            result = subprocess.run(['uptime'], capture_output=True, text=True, timeout=5)
            if result.returncode == 0:
                diagnostics.append(f"\nSystem Uptime: {result.stdout.strip()}")
        except Exception:
            pass

        # 11. Recent kernel messages (dmesg) - may show lock contention or panics
        try:
            result = subprocess.run(['dmesg'], capture_output=True, text=True, timeout=5)
            if result.returncode == 0:
                lines = result.stdout.strip().split('\n')
                # Look for concerning messages in last 50 lines
                recent_lines = lines[-50:] if len(lines) > 50 else lines
                concerning = []
                keywords = ['lock', 'timeout', 'error', 'fail', 'panic', 'trap',
                           'witness', 'deadlock', 'spin', 'stuck', 'unbound', 'pf']
                for line in recent_lines:
                    line_lower = line.lower()
                    if any(kw in line_lower for kw in keywords):
                        concerning.append(line)

                if concerning:
                    diagnostics.append("\nRecent Kernel Messages (concerning):")
                    diagnostics.append("-" * 40)
                    for line in concerning[-10:]:  # Last 10 concerning messages
                        diagnostics.append(f"  {line[:120]}")
        except Exception:
            pass

        # 12. Network interface statistics (errors, drops)
        try:
            result = subprocess.run(['netstat', '-i', '-b'],
                                   capture_output=True, text=True, timeout=5)
            if result.returncode == 0:
                lines = result.stdout.strip().split('\n')
                diagnostics.append("\nNetwork Interface Statistics:")
                diagnostics.append("-" * 40)
                # Show header and first few interfaces
                for line in lines[:6]:
                    diagnostics.append(f"  {line[:100]}")
        except Exception:
            pass

        # 13. vmstat snapshot - overall system state
        try:
            result = subprocess.run(['vmstat', '1', '1'],
                                   capture_output=True, text=True, timeout=5)
            if result.returncode == 0:
                diagnostics.append("\nVMstat (system state):")
                diagnostics.append("-" * 40)
                for line in result.stdout.strip().split('\n'):
                    diagnostics.append(f"  {line}")
        except Exception:
            pass

        # 14. Check for processes in uninterruptible sleep (D state)
        try:
            result = subprocess.run(['ps', 'auxww'], capture_output=True, text=True, timeout=10)
            if result.returncode == 0:
                stuck_procs = []
                for line in result.stdout.split('\n')[1:]:  # Skip header
                    parts = line.split()
                    if len(parts) >= 8:
                        stat = parts[7]
                        # D = uninterruptible sleep, T = stopped, Z = zombie
                        if stat.startswith('D') or stat.startswith('T') or stat.startswith('Z'):
                            stuck_procs.append((parts[1], stat, ' '.join(parts[10:])[:50]))

                if stuck_procs:
                    diagnostics.append("\nProcesses in Problematic States (D/T/Z):")
                    diagnostics.append("-" * 40)
                    for pid, stat, cmd in stuck_procs[:10]:
                        diagnostics.append(f"  PID {pid} [{stat}]: {cmd}")
        except Exception:
            pass

        # 15. Mutex/lock statistics if available (FreeBSD)
        try:
            result = subprocess.run(['sysctl', 'debug.lock.prof.enable'],
                                   capture_output=True, text=True, timeout=2)
            if result.returncode == 0 and '1' in result.stdout:
                # Lock profiling is enabled, get stats
                result = subprocess.run(['sysctl', 'debug.lock.prof.stats'],
                                       capture_output=True, text=True, timeout=5)
                if result.returncode == 0 and result.stdout.strip():
                    diagnostics.append("\nLock Profiling Stats:")
                    diagnostics.append("-" * 40)
                    lines = result.stdout.strip().split('\n')
                    for line in lines[:15]:
                        diagnostics.append(f"  {line[:100]}")
        except Exception:
            pass

        # 16. pf lock contention analysis - identifies the specific root cause
        # seen when unbound is stuck in _rm_rlock_hard waiting on pf_test.
        pf_detection = self._detect_pf_lock_contention()
        self._pf_lock_detection = pf_detection

        if pf_detection["detected"]:
            sep = "=" * 50
            diagnostics.append(f"\n{sep}")
            diagnostics.append(
                f"*** PF LOCK CONTENTION DETECTED "
                f"({pf_detection['confidence'].upper()} CONFIDENCE) ***"
            )
            diagnostics.append(sep)
            diagnostics.append(
                "Root Cause: Unbound is stuck spinning on the pf (packet filter)"
            )
            diagnostics.append(
                "read lock while trying to send a DNS response. Another process"
            )
            diagnostics.append(
                "is holding the pf write lock (typically during a pfBlocker or"
            )
            diagnostics.append("filterdns table update).")

            if pf_detection["evidence"]:
                diagnostics.append("\nEvidence:")
                for ev in pf_detection["evidence"]:
                    diagnostics.append(f"  - {ev}")

            if pf_detection["suspect_processes"]:
                diagnostics.append("\nProcesses that may hold the pf write lock:")
                for proc in pf_detection["suspect_processes"]:
                    diagnostics.append(f"  - {proc}")

            if pf_detection["pf_search_rate"] is not None:
                diagnostics.append(
                    f"\npf state search rate: {pf_detection['pf_search_rate']:.0f}/s"
                )

            diagnostics.append("\nRecommended fixes:")
            diagnostics.append(
                "  1. Reduce pfBlocker update frequency "
                "(Firewall > pfBlocker > Update)"
            )
            diagnostics.append(
                "  2. Fix or remove failing DNS entries in filterdns/pfBlocker"
                " (check resolver.log for repeated failures)"
            )
            diagnostics.append(
                "  3. Tune pf state table "
                "(System > Advanced > Firewall & NAT > Firewall Optimization)"
            )
            diagnostics.append(sep)
        else:
            diagnostics.append("\npf Lock Contention Analysis: No contention signatures found")
            if pf_detection["pf_search_rate"] is not None:
                diagnostics.append(
                    f"  pf state search rate: {pf_detection['pf_search_rate']:.0f}/s"
                )

        return '\n'.join(diagnostics)

    def check_unbound_state(self):
        """
        Check if unbound process is healthy.
        Performs three checks:
        1. Process state (SBWAIT, STOP, etc)
        2. CPU usage (sustained high CPU indicates spinning/deadlock)
        3. DNS responsiveness (actual query test - most reliable indicator)

        Returns True if healthy, False if ANY check fails.
        """
        if not self.config["unbound"]["enabled"]:
            return True  # Skip check if disabled

        process_name = self.config["unbound"]["process_name"]
        bad_states = self.config["unbound"]["bad_states"]

        # First, check process state
        state_ok = True
        try:
            # Use ps command to check process state
            result = subprocess.run(['ps', 'aux'], capture_output=True, text=True, timeout=10)

            if result.returncode == 0:
                lines = result.stdout.split('\n')
                process_found = False

                for line in lines:
                    if process_name in line and not 'grep' in line:
                        # Extract command from ps output (column 10+)
                        # Check that the executable name (not args) matches process_name
                        parts = line.split()
                        if len(parts) >= 11:
                            # Command is everything from index 10 onwards
                            command = ' '.join(parts[10:])
                            # Get just the executable name (first part of command)
                            executable = command.split()[0] if command else ""
                            # Extract basename in case it's a full path
                            executable_name = executable.split('/')[-1]

                            # Only match if the actual executable is our target process
                            if executable_name == process_name or executable_name.startswith(process_name + ':'):
                                process_found = True
                                # Extract state from ps output
                                # On FreeBSD/pfSense, the STAT column shows process state
                                state = parts[7]  # STAT column
                                self.logger.info(f"Unbound process found with state: {state}")

                                # Check if state contains any bad states
                                for bad_state in bad_states:
                                    if bad_state in state:
                                        self.logger.error(f"Unbound process in bad state: {state}")
                                        state_ok = False
                                        break
                                break

                if not process_found:
                    self.logger.warning("Unbound process not found")
                    state_ok = False
            else:
                self.logger.error("Failed to check process state: ps command failed")
                state_ok = False

        except subprocess.TimeoutExpired:
            self.logger.error("Timeout checking unbound process state")
            state_ok = False
        except Exception as e:
            self.logger.error(f"Error checking unbound state: {e}")
            state_ok = False

        # Second, check CPU usage (if enabled)
        cpu_ok = self._check_unbound_cpu_sustained()

        # Third, check DNS responsiveness (most reliable - catches deadlocks like the pf lock issue)
        # This would have caught the kernel lock deadlock where process was "running" but unresponsive
        dns_ok = self._test_dns_responsiveness()

        # Return False if ANY check failed
        if not state_ok:
            self.logger.error("Health check failed: bad process state")
            return False
        if not cpu_ok:
            self.logger.error("Health check failed: sustained high CPU")
            return False
        if not dns_ok:
            self.logger.error("Health check failed: DNS not responding (possible deadlock)")
            return False

        self.logger.info("Unbound process is healthy (state, CPU, and DNS responsiveness all normal)")
        return True

    def restart_unbound(self):
        """Restart the unbound DNS service."""
        try:
            self.logger.info("Attempting to restart unbound service")

            # Try using pfSense's service control first
            result = subprocess.run(['/usr/local/sbin/pfSsh.php', 'playback', 'svc', 'restart', 'unbound'],
                                  capture_output=True, text=True, timeout=30)

            if result.returncode == 0:
                self.logger.info("Unbound restart command sent via pfSsh")
                time.sleep(5)
                return True
            else:
                # Fallback to service command
                self.logger.warning("pfSsh restart failed, trying service command")
                result = subprocess.run(['service', 'unbound', 'restart'],
                                      capture_output=True, text=True, timeout=30)

                if result.returncode == 0:
                    self.logger.info("Unbound restarted via service command")
                    time.sleep(5)
                    return True
                else:
                    self.logger.error(f"Service restart failed: {result.stderr}")
                    return False

        except subprocess.TimeoutExpired:
            self.logger.error("Timeout while trying to restart unbound")
            return False
        except Exception as e:
            self.logger.error(f"Error restarting unbound: {e}")
            return False

    def kill_unbound_forcefully(self):
        """Attempt to forcefully kill unbound process."""
        process_name = self.config["unbound"]["process_name"]

        try:
            self.logger.warning("Attempting to forcefully kill unbound process")

            # Get unbound PID
            result = subprocess.run(['pgrep', process_name], capture_output=True, text=True, timeout=10)

            if result.returncode == 0 and result.stdout.strip():
                pid = result.stdout.strip().split()[0]
                self.logger.info(f"Found unbound PID: {pid}")

                # Try SIGTERM first
                subprocess.run(['kill', '-TERM', pid], timeout=5)
                time.sleep(3)

                # Check if still running
                result = subprocess.run(['pgrep', process_name], capture_output=True, text=True, timeout=10)
                if result.returncode != 0:
                    self.logger.info("Unbound process terminated successfully")
                    return True

                # Try SIGKILL if SIGTERM didn't work
                self.logger.warning("SIGTERM failed, trying SIGKILL")
                subprocess.run(['kill', '-KILL', pid], timeout=5)
                time.sleep(3)

                # Check again
                result = subprocess.run(['pgrep', process_name], capture_output=True, text=True, timeout=10)
                if result.returncode != 0:
                    self.logger.info("Unbound process killed successfully")
                    return True
                else:
                    self.logger.error("Failed to kill unbound process even with SIGKILL")
                    return False
            else:
                self.logger.info("Unbound process not found (already terminated?)")
                return True

        except Exception as e:
            self.logger.error(f"Error killing unbound process: {e}")
            return False

    def run_unbound_monitor(self):
        """Monitor unbound service and take corrective action if needed."""
        if not self.config["unbound"]["enabled"]:
            self.logger.info("Unbound monitoring is disabled, skipping")
            return True

        self.logger.info("Starting unbound service check")

        # Check if unbound is in a bad state
        if self.check_unbound_state():
            self.logger.info("Unbound service is running normally")
            return True

        # Unbound is in a bad state - collect diagnostics and try to restart
        self.logger.error("Unbound service detected in unhealthy state")
        diagnostics = self.get_unbound_diagnostics()
        error_msg = (
            "Unbound DNS service detected in unhealthy state.\n"
            "Possible causes:\n"
            "- Process in bad state (SBWAIT/STOP)\n"
            "- Sustained high CPU usage (spinning/livelock)\n"
            "- DNS queries timing out (kernel deadlock, e.g., pf lock contention)\n"
            "- Process not running\n\n"
            "DIAGNOSTIC INFORMATION:\n"
            "========================\n"
            f"{diagnostics}"
        )
        # Use pf lock detection result (populated by get_unbound_diagnostics) for subject.
        pf_detection = getattr(self, '_pf_lock_detection', None)
        if pf_detection and pf_detection.get("detected"):
            confidence = pf_detection.get("confidence", "unknown")
            alert_subject = f"Unbound Hung - pf Lock Contention ({confidence} confidence)"
        else:
            alert_subject = "Unbound Service Hung"

        self.logger.info("Collected diagnostic information for email alert")
        self.send_email_alert(alert_subject, error_msg)

        # If pf lock contention is detected, wait for the lock to release before
        # attempting restarts. Restart attempts will also fail while the lock is
        # held — unbound-control cannot communicate with the process, and a fresh
        # unbound process will hit the same lock immediately. Waiting is cheaper.
        if pf_detection and pf_detection.get("detected"):
            if self._wait_for_pf_lock_release(pf_detection):
                elapsed = self.config["unbound"].get("pf_lock_contention", {}).get("max_wait_seconds", 90)
                recovery_msg = (
                    "Unbound self-recovered after the pf write lock was released. "
                    "No service restart was needed.\n\n"
                    "Root cause: A filterdns alias table update held the pf write lock, "
                    "blocking Unbound from sending DNS responses through the packet filter.\n\n"
                    "To reduce the frequency of these events, consider increasing the "
                    "alias refresh interval in System > Advanced > Firewall & NAT."
                )
                self.send_email_alert("Unbound Recovered - pf Lock Released", recovery_msg)
                return True
            self.logger.warning(
                "pf lock did not release within the wait period. "
                "Proceeding with restart attempts."
            )

        restart_attempts = self.config["unbound"]["restart_attempts"]
        restart_timeout = self.config["unbound"]["restart_timeout"]

        for attempt in range(restart_attempts):
            self.logger.info(f"Restart attempt {attempt + 1}/{restart_attempts}")

            # First try to kill the process
            if not self.kill_unbound_forcefully():
                self.logger.warning("Failed to kill unbound process")

            time.sleep(5)

            # Try to restart the service
            if self.restart_unbound():
                # Wait for service to stabilize
                time.sleep(restart_timeout)

                # Verify it restarted successfully
                if self.check_unbound_state():
                    success_msg = f"Unbound service successfully restarted on attempt {attempt + 1}"
                    self.logger.info(success_msg)
                    self.send_email_alert("Unbound Service Recovered", success_msg)
                    return True
                else:
                    self.logger.warning(f"Unbound still in bad state after restart attempt {attempt + 1}")
            else:
                self.logger.error(f"Failed to restart unbound on attempt {attempt + 1}")

        # All restart attempts failed - collect fresh diagnostics
        self.logger.critical(f"Failed to restart unbound service after {restart_attempts} attempts")
        post_diagnostics = self.get_unbound_diagnostics()
        failure_msg = (
            f"Failed to restart unbound service after {restart_attempts} attempts.\n\n"
            "CURRENT DIAGNOSTIC INFORMATION:\n"
            "================================\n"
            f"{post_diagnostics}"
        )
        self.send_email_alert("Unbound Service Restart Failed", failure_msg)

        # Force reboot if configured
        if self.config["unbound"]["force_reboot_on_failure"]:
            self.logger.critical("Forcing system reboot due to unbound failure")
            reboot_msg = (
                "System is being rebooted because unbound DNS service cannot be restarted.\n\n"
                f"Restart attempts made: {restart_attempts}\n"
                "All attempts failed to restore DNS service.\n\n"
                "FINAL DIAGNOSTIC STATE:\n"
                "=======================\n"
                f"{post_diagnostics}"
            )
            self.send_email_alert("System Reboot - Unbound Failure", reboot_msg)
            self.force_reboot()
            return False

        return False

    def check_watchdog_available(self):
        """Check if hardware watchdog is available and configured."""
        try:
            # Check if watchdog device exists
            import os
            if not os.path.exists('/dev/watchdog'):
                self.logger.warning("Watchdog device /dev/watchdog not found")
                return False

            # Check if watchdogd service is running
            result = subprocess.run(['service', 'watchdogd', 'status'],
                                  capture_output=True, text=True, timeout=5)

            if 'is running' in result.stdout or result.returncode == 0:
                self.logger.info("Watchdog daemon is running and available")
                return True
            else:
                self.logger.warning("Watchdog daemon is not running")
                return False

        except Exception as e:
            self.logger.error(f"Error checking watchdog availability: {e}")
            return False

    def trigger_watchdog_reboot(self):
        """
        Trigger hardware watchdog reboot by stopping the watchdog daemon.
        The hardware will force a reboot when the watchdog timer expires.
        This is a last-resort method when all software reboot methods fail.
        """
        if not self.config["watchdog"]["enabled"]:
            self.logger.warning("Watchdog reboot requested but watchdog is disabled in config")
            return False

        if not self.config["watchdog"]["use_as_last_resort"]:
            self.logger.warning("Watchdog last resort is disabled in config")
            return False

        timeout = self.config["watchdog"]["timeout_seconds"]

        self.logger.critical("=" * 60)
        self.logger.critical("INITIATING HARDWARE WATCHDOG REBOOT")
        self.logger.critical("=" * 60)

        # Check if watchdog is available
        if not self.check_watchdog_available():
            self.logger.error("Watchdog not available - cannot trigger watchdog reboot")
            self.logger.error("Please enable watchdog in pfSense: System > Advanced > Miscellaneous")
            return False

        try:
            self.logger.critical("Stopping watchdog daemon to trigger hardware reset")
            self.logger.critical(f"System will hard reboot in approximately {timeout} seconds")

            # Send final email alert
            self.send_email_alert(
                "HARDWARE WATCHDOG REBOOT TRIGGERED",
                f"All software reboot methods failed. Hardware watchdog timer has been triggered.\n"
                f"System will perform a hard reset in approximately {timeout} seconds.\n"
                f"This is a last-resort recovery mechanism."
            )

            # Stop the watchdog daemon - this will cause hardware to reboot
            result = subprocess.run(['service', 'watchdogd', 'stop'],
                                  capture_output=True, text=True, timeout=10)

            if result.returncode == 0:
                self.logger.critical("Watchdog daemon stopped successfully")
                self.logger.critical("Hardware watchdog timer is now counting down to forced reboot")
                self.logger.critical(f"System will reset in ~{timeout} seconds")

                # Wait for the hardware watchdog to trigger
                # Log countdown messages
                for remaining in range(timeout, 0, -10):
                    self.logger.critical(f"Hardware reboot in approximately {remaining} seconds...")
                    time.sleep(10)

                self.logger.critical("Hardware watchdog should have triggered by now")
                return True
            else:
                self.logger.error(f"Failed to stop watchdog daemon: {result.stderr}")
                return False

        except subprocess.TimeoutExpired:
            self.logger.error("Timeout while stopping watchdog daemon")
            return False
        except Exception as e:
            self.logger.error(f"Error triggering watchdog reboot: {e}")
            return False

    def force_reboot(self):
        """Force an immediate system reboot using multiple fallback methods."""
        self.logger.critical("Executing immediate system reboot")

        reboot_methods = [
            # Method 1: Standard shutdown command
            (['shutdown', '-r', 'now'], "shutdown -r now"),

            # Method 2: Simple reboot command
            (['reboot'], "reboot"),

            # Method 3: Reboot with full path
            (['/sbin/reboot'], "/sbin/reboot"),

            # Method 4: Quick reboot (skip some cleanup)
            (['/sbin/reboot', '-q'], "reboot -q (quick)"),

            # Method 5: Reboot without syncing disks (aggressive)
            (['/sbin/reboot', '-n'], "reboot -n (no sync)"),

            # Method 6: Reboot with both flags (most aggressive)
            (['/sbin/reboot', '-q', '-n'], "reboot -qn (quick, no sync)"),

            # Method 7: Alternative shutdown syntax
            (['shutdown', '-r', '+0'], "shutdown -r +0"),

            # Method 8: Try halt and reboot
            (['/sbin/halt', '-r'], "halt -r"),
        ]

        for method, description in reboot_methods:
            try:
                self.logger.critical(f"Attempting reboot method: {description}")
                result = subprocess.run(method, capture_output=True, text=True, timeout=10)

                if result.returncode == 0:
                    self.logger.critical(f"Reboot command accepted: {description}")
                    # If we get here, command was accepted (system will reboot)
                    return
                else:
                    self.logger.error(f"Reboot method failed with return code {result.returncode}: {description}")
                    if result.stderr:
                        self.logger.error(f"Error output: {result.stderr}")

            except subprocess.TimeoutExpired:
                self.logger.error(f"Timeout executing reboot method: {description}")
            except FileNotFoundError:
                self.logger.error(f"Reboot command not found: {description}")
            except Exception as e:
                self.logger.error(f"Exception with reboot method {description}: {e}")

        # If all methods failed, try kernel-level reboot via sysctl
        try:
            self.logger.critical("All standard reboot methods failed, attempting sysctl kernel reboot")
            subprocess.run(['sysctl', 'kern.reboot', '1'], timeout=5)
        except Exception as e:
            self.logger.error(f"sysctl reboot also failed: {e}")

        # ABSOLUTE LAST RESORT: Hardware Watchdog Timer
        if self.config["watchdog"]["enabled"] and self.config["watchdog"]["use_as_last_resort"]:
            self.logger.critical("=" * 60)
            self.logger.critical("All software reboot methods exhausted")
            self.logger.critical("Attempting HARDWARE WATCHDOG as final fallback")
            self.logger.critical("=" * 60)

            if self.trigger_watchdog_reboot():
                # If watchdog was successfully triggered, we'll never reach here
                # (system will have rebooted)
                self.logger.critical("Watchdog triggered - awaiting hardware reboot")
                return
            else:
                self.logger.error("Hardware watchdog trigger failed")

        # If we get here, absolutely everything failed
        self.logger.critical("=" * 60)
        self.logger.critical("CRITICAL: ALL REBOOT METHODS FAILED")
        self.logger.critical("Including hardware watchdog (if enabled)")
        self.logger.critical("Manual intervention required - system needs hard power cycle")
        self.logger.critical("=" * 60)

        # Send final critical email
        self.send_email_alert(
            "CRITICAL: ALL REBOOT METHODS FAILED",
            "All automatic reboot methods have been exhausted, including:\n"
            "- Software reboot commands (8 methods)\n"
            "- Kernel-level reboot via sysctl\n"
            "- Hardware watchdog timer (if enabled)\n\n"
            "MANUAL INTERVENTION REQUIRED\n"
            "The system requires a hard power cycle to recover."
        )

    def resolve_smtp_server(self, hostname):
        """
        Resolve SMTP server hostname with fallback DNS servers.

        When local DNS (Unbound) is failing, normal resolution won't work.
        This method tries:
        1. Normal socket resolution
        2. External DNS servers (dig/drill) as fallback
        3. Hardcoded fallback IPs as last resort

        Returns the resolved IP address or a fallback IP.
        """
        email_config = self.config["email"]
        fallback_dns_servers = email_config.get("fallback_dns_servers", ["8.8.8.8", "1.1.1.1"])
        fallback_ips = email_config.get("smtp_server_fallback_ips", [])

        # Method 1: Try normal socket resolution
        try:
            result = socket.getaddrinfo(hostname, None, socket.AF_INET)
            if result:
                ip = result[0][4][0]
                self.logger.debug(f"Resolved {hostname} to {ip} via normal DNS")
                return ip
        except socket.gaierror as e:
            self.logger.warning(f"Normal DNS resolution failed for {hostname}: {e}")
        except Exception as e:
            self.logger.warning(f"Unexpected error in normal DNS resolution: {e}")

        # Method 2: Try using external DNS servers via dig or drill
        for dns_server in fallback_dns_servers:
            # Try drill first (common on FreeBSD/pfSense)
            for cmd in ["drill", "dig"]:
                try:
                    result = subprocess.run(
                        [cmd, f"@{dns_server}", hostname, "A", "+short"],
                        capture_output=True,
                        text=True,
                        timeout=5
                    )
                    if result.returncode == 0 and result.stdout.strip():
                        # Get first IP from output (may have multiple lines)
                        ip = result.stdout.strip().split('\n')[0]
                        # Validate it looks like an IP
                        if ip and ip[0].isdigit():
                            self.logger.info(f"Resolved {hostname} to {ip} via {cmd} @{dns_server}")
                            return ip
                except FileNotFoundError:
                    continue  # Command not available, try next
                except subprocess.TimeoutExpired:
                    self.logger.warning(f"{cmd} @{dns_server} timed out")
                except Exception as e:
                    self.logger.warning(f"{cmd} @{dns_server} failed: {e}")

        # Method 3: Use hardcoded fallback IPs
        if fallback_ips:
            self.logger.warning(f"All DNS resolution failed, using fallback IP: {fallback_ips[0]}")
            return fallback_ips[0]

        # No fallback available, return original hostname (will likely fail)
        self.logger.error(f"No fallback IPs configured, returning hostname: {hostname}")
        return hostname

    def send_email_alert(self, subject, message):
        """Send email alert via Gmail with fallback DNS support."""
        import ssl

        email_config = self.config["email"]
        smtp_hostname = email_config["smtp_server"]
        smtp_port = email_config["smtp_port"]

        # Build email message
        msg = MIMEMultipart()
        msg['From'] = email_config["sender_email"]
        msg['To'] = email_config["recipient_email"]
        msg['Subject'] = f"pfSense Alert: {subject}"

        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        wan_interface = self.config["network"]["wan_interface"]

        body = f"""
pfSense Monitor Alert

Time: {timestamp}
Host: {socket.gethostname()}
WAN Interface: {wan_interface}
Alert: {subject}

Details:
{message}

This is an automated message from the pfSense monitor.
        """

        msg.attach(MIMEText(body, 'plain'))
        text = msg.as_string()

        # Method 1: Try normal connection via hostname (when DNS works)
        try:
            self.logger.debug(f"Attempting email via hostname: {smtp_hostname}")
            server = smtplib.SMTP(smtp_hostname, smtp_port, timeout=10)
            server.starttls()
            server.login(email_config["sender_email"], email_config["sender_password"])
            server.sendmail(email_config["sender_email"], email_config["recipient_email"], text)
            server.quit()
            self.logger.info(f"Email alert sent successfully: {subject}")
            return
        except socket.gaierror as e:
            self.logger.warning(f"DNS resolution failed for {smtp_hostname}: {e}")
        except socket.timeout:
            self.logger.warning(f"Connection to {smtp_hostname} timed out")
        except Exception as e:
            # If it's not a DNS issue, log and still try fallback
            self.logger.warning(f"Normal SMTP connection failed: {e}")

        # Method 2: Try fallback DNS resolution and connect via IP
        try:
            smtp_ip = self.resolve_smtp_server(smtp_hostname)
            if smtp_ip == smtp_hostname:
                # Resolution failed and no fallback IPs available
                self.logger.error("Cannot resolve SMTP server and no fallback IPs configured")
                return

            self.logger.info(f"Attempting email via resolved IP: {smtp_ip}")
            server = smtplib.SMTP(smtp_ip, smtp_port, timeout=10)

            # When connecting via IP, certificate hostname won't match.
            # Use relaxed TLS since we resolved the IP ourselves from known DNS/fallback.
            # This is acceptable for alerting when local DNS is down.
            context = ssl.create_default_context()
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
            server.starttls(context=context)

            server.login(email_config["sender_email"], email_config["sender_password"])
            server.sendmail(email_config["sender_email"], email_config["recipient_email"], text)
            server.quit()
            self.logger.info(f"Email alert sent successfully via fallback: {subject}")
            return
        except Exception as e:
            self.logger.error(f"Failed to send email alert via fallback: {e}")

        self.logger.error(f"Failed to send email alert: {subject}")
    
    def run_monitor_cycle(self):
        """Main method to run all monitoring checks and take corrective actions."""
        self.logger.info("=" * 60)
        self.logger.info("Starting pfSense monitoring cycle")
        self.logger.info("=" * 60)

        # First, run unbound monitoring (critical service)
        self.run_unbound_monitor()

        # Then run WAN gateway monitoring
        self.logger.info("Starting WAN gateway connectivity check")
        
        # Initial connectivity test
        if self.test_gateway_connectivity():
            self.logger.info("WAN gateway connectivity is working normally")
            return True
        
        # First failure - try multiple times before taking action
        retry_attempts = self.config["network"]["retry_attempts"]
        retry_delay = self.config["network"]["retry_delay"]
        
        self.logger.warning("Initial gateway connectivity test failed, retrying...")
        
        for attempt in range(retry_attempts):
            time.sleep(retry_delay)
            if self.test_gateway_connectivity():
                self.logger.info(f"Gateway connectivity restored on retry attempt {attempt + 1}")
                return True
            
            self.logger.warning(f"Gateway retry attempt {attempt + 1} failed")
        
        # All retries failed - send alert and restart interface
        error_msg = f"WAN gateway connectivity lost after {retry_attempts + 1} attempts"
        self.logger.error(error_msg)
        self.send_email_alert("WAN Gateway Connectivity Lost", error_msg)
        
        # Attempt to restart WAN interface
        if self.restart_wan_interface():
            # Test connectivity after interface restart
            if self.test_gateway_connectivity():
                success_msg = "Gateway connectivity restored after WAN interface restart"
                self.logger.info(success_msg)
                self.send_email_alert("Gateway Connectivity Restored", success_msg)
                return True
            else:
                failure_msg = "Gateway connectivity still failed after WAN interface restart"
                self.logger.error(failure_msg)
                self.send_email_alert("WAN Interface Restart Failed", failure_msg)
        else:
            failure_msg = "Failed to restart WAN interface"
            self.logger.error(failure_msg)
            self.send_email_alert("WAN Interface Restart Failed", failure_msg)
        
        # Final option - reboot system
        self.reboot_system()
        return False

    def finalize_monitor_cycle(self):
        """Log the completion of the monitoring cycle."""
        self.logger.info("=" * 60)
        self.logger.info("pfSense monitoring cycle completed")
        self.logger.info("=" * 60)

def main():
    """Main entry point for the script."""
    try:
        # Check if running as root (required for network operations)
        if os.geteuid() != 0:
            print("This script must be run as root for network operations.")
            sys.exit(1)
        
        # Initialize and run monitor
        monitor = PfSenseMonitor()
        monitor.run_monitor_cycle()
        monitor.finalize_monitor_cycle()

    except KeyboardInterrupt:
        print("\nScript interrupted by user")
        sys.exit(0)
    except Exception as e:
        print(f"Unexpected error: {e}")
        sys.exit(1)

if __name__ == "__main__":
    main()