server.js

import { createServer } from 'http';
import { parse } from 'url';
import next from 'next';
import { WebSocketServer } from 'ws';
import { spawn } from 'child_process';
import { existsSync } from 'fs';
import { join, resolve } from 'path';
import stripAnsi from 'strip-ansi';
import { spawn as ptySpawn } from 'node-pty';
import { getSSHExecutionService } from './src/server/ssh-execution-service.js';
import { NEWT_COLORS_EXPORT } from './src/server/newt-colors.js';
import { getDatabase } from './src/server/database-prisma.js';
import { getAuthConfig, verifyToken, decodeToken } from './src/lib/auth.js';
import dotenv from 'dotenv';

// Dynamic import for auto sync init to avoid tsx caching issues
/** @type {any} */
let autoSyncModule = null;

// Load environment variables from .env file
dotenv.config();
// Fallback minimal global error handlers for Node runtime (avoid TS import)
function registerGlobalErrorHandlers() {
  if (registerGlobalErrorHandlers._registered) return;
  registerGlobalErrorHandlers._registered = true;
  process.on('uncaughtException', (err) => {
    console.error('uncaught_exception', err);
  });
  process.on('unhandledRejection', (reason) => {
    console.error('unhandled_rejection', reason);
  });
}
registerGlobalErrorHandlers._registered = false;

// JWT `exp` claim is a UNIX timestamp in seconds; Date.now() is milliseconds.
const JWT_EXP_MS_FACTOR = 1000;
// Maximum number of characters retained in the output buffer per execution.
const OUTPUT_BUFFER_MAX_LENGTH = 1000;
// Delay (ms) between backup completion and update start.
const BACKUP_UPDATE_DELAY_MS = 1000;

// Proxmox VMIDs are always purely numeric (typically 100-999999999).
const CONTAINER_ID_PATTERN = /^\d+$/;
// Proxmox storage identifiers only contain alphanumerics, underscore, hyphen, dot.
const STORAGE_ID_PATTERN = /^[a-zA-Z0-9_.-]+$/;
// Conservative hostname pattern (labels of alphanumerics/hyphens, dot-separated).
const HOSTNAME_PATTERN =
  /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/;
const MAX_CLONE_COUNT = 50;

/**
 * Validate the subset of a WebSocket "start" message that ends up interpolated
 * directly into shell commands executed on the Proxmox host (pct/qm/vzdump).
 * These values normally come from trusted UI dropdowns/DB records, but the
 * WebSocket endpoint itself has no other request-shape enforcement, so a
 * malformed or malicious client could otherwise inject arbitrary shell syntax.
 * @param {WebSocketMessage} message
 * @returns {string | null} an error message if invalid, otherwise null
 */
function validateExecutionParams(message) {
  const { containerId, storage, backupStorage, hostnames, containerType, cloneCount } = message;

  if (containerId !== undefined && !CONTAINER_ID_PATTERN.test(String(containerId))) {
    return `Invalid containerId: ${containerId}`;
  }
  if (storage !== undefined && !STORAGE_ID_PATTERN.test(String(storage))) {
    return `Invalid storage identifier: ${storage}`;
  }
  if (backupStorage !== undefined && !STORAGE_ID_PATTERN.test(String(backupStorage))) {
    return `Invalid backup storage identifier: ${backupStorage}`;
  }
  if (containerType !== undefined && containerType !== 'lxc' && containerType !== 'vm') {
    return `Invalid containerType: ${containerType}`;
  }
  if (cloneCount !== undefined) {
    const n = Number(cloneCount);
    if (!Number.isInteger(n) || n < 1 || n > MAX_CLONE_COUNT) {
      return `Invalid cloneCount: ${cloneCount}`;
    }
  }
  if (hostnames !== undefined) {
    if (!Array.isArray(hostnames) || !hostnames.every((h) => typeof h === 'string' && HOSTNAME_PATTERN.test(h))) {
      return 'Invalid hostnames';
    }
  }
  return null;
}

const dev = process.env.NODE_ENV !== 'production';
const hostname = '::';
const port = parseInt(process.env.PORT || '3000', 10);

const app = next({ dev, hostname, port });
// Register global handlers once at bootstrap
registerGlobalErrorHandlers();
const handle = app.getRequestHandler();

/**
 * Extract a cookie value from request cookie header.
 * @param {import('http').IncomingMessage} request
 * @param {string} cookieName
 * @returns {string | null}
 */
function getCookieValue(request, cookieName) {
  const rawCookieHeader = request.headers.cookie;
  if (!rawCookieHeader) return null;

  const cookies = rawCookieHeader.split(';');
  for (const cookie of cookies) {
    const [name, ...rest] = cookie.trim().split('=');
    if (name === cookieName) {
      return decodeURIComponent(rest.join('='));
    }
  }

  return null;
}

/**
 * Validate auth for WebSocket upgrades when authentication is enabled.
 * @param {import('http').IncomingMessage} request
 * @returns {boolean}
 */
function isWebSocketUpgradeAuthorized(request) {
  const authConfig = getAuthConfig();

  // Preserve current behavior if auth is disabled / not set up.
  if (!authConfig.enabled || !authConfig.setupCompleted) {
    return true;
  }

  const token = getCookieValue(request, 'auth-token');
  if (!token) {
    console.warn('ws_auth_rejected: missing auth-token cookie');
    return false;
  }

  const decoded = decodeToken(token);
  if (decoded && typeof decoded.exp === 'number' && decoded.exp * JWT_EXP_MS_FACTOR < Date.now()) {
    console.warn('ws_auth_rejected: token expired');
    return false;
  }

  const verified = verifyToken(token);
  if (!verified) {
    console.warn('ws_auth_rejected: token verification failed');
    return false;
  }

  return true;
}

// WebSocket handler for script execution
/**
 * @typedef {import('ws').WebSocket & {connectionTime?: number, clientIP?: string}} ExtendedWebSocket
 */

/**
 * @typedef {Object} Execution
 * @property {any} process
 * @property {ExtendedWebSocket} ws
 */

/**
 * @typedef {Object} ServerInfo
 * @property {string} name
 * @property {string} ip
 * @property {string} user
 * @property {string} password
 * @property {number} [id]
 * @property {string} [auth_type]
 * @property {string} [ssh_key_path]
 */

/**
 * @typedef {Object} ExecutionResult
 * @property {any} process
 * @property {Function} kill
 */

/**
 * @typedef {Object} WebSocketMessage
 * @property {string} action
 * @property {string} [scriptPath]
 * @property {string} [executionId]
 * @property {string} [input]
 * @property {string} [mode]
 * @property {ServerInfo} [server]
 * @property {boolean} [isUpdate]
 * @property {boolean} [isShell]
 * @property {boolean} [isBackup]
 * @property {boolean} [isClone]
 * @property {boolean} [executeInContainer]
 * @property {string} [containerId]
 * @property {string} [storage]
 * @property {string} [backupStorage]
 * @property {number} [cloneCount]
 * @property {string[]} [hostnames]
 * @property {'lxc'|'vm'} [containerType]
 * @property {Record<string, string|number|boolean>} [envVars]
 * @property {number} [cols]
 * @property {number} [rows]
 * @property {number} [installedScriptId]
 */

class ScriptExecutionHandler {
  /**
   * @param {import('http').Server} server
   */
  constructor(server) {
    // Create WebSocketServer without attaching to server
    // We'll handle upgrades manually to avoid interfering with Next.js HMR
    this.wss = new WebSocketServer({
      noServer: true
    });
    this.activeExecutions = new Map();
    this.db = getDatabase();
    this.setupWebSocket();
  }

  /**
   * Handle WebSocket upgrade for our endpoint
   * @param {import('http').IncomingMessage} request
   * @param {import('stream').Duplex} socket
   * @param {Buffer} head
   */
  handleUpgrade(request, socket, head) {
    this.wss.handleUpgrade(request, socket, head, (ws) => {
      this.wss.emit('connection', ws, request);
    });
  }

  /**
   * Parse Container ID from terminal output
   * @param {string} output - Terminal output to parse
   * @returns {string|null} - Container ID if found, null otherwise
   */
  parseContainerId(output) {
    // First, strip ANSI color codes to make pattern matching more reliable
    const cleanOutput = output.replace(/\x1b\[[0-9;]*m/g, '');

    // Look for various patterns that Proxmox scripts might use
    const patterns = [
      // Primary pattern - the exact format from the output
      /🆔\s+Container\s+ID:\s+(\d+)/i,

      // Standard patterns with flexible spacing
      /🆔\s*Container\s*ID:\s*(\d+)/i,
      /Container\s*ID:\s*(\d+)/i,
      /CT\s*ID:\s*(\d+)/i,
      /Container\s*(\d+)/i,

      // Alternative patterns
      /CT\s*(\d+)/i,
      /Container\s*created\s*with\s*ID\s*(\d+)/i,
      /Created\s*container\s*(\d+)/i,
      /Container\s*(\d+)\s*created/i,
      /ID:\s*(\d+)/i,

      // Patterns with different spacing and punctuation
      /Container\s*ID\s*:\s*(\d+)/i,
      /CT\s*ID\s*:\s*(\d+)/i,
      /Container\s*#\s*(\d+)/i,
      /CT\s*#\s*(\d+)/i,

      // Patterns that might appear in success messages
      /Successfully\s*created\s*container\s*(\d+)/i,
      /Container\s*(\d+)\s*is\s*ready/i,
      /Container\s*(\d+)\s*started/i,

      // Deliberately no generic bare-number fallback here: a stray 3-4 digit
      // number in unrelated output (port, percentage, byte count) would be
      // misdetected as the container ID and could overwrite a correct one.
    ];

    // Try patterns on both original and cleaned output
    const outputsToTry = [output, cleanOutput];

    for (const testOutput of outputsToTry) {
      for (const pattern of patterns) {
        const match = testOutput.match(pattern);
        if (match && match[1]) {
          const containerId = match[1];
          // Additional validation: container IDs are typically 3-4 digits
          if (containerId.length >= 3 && containerId.length <= 4) {
            return containerId;
          }
        }
      }
    }


    return null;
  }

  /**
   * Parse Web UI URL from terminal output
   * @param {string} output - Terminal output to parse
   * @returns {{ip: string, port: number}|null} - Object with ip and port if found, null otherwise
   */
  parseWebUIUrl(output) {
    // First, strip ANSI color codes to make pattern matching more reliable
    const cleanOutput = output.replace(/\x1b\[[0-9;]*m/g, '');

    // Look for URL patterns with any valid IP address (private or public)
    const patterns = [
      // HTTP/HTTPS URLs with IP and port
      /https?:\/\/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)/gi,
      // URLs without explicit port (assume default ports)
      /https?:\/\/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?:\/|$|\s)/gi,
      // URLs with trailing slash and port
      /https?:\/\/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)\//gi,
      // URLs with just IP and port (no protocol)
      /(?:^|\s)(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)(?:\s|$)/gi,

      // Deliberately no bare-IP-only fallback (no protocol, no port): a
      // netmask, DNS resolver IP, or unrelated dotted-quad printed by the
      // script would be misdetected as the web UI address.
    ];

    // Try patterns on both original and cleaned output
    const outputsToTry = [output, cleanOutput];

    for (const testOutput of outputsToTry) {
      for (const pattern of patterns) {
        const matches = [...testOutput.matchAll(pattern)];
        for (const match of matches) {
          if (match[1]) {
            const ip = match[1];
            const port = match[2] || (match[0].startsWith('https') ? '443' : '80');

            // Validate IP address format
            if (ip.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)) {
              return {
                ip: ip,
                port: parseInt(port, 10)
              };
            }
          }
        }
      }
    }

    return null;
  }

  /**
   * Create installation record
   * @param {string} scriptName - Name of the script
   * @param {string} scriptPath - Path to the script
   * @param {string} executionMode - 'local' or 'ssh'
   * @param {number|null} serverId - Server ID for SSH executions
   * @param {string|number|null} [containerId] - Container ID, when already known at start time
   * @returns {Promise<number|null>} - Installation record ID
   */
  async createInstallationRecord(scriptName, scriptPath, executionMode, serverId = null, containerId = null) {
    try {
      const result = await this.db.createInstalledScript({
        script_name: scriptName,
        script_path: scriptPath,
        container_id: containerId != null ? String(containerId) : undefined,
        server_id: serverId ?? undefined,
        execution_mode: executionMode,
        status: 'in_progress',
        output_log: ''
      });
      return Number(result.id);
    } catch (error) {
      console.error('Error creating installation record:', error);
      return null;
    }
  }

  /**
   * Update installation record
   * @param {number} installationId - Installation record ID
   * @param {Object} updateData - Data to update
   */
  async updateInstallationRecord(installationId, updateData) {
    try {
      await this.db.updateInstalledScript(installationId, updateData);
    } catch (error) {
      console.error('Error updating installation record:', error);
    }
  }

  setupWebSocket() {
    this.wss.on('connection', (ws, request) => {

      // Set connection metadata
      /** @type {ExtendedWebSocket} */ (ws).connectionTime = Date.now();
      /** @type {ExtendedWebSocket} */ (ws).clientIP = request.socket.remoteAddress || 'unknown';

      ws.on('message', (data) => {
        try {
          const rawMessage = data.toString();
          const message = JSON.parse(rawMessage);
          this.handleMessage(/** @type {ExtendedWebSocket} */(ws), message);
        } catch (error) {
          console.error('Error parsing WebSocket message:', error);
          this.sendMessage(ws, {
            type: 'error',
            data: 'Invalid message format',
            timestamp: Date.now()
          });
        }
      });

      ws.on('close', (code, reason) => {
        this.cleanupActiveExecutions(/** @type {ExtendedWebSocket} */(ws));
      });

      ws.on('error', (error) => {
        console.error('WebSocket error:', error);
        this.cleanupActiveExecutions(/** @type {ExtendedWebSocket} */(ws));
      });
    });
  }

  /**
   * Resolve full server from DB when client sends a redacted server payload.
   * Since v1.1.2 server API responses redact secrets, SSH execution must always
   * rehydrate credentials (password/key/passphrase/path) by server id.
   * @param {ServerInfo|null} server - Server from WebSocket message
   * @returns {Promise<ServerInfo|null>} Same server or full server from DB
   */
  async resolveServerForSSH(server) {
    if (!server?.id) return server;

    const serverId = Number(server.id);
    if (!Number.isFinite(serverId) || serverId <= 0) {
      return server;
    }

    const full = await this.db.getServerById(serverId);
    if (!full) {
      return server;
    }

    return /** @type {ServerInfo|null} */ ({ ...server, ...full });
  }

  /**
   * @param {ExtendedWebSocket} ws
   * @param {WebSocketMessage} message
   */
  async handleMessage(ws, message) {
    const { action, scriptPath, executionId, input, mode, server, isUpdate, isShell, isBackup, isClone, executeInContainer, containerId, storage, backupStorage, cloneCount, hostnames, containerType, envVars, cols, rows, installedScriptId } = message;

    switch (action) {
      case 'start': {
        const validationError = validateExecutionParams(message);
        if (validationError) {
          this.sendMessage(ws, {
            type: 'error',
            data: validationError,
            timestamp: Date.now()
          });
          break;
        }
        if (scriptPath && executionId) {
          let serverToUse = server;
          if (serverToUse?.id) {
            serverToUse = await this.resolveServerForSSH(serverToUse) ?? serverToUse;
          }
          const resolved = serverToUse ?? server;
          // Fall back to reasonably large defaults if the client didn't report
          // its actual terminal size — better than the old hardcoded 80x24/120x30
          // which made whiptail/ncurses dialogs draw off-center in real usage.
          const effectiveCols = typeof cols === 'number' && cols > 0 ? cols : 220;
          const effectiveRows = typeof rows === 'number' && rows > 0 ? rows : 50;
          if (isClone && containerId && storage && server && cloneCount && hostnames && containerType) {
            await this.startSSHCloneExecution(ws, containerId, executionId, storage, /** @type {ServerInfo} */(resolved), containerType, cloneCount, hostnames, effectiveCols, effectiveRows);
          } else if (isBackup && containerId && storage) {
            await this.startBackupExecution(ws, containerId, executionId, storage, mode, resolved, effectiveCols, effectiveRows);
          } else if (isUpdate && containerId) {
            await this.startUpdateExecution(ws, containerId, executionId, mode, resolved, backupStorage, envVars, installedScriptId, effectiveCols, effectiveRows);
          } else if (isShell && containerId) {
            await this.startShellExecution(ws, containerId, executionId, mode, resolved, containerType, effectiveCols, effectiveRows);
          } else if (executeInContainer && containerId) {
            await this.startInContainerScriptExecution(ws, scriptPath, executionId, mode, resolved, envVars, containerId, containerType ?? 'lxc', effectiveCols, effectiveRows);
          } else {
            await this.startScriptExecution(ws, scriptPath, executionId, mode, resolved, envVars, effectiveCols, effectiveRows);
          }
        } else {
          this.sendMessage(ws, {
            type: 'error',
            data: 'Missing scriptPath or executionId',
            timestamp: Date.now()
          });
        }
        break;
      }

      case 'stop':
        if (executionId) {
          this.stopScriptExecution(executionId);
        }
        break;

      case 'input':
        if (executionId && input !== undefined) {
          this.sendInputToProcess(executionId, input);
        }
        break;

      case 'resize':
        if (executionId && typeof cols === 'number' && typeof rows === 'number') {
          this.resizeProcess(executionId, cols, rows);
        }
        break;

      default:
        this.sendMessage(ws, {
          type: 'error',
          data: 'Unknown action',
          timestamp: Date.now()
        });
    }
  }

  /**
   * @param {ExtendedWebSocket} ws
   * @param {string} scriptPath
   * @param {string} executionId
   * @param {string} mode
   * @param {ServerInfo|null} server
   * @param {Object} [envVars] - Optional environment variables to pass to the script
   */
  async startScriptExecution(ws, scriptPath, executionId, mode = 'local', server = null, envVars = {}, cols = 220, rows = 50) {
    /** @type {number|null} */
    let installationId = null;

    try {

      // Check if execution is already running
      if (this.activeExecutions.has(executionId)) {
        this.sendMessage(ws, {
          type: 'error',
          data: 'Script execution already running',
          timestamp: Date.now()
        });
        return;
      }

      // Extract script name from path
      const scriptName = scriptPath.split('/').pop() ?? scriptPath.split('\\').pop() ?? 'Unknown Script';

      // Create installation record
      const serverId = server ? (server.id ?? null) : null;
      installationId = await this.createInstallationRecord(scriptName, scriptPath, mode, serverId);

      if (!installationId) {
        console.error('Failed to create installation record');
      }

      // Handle SSH execution
      if (mode === 'ssh' && server) {
        await this.startSSHScriptExecution(ws, scriptPath, executionId, server, installationId, envVars, cols, rows);
        return;
      }

      if (mode === 'ssh' && !server) {
        // SSH mode requested but no server provided, falling back to local execution
      }

      // Basic validation for local execution
      const scriptsDir = join(process.cwd(), 'scripts');
      const resolvedPath = resolve(scriptPath);

      if (!resolvedPath.startsWith(resolve(scriptsDir))) {
        this.sendMessage(ws, {
          type: 'error',
          data: 'Script path is not within the allowed scripts directory',
          timestamp: Date.now()
        });

        // Update installation record with failure
        if (installationId) {
          await this.updateInstallationRecord(installationId, { status: 'failed' });
        }
        return;
      }

      // Format environment variables for local execution
      // Convert envVars object to environment variables
      const envWithVars = {
        ...process.env,
        TERM: 'xterm-256color', // Enable proper terminal support
        FORCE_ANSI: 'true', // Allow ANSI codes for proper display
        COLUMNS: '80', // Set terminal width
        LINES: '24' // Set terminal height
      };

      // Add envVars to environment
      if (envVars && typeof envVars === 'object') {
        for (const [key, value] of Object.entries(envVars)) {
          /** @type {Record<string, string>} */
          const envRecord = envWithVars;
          envRecord[key] = String(value);
        }
      }

      // Start script execution with pty for proper TTY support
      const childProcess = ptySpawn('bash', [resolvedPath], {
        cwd: scriptsDir,
        name: 'xterm-256color',
        cols,
        rows,
        env: envWithVars
      });

      // pty handles encoding automatically

      // Store the execution with installation ID
      this.activeExecutions.set(executionId, {
        process: childProcess,
        ws,
        installationId,
        outputBuffer: ''
      });

      // Send start message
      this.sendMessage(ws, {
        type: 'start',
        data: `Starting execution of ${scriptPath}`,
        timestamp: Date.now()
      });

      // Handle pty data (both stdout and stderr combined)
      childProcess.onData(async (data) => {
        const output = data.toString();

        // Store output in buffer for logging
        const execution = this.activeExecutions.get(executionId);
        if (execution) {
          execution.outputBuffer += output;
          // Trim buffer to maximum allowed length to avoid memory issues
          if (execution.outputBuffer.length > OUTPUT_BUFFER_MAX_LENGTH) {
            execution.outputBuffer = execution.outputBuffer.slice(-OUTPUT_BUFFER_MAX_LENGTH);
          }
        }

        // Parse for Container ID
        const containerId = this.parseContainerId(output);
        if (containerId && installationId) {
          await this.updateInstallationRecord(installationId, { container_id: containerId });
        }

        // Parse for Web UI URL
        const webUIUrl = this.parseWebUIUrl(output);
        if (webUIUrl && installationId) {
          const { ip, port } = webUIUrl;
          if (ip && port) {
            await this.updateInstallationRecord(installationId, {
              web_ui_ip: ip,
              web_ui_port: port
            });
          }
        }

        this.sendMessage(ws, {
          type: 'output',
          data: output,
          timestamp: Date.now()
        });
      });

      // Handle process exit
      childProcess.onExit((e) => {
        const execution = this.activeExecutions.get(executionId);
        const isSuccess = e.exitCode === 0;

        // Update installation record with final status and output
        if (installationId && execution) {
          this.updateInstallationRecord(installationId, {
            status: isSuccess ? 'success' : 'failed',
            output_log: execution.outputBuffer
          });
        }

        this.sendMessage(ws, {
          type: 'end',
          data: `Script execution finished with code: ${e.exitCode}, signal: ${e.signal}`,
          timestamp: Date.now()
        });

        // Clean up
        this.activeExecutions.delete(executionId);
      });

    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `Failed to start script: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });

      // Update installation record with failure
      if (installationId) {
        await this.updateInstallationRecord(installationId, { status: 'failed' });
      }
    }
  }


  /**
   * Execute a script INSIDE a container via `pct exec` (LXC) or `qm guest exec` (VM).
   * For SSH mode the scripts folder is already transferred to /tmp/scripts on the PVE host by
   * startSSHScriptExecution; we re-use that mechanism and then run the script inside the CT.
   * For local mode we pipe the script through `pct exec`.
   *
   * @param {ExtendedWebSocket} ws
   * @param {string} scriptPath
   * @param {string} executionId
   * @param {string} mode
   * @param {ServerInfo|null} server
   * @param {Object} envVars
   * @param {string} containerId
   * @param {'lxc'|'vm'} containerType
   */
  async startInContainerScriptExecution(ws, scriptPath, executionId, mode = 'local', server = null, envVars = {}, containerId, containerType = 'lxc', cols = 220, rows = 50) {
    /** @type {number|null} */
    let installationId = null;
    try {
      if (this.activeExecutions.has(executionId)) {
        this.sendMessage(ws, { type: 'error', data: 'Script execution already running', timestamp: Date.now() });
        return;
      }

      // Guard against path traversal — every script must resolve inside the
      // scripts directory, regardless of execution mode (local or SSH).
      const guardScriptsDir = join(process.cwd(), 'scripts');
      const resolvedScriptPath = resolve(scriptPath);
      if (!resolvedScriptPath.startsWith(resolve(guardScriptsDir))) {
        this.sendMessage(ws, { type: 'error', data: 'Script path outside scripts directory', timestamp: Date.now() });
        return;
      }

      const scriptName = scriptPath.split('/').pop() ?? scriptPath.split('\\').pop() ?? 'Unknown Script';
      const serverId = server ? (server.id ?? null) : null;
      installationId = await this.createInstallationRecord(scriptName, scriptPath, mode, serverId, containerId ?? null);

      // Build env-var export prefix
      const envExports = Object.entries(envVars ?? {})
        .filter(([k]) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(k))
        .map(([k, v]) => `export ${k}=${JSON.stringify(String(v))}`)
        .join('; ');
      const envPrefix = `${NEWT_COLORS_EXPORT}; ${envExports ? `${envExports}; ` : ''}`;

      if (mode === 'ssh' && server) {
        // Transfer scripts folder to PVE host, then exec inside container
        const sshService = getSSHExecutionService();
        this.sendMessage(ws, { type: 'start', data: `Connecting to ${server.ip}…`, timestamp: Date.now() });

        const relScript = scriptPath.replace(/^scripts[/\\]/, '');
        const remoteScript = `/tmp/scripts/${relScript}`;

        // Transfer scripts folder silently — suppress verbose rsync file listing
        this.sendMessage(ws, { type: 'output', data: 'Syncing scripts…\r\n', timestamp: Date.now() });
        await sshService.transferScriptsFolder(server,
          () => { }, // suppress rsync stdout (verbose file listing)
          (/** @type {string} */ err) => {
            // Ignore harmless SSH host-key / known-hosts notices from rsync stderr
            if (!err.includes('Warning:') && !err.includes('Permanently added')) {
              this.sendMessage(ws, { type: 'error', data: err, timestamp: Date.now() });
            }
          }
        );

        let inContainerCmd;

        if (containerType === 'lxc') {
          // For LXC we must copy the script from host (/tmp/scripts/...) into the container first.
          // Otherwise pct exec tries to run a host path that does not exist inside the CT.
          const remoteTarget = `/tmp/${scriptName}`;
          const pushCmd = `pct push ${containerId} ${remoteScript} ${remoteTarget}`;

          // Run pct push silently (no terminal output needed for this bookkeeping step)
          await new Promise((resolve, reject) => {
            sshService.executeCommand(server, pushCmd,
              () => { }, // suppress push output
              () => { }, // suppress push stderr
              (/** @type {number} */ code) => {
                if (code === 0) resolve(true);
                else reject(new Error(`pct push failed with exit code ${code}`));
              },
            );
          });

          inContainerCmd = `pct exec ${containerId} -- bash -c "chmod +x ${remoteTarget}; ${envPrefix}bash ${remoteTarget}"`;
        } else {
          // VM execution currently relies on guest exec and assumes script path exists in guest context.
          inContainerCmd = `qm guest exec ${containerId} -- bash -c "${envPrefix}bash ${remoteScript}"`;
        }

        this.activeExecutions.set(executionId, { process: null, ws, installationId, outputBuffer: '' });

        // executeCommand resolves immediately with the PTY process — store it so
        // interactive input (y/n prompts etc.) can be forwarded via sendInputToProcess.
        const execResult = await sshService.executeCommand(server, inContainerCmd,
          (/** @type {string} */ data) => {
            const execution = this.activeExecutions.get(executionId);
            if (execution) execution.outputBuffer += data;
            this.sendMessage(ws, { type: 'output', data, timestamp: Date.now() });
          },
          (/** @type {string} */ err) => {
            this.sendMessage(ws, { type: 'error', data: err, timestamp: Date.now() });
          },
          async (/** @type {number} */ exitCode) => {
            const execution = this.activeExecutions.get(executionId);
            if (installationId && execution) {
              await this.updateInstallationRecord(installationId, {
                status: exitCode === 0 ? 'success' : 'failed',
                output_log: execution.outputBuffer
              });
            }
            this.sendMessage(ws, { type: 'end', data: `Finished with code: ${exitCode}`, timestamp: Date.now() });
            this.activeExecutions.delete(executionId);
          },
          cols,
          rows
        );

        // Attach the real PTY process so keyboard input can reach interactive prompts
        const storedExec = this.activeExecutions.get(executionId);
        if (storedExec && execResult && /** @type {any} */(execResult).process) {
          storedExec.process = /** @type {any} */(execResult).process;
        }
        return;
      }

      // Local mode: pipe the script content into `pct exec <ctid> -- bash`
      const scriptsDir = join(process.cwd(), 'scripts');
      const resolvedPath = resolve(scriptPath);
      if (!resolvedPath.startsWith(resolve(scriptsDir))) {
        this.sendMessage(ws, { type: 'error', data: 'Script path outside scripts directory', timestamp: Date.now() });
        if (installationId) await this.updateInstallationRecord(installationId, { status: 'failed' });
        return;
      }

      const bashCmd = `${envPrefix}bash ${resolvedPath}`;
      const args = containerType === 'lxc'
        ? ['exec', containerId, '--', 'bash', '-c', bashCmd]
        : ['guest', 'exec', containerId, '--', 'bash', '-c', bashCmd];
      const cmd = containerType === 'lxc' ? 'pct' : 'qm';

      const childProcess = ptySpawn(cmd, args, {
        cwd: scriptsDir,
        name: 'xterm-256color',
        cols,
        rows,
        env: { ...process.env, TERM: 'xterm-256color' }
      });

      this.activeExecutions.set(executionId, { process: childProcess, ws, installationId, outputBuffer: '' });
      this.sendMessage(ws, { type: 'start', data: `Executing inside container ${containerId}…`, timestamp: Date.now() });

      childProcess.onData((data) => {
        const execution = this.activeExecutions.get(executionId);
        if (execution) execution.outputBuffer += data;
        this.sendMessage(ws, { type: 'output', data, timestamp: Date.now() });
      });

      childProcess.onExit(async (e) => {
        const execution = this.activeExecutions.get(executionId);
        if (installationId && execution) {
          await this.updateInstallationRecord(installationId, {
            status: e.exitCode === 0 ? 'success' : 'failed',
            output_log: execution.outputBuffer
          });
        }
        this.sendMessage(ws, { type: 'end', data: `Script finished with code: ${e.exitCode}`, timestamp: Date.now() });
        this.activeExecutions.delete(executionId);
      });

    } catch (error) {
      this.sendMessage(ws, { type: 'error', data: `Failed to start in-container script: ${error instanceof Error ? error.message : String(error)}`, timestamp: Date.now() });
      if (installationId) await this.updateInstallationRecord(installationId, { status: 'failed' });
    }
  }

  /**
   * Start SSH script execution
   * @param {ExtendedWebSocket} ws
   * @param {string} scriptPath
   * @param {string} executionId
   * @param {ServerInfo} server
   * @param {number|null} installationId
   * @param {Object} [envVars] - Optional environment variables to pass to the script
   */
  async startSSHScriptExecution(ws, scriptPath, executionId, server, installationId = null, envVars = {}, cols = 220, rows = 50) {
    const sshService = getSSHExecutionService();

    // Send start message
    this.sendMessage(ws, {
      type: 'start',
      data: `Starting SSH execution of ${scriptPath} on ${server.name} (${server.ip})`,
      timestamp: Date.now()
    });

    try {
      const execution = /** @type {ExecutionResult} */ (await sshService.executeScript(
        server,
        scriptPath,
        /** @param {string} data */ async (data) => {
          // Store output in buffer for logging
          const exec = this.activeExecutions.get(executionId);
          if (exec) {
            exec.outputBuffer += data;
            // Trim buffer to maximum allowed length to avoid memory issues
            if (exec.outputBuffer.length > OUTPUT_BUFFER_MAX_LENGTH) {
              exec.outputBuffer = exec.outputBuffer.slice(-OUTPUT_BUFFER_MAX_LENGTH);
            }
          }

          // Parse for Container ID
          const containerId = this.parseContainerId(data);
          if (containerId && installationId) {
            await this.updateInstallationRecord(installationId, { container_id: containerId });
          }

          // Parse for Web UI URL
          const webUIUrl = this.parseWebUIUrl(data);
          if (webUIUrl && installationId) {
            const { ip, port } = webUIUrl;
            if (ip && port) {
              await this.updateInstallationRecord(installationId, {
                web_ui_ip: ip,
                web_ui_port: port
              });
            }
          }

          // Handle data output
          this.sendMessage(ws, {
            type: 'output',
            data: data,
            timestamp: Date.now()
          });
        },
        /** @param {string} error */(error) => {
          // Store error in buffer for logging
          const exec = this.activeExecutions.get(executionId);
          if (exec) {
            exec.outputBuffer += error;
            // Trim buffer to maximum allowed length to avoid memory issues
            if (exec.outputBuffer.length > OUTPUT_BUFFER_MAX_LENGTH) {
              exec.outputBuffer = exec.outputBuffer.slice(-OUTPUT_BUFFER_MAX_LENGTH);
            }
          }

          // Handle errors
          this.sendMessage(ws, {
            type: 'error',
            data: error,
            timestamp: Date.now()
          });
        },
        /** @param {number} code */ async (code) => {
          const exec = this.activeExecutions.get(executionId);
          const isSuccess = code === 0;

          // Update installation record with final status and output
          if (installationId && exec) {
            await this.updateInstallationRecord(installationId, {
              status: isSuccess ? 'success' : 'failed',
              output_log: exec.outputBuffer
            });
          }

          // Handle process exit
          this.sendMessage(ws, {
            type: 'end',
            data: `SSH script execution finished with code: ${code}`,
            timestamp: Date.now()
          });

          // Clean up
          this.activeExecutions.delete(executionId);
        },
        envVars,
        cols,
        rows
      ));

      // Store the execution with installation ID
      this.activeExecutions.set(executionId, {
        process: execution.process,
        ws,
        installationId,
        outputBuffer: ''
      });

    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `Failed to start SSH execution: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });

      // Update installation record with failure
      if (installationId) {
        await this.updateInstallationRecord(installationId, { status: 'failed' });
      }
    }
  }

  /**
   * @param {string} executionId
   */
  stopScriptExecution(executionId) {
    const execution = this.activeExecutions.get(executionId);
    if (execution) {
      execution.process?.kill('SIGTERM');
      this.activeExecutions.delete(executionId);

      if (execution.installationId) {
        this.updateInstallationRecord(execution.installationId, {
          status: 'failed',
          output_log: execution.outputBuffer
        }).catch((error) => {
          console.error('Error updating installation record after manual stop:', error);
        });
      }

      this.sendMessage(execution.ws, {
        type: 'end',
        data: 'Script execution stopped by user',
        timestamp: Date.now()
      });
    }
  }

  /**
   * @param {string} executionId
   * @param {string} input
   */
  sendInputToProcess(executionId, input) {
    const execution = this.activeExecutions.get(executionId);
    if (execution?.process?.write) {
      execution.process.write(input);
    }
  }

  /**
   * @param {string} executionId
   * @param {number} cols
   * @param {number} rows
   */
  resizeProcess(executionId, cols, rows) {
    const execution = this.activeExecutions.get(executionId);
    if (!execution?.process) return;

    if (typeof execution.process.resize === 'function') {
      execution.process.resize(cols, rows);
    }
  }

  /**
   * @param {ExtendedWebSocket} ws
   * @param {any} message
   */
  sendMessage(ws, message) {
    if (ws.readyState === 1) { // WebSocket.OPEN
      ws.send(JSON.stringify(message));
    }
  }

  /**
   * @param {ExtendedWebSocket} ws
   */
  cleanupActiveExecutions(ws) {
    for (const [executionId, execution] of this.activeExecutions.entries()) {
      if (execution.ws === ws) {
        execution.process?.kill('SIGTERM');
        // Mark the installation record as failed now, since the exit handler
        // won't find this entry anymore once it's removed below and would
        // otherwise leave the record stuck at 'in_progress' forever.
        if (execution.installationId) {
          this.updateInstallationRecord(execution.installationId, {
            status: 'failed',
            output_log: execution.outputBuffer
          }).catch((error) => {
            console.error('Error updating installation record during cleanup:', error);
          });
        }
        this.activeExecutions.delete(executionId);
      }
    }
  }

  /**
   * Start backup execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {string} storage
   * @param {string} mode
   * @param {ServerInfo|null} server
   */
  async startBackupExecution(ws, containerId, executionId, storage, mode = 'local', server = null, cols = 220, rows = 50) {
    try {
      // Send start message
      this.sendMessage(ws, {
        type: 'start',
        data: `Starting backup for container ${containerId} to storage ${storage}...`,
        timestamp: Date.now()
      });

      if (mode === 'ssh' && server) {
        await this.startSSHBackupExecution(ws, containerId, executionId, storage, server, undefined, cols, rows);
      } else {
        this.sendMessage(ws, {
          type: 'error',
          data: 'Backup is only supported via SSH',
          timestamp: Date.now()
        });
      }
    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `Failed to start backup: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });
    }
  }

  /**
   * Start SSH backup execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {string} storage
   * @param {ServerInfo} server
   * @param {Function} [onComplete] - Optional callback when backup completes
   */
  startSSHBackupExecution(ws, containerId, executionId, storage, server, onComplete = undefined, cols = 220, rows = 50) {
    const sshService = getSSHExecutionService();

    return new Promise((resolve, reject) => {
      try {
        const backupCommand = `vzdump ${containerId} --storage ${storage} --mode snapshot`;

        // Wrap the onExit callback to resolve our promise
        let promiseResolved = false;

        sshService.executeCommand(
          server,
          backupCommand,
          /** @param {string} data */
          (data) => {
            this.sendMessage(ws, {
              type: 'output',
              data: data,
              timestamp: Date.now()
            });
          },
          /** @param {string} error */
          (error) => {
            this.sendMessage(ws, {
              type: 'error',
              data: error,
              timestamp: Date.now()
            });
          },
          /** @param {number} code */
          (code) => {
            // Don't send 'end' message here if this is part of a backup+update flow
            // The update flow will handle completion messages
            const success = code === 0;

            if (!success) {
              this.sendMessage(ws, {
                type: 'error',
                data: `Backup failed with exit code: ${code}`,
                timestamp: Date.now()
              });
            }

            // Send a completion message (but not 'end' type to avoid stopping terminal)
            this.sendMessage(ws, {
              type: 'output',
              data: `\n[Backup ${success ? 'completed' : 'failed'} with exit code: ${code}]\n`,
              timestamp: Date.now()
            });

            if (onComplete) onComplete(success);

            // Resolve the promise when backup completes
            // Use setImmediate to ensure resolution happens in the right execution context
            if (!promiseResolved) {
              promiseResolved = true;
              const result = { success, code };

              // Use setImmediate to ensure promise resolution happens in the next tick
              // This ensures the await in startUpdateExecution can properly resume
              setImmediate(() => {
                try {
                  resolve(result);
                } catch (resolveError) {
                  console.error('Error resolving backup promise:', resolveError);
                  reject(resolveError);
                }
              });
            }

            this.activeExecutions.delete(executionId);
          },
          cols,
          rows
        ).then((execution) => {
          // Store the execution
          this.activeExecutions.set(executionId, {
            process: /** @type {any} */ (execution).process,
            ws
          });
          // Note: Don't resolve here - wait for onExit callback
        }).catch((error) => {
          console.error('Error starting backup execution:', error);
          this.sendMessage(ws, {
            type: 'error',
            data: `SSH backup execution failed: ${error instanceof Error ? error.message : String(error)}`,
            timestamp: Date.now()
          });
          if (onComplete) onComplete(false);
          if (!promiseResolved) {
            promiseResolved = true;
            reject(error);
          }
        });

      } catch (error) {
        console.error('Error in startSSHBackupExecution:', error);
        this.sendMessage(ws, {
          type: 'error',
          data: `SSH backup execution failed: ${error instanceof Error ? error.message : String(error)}`,
          timestamp: Date.now()
        });
        if (onComplete) onComplete(false);
        reject(error);
      }
    });
  }

  /**
   * Start SSH clone execution
   * Gets next IDs sequentially: get next ID → clone → get next ID → clone, etc.
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {string} storage
   * @param {ServerInfo} server
   * @param {'lxc'|'vm'} containerType
   * @param {number} cloneCount
   * @param {string[]} hostnames
   */
  async startSSHCloneExecution(ws, containerId, executionId, storage, server, containerType, cloneCount, hostnames, cols = 220, rows = 50) {
    const sshService = getSSHExecutionService();

    this.sendMessage(ws, {
      type: 'start',
      data: `Starting clone operation: Creating ${cloneCount} clone(s) of ${containerType.toUpperCase()} ${containerId}...`,
      timestamp: Date.now()
    });

    try {
      // Step 1: Stop source container/VM
      this.sendMessage(ws, {
        type: 'output',
        data: `\n[Step 1/${4 + cloneCount}] Stopping source ${containerType.toUpperCase()} ${containerId}...\n`,
        timestamp: Date.now()
      });

      const stopCommand = containerType === 'lxc' ? `pct stop ${containerId}` : `qm stop ${containerId}`;
      await new Promise(/** @type {(resolve: (value?: void) => void, reject: (error?: any) => void) => void} */((resolve, reject) => {
        sshService.executeCommand(
          server,
          stopCommand,
          /** @param {string} data */
          (data) => {
            this.sendMessage(ws, {
              type: 'output',
              data: data,
              timestamp: Date.now()
            });
          },
          /** @param {string} error */
          (error) => {
            this.sendMessage(ws, {
              type: 'error',
              data: error,
              timestamp: Date.now()
            });
          },
          /** @param {number} code */
          (code) => {
            if (code === 0) {
              this.sendMessage(ws, {
                type: 'output',
                data: `\n[Step 1/${4 + cloneCount}] Source ${containerType.toUpperCase()} stopped successfully.\n`,
                timestamp: Date.now()
              });
              resolve();
            } else {
              // Continue even if stop fails (might already be stopped)
              this.sendMessage(ws, {
                type: 'output',
                data: `\n[Step 1/${4 + cloneCount}] Stop command completed with exit code ${code} (container may already be stopped).\n`,
                timestamp: Date.now()
              });
              resolve();
            }
          }
        );
      }));

      // Step 2: Clone for each clone count (get next ID sequentially before each clone)
      const clonedIds = [];
      for (let i = 0; i < cloneCount; i++) {
        const cloneNumber = i + 1;
        const hostname = hostnames[i];

        // Get next ID for this clone
        this.sendMessage(ws, {
          type: 'output',
          data: `\n[Step ${2 + i}/${4 + cloneCount}] Getting next available ID for clone ${cloneNumber}...\n`,
          timestamp: Date.now()
        });

        let nextId = '';
        try {
          let output = '';
          await new Promise(/** @type {(resolve: (value?: void) => void, reject: (error?: any) => void) => void} */((resolve, reject) => {
            sshService.executeCommand(
              server,
              'pvesh get /cluster/nextid',
              /** @param {string} data */
              (data) => {
                output += data;
              },
              /** @param {string} error */
              (error) => {
                reject(new Error(`Failed to get next ID: ${error}`));
              },
              /** @param {number} exitCode */
              (exitCode) => {
                if (exitCode === 0) {
                  resolve();
                } else {
                  reject(new Error(`pvesh command failed with exit code ${exitCode}`));
                }
              }
            );
          }));

          nextId = output.trim();
          if (!nextId || !/^\d+$/.test(nextId)) {
            throw new Error('Invalid next ID received');
          }

          this.sendMessage(ws, {
            type: 'output',
            data: `\n[Step ${2 + i}/${4 + cloneCount}] Got next ID: ${nextId}\n`,
            timestamp: Date.now()
          });
        } catch (error) {
          this.sendMessage(ws, {
            type: 'error',
            data: `\n[Step ${2 + i}/${4 + cloneCount}] Failed to get next ID: ${error instanceof Error ? error.message : String(error)}\n`,
            timestamp: Date.now()
          });
          throw error;
        }

        clonedIds.push(nextId);

        // Clone the container/VM
        this.sendMessage(ws, {
          type: 'output',
          data: `\n[Step ${2 + i}/${4 + cloneCount}] Cloning ${containerType.toUpperCase()} ${containerId} to ${nextId} with hostname ${hostname}...\n`,
          timestamp: Date.now()
        });

        const cloneCommand = containerType === 'lxc'
          ? `pct clone ${containerId} ${nextId} --hostname ${hostname} --storage ${storage}`
          : `qm clone ${containerId} ${nextId} --name ${hostname} --storage ${storage}`;

        await new Promise(/** @type {(resolve: (value?: void) => void, reject: (error?: any) => void) => void} */((resolve, reject) => {
          sshService.executeCommand(
            server,
            cloneCommand,
            /** @param {string} data */
            (data) => {
              this.sendMessage(ws, {
                type: 'output',
                data: data,
                timestamp: Date.now()
              });
            },
            /** @param {string} error */
            (error) => {
              this.sendMessage(ws, {
                type: 'error',
                data: error,
                timestamp: Date.now()
              });
            },
            /** @param {number} code */
            (code) => {
              if (code === 0) {
                this.sendMessage(ws, {
                  type: 'output',
                  data: `\n[Step ${2 + i}/${4 + cloneCount}] Clone ${cloneNumber} created successfully.\n`,
                  timestamp: Date.now()
                });
                resolve();
              } else {
                this.sendMessage(ws, {
                  type: 'error',
                  data: `\nClone ${cloneNumber} failed with exit code: ${code}\n`,
                  timestamp: Date.now()
                });
                reject(new Error(`Clone ${cloneNumber} failed with exit code ${code}`));
              }
            }
          );
        }));
      }

      // Step 3: Start source container/VM
      this.sendMessage(ws, {
        type: 'output',
        data: `\n[Step ${2 + cloneCount + 1}/${4 + cloneCount}] Starting source ${containerType.toUpperCase()} ${containerId}...\n`,
        timestamp: Date.now()
      });

      const startSourceCommand = containerType === 'lxc' ? `pct start ${containerId}` : `qm start ${containerId}`;
      await new Promise(/** @type {(resolve: (value?: void) => void, reject: (error?: any) => void) => void} */((resolve) => {
        sshService.executeCommand(
          server,
          startSourceCommand,
          /** @param {string} data */
          (data) => {
            this.sendMessage(ws, {
              type: 'output',
              data: data,
              timestamp: Date.now()
            });
          },
          /** @param {string} error */
          (error) => {
            this.sendMessage(ws, {
              type: 'error',
              data: error,
              timestamp: Date.now()
            });
          },
          /** @param {number} code */
          (code) => {
            if (code === 0) {
              this.sendMessage(ws, {
                type: 'output',
                data: `\n[Step ${2 + cloneCount + 1}/${4 + cloneCount}] Source ${containerType.toUpperCase()} started successfully.\n`,
                timestamp: Date.now()
              });
            } else {
              this.sendMessage(ws, {
                type: 'output',
                data: `\n[Step ${2 + cloneCount + 1}/${4 + cloneCount}] Start command completed with exit code ${code}.\n`,
                timestamp: Date.now()
              });
            }
            resolve();
          }
        );
      }));

      // Step 4: Start target containers/VMs
      this.sendMessage(ws, {
        type: 'output',
        data: `\n[Step ${2 + cloneCount + 2}/${4 + cloneCount}] Starting cloned ${containerType.toUpperCase()}(s)...\n`,
        timestamp: Date.now()
      });

      for (let i = 0; i < cloneCount; i++) {
        const cloneNumber = i + 1;
        const nextId = clonedIds[i];

        const startTargetCommand = containerType === 'lxc' ? `pct start ${nextId}` : `qm start ${nextId}`;
        await new Promise(/** @type {(resolve: (value?: void) => void, reject: (error?: any) => void) => void} */((resolve) => {
          sshService.executeCommand(
            server,
            startTargetCommand,
            /** @param {string} data */
            (data) => {
              this.sendMessage(ws, {
                type: 'output',
                data: data,
                timestamp: Date.now()
              });
            },
            /** @param {string} error */
            (error) => {
              this.sendMessage(ws, {
                type: 'error',
                data: error,
                timestamp: Date.now()
              });
            },
            /** @param {number} code */
            (code) => {
              if (code === 0) {
                this.sendMessage(ws, {
                  type: 'output',
                  data: `\nClone ${cloneNumber} (ID: ${nextId}) started successfully.\n`,
                  timestamp: Date.now()
                });
              } else {
                this.sendMessage(ws, {
                  type: 'output',
                  data: `\nClone ${cloneNumber} (ID: ${nextId}) start completed with exit code ${code}.\n`,
                  timestamp: Date.now()
                });
              }
              resolve();
            }
          );
        }));
      }

      // Step 5: Add to database
      this.sendMessage(ws, {
        type: 'output',
        data: `\n[Step ${2 + cloneCount + 3}/${4 + cloneCount}] Adding cloned ${containerType.toUpperCase()}(s) to database...\n`,
        timestamp: Date.now()
      });

      for (let i = 0; i < cloneCount; i++) {
        const nextId = clonedIds[i];
        const hostname = hostnames[i];

        try {
          // Read config file to get hostname/name (node-specific path)
          const nodeName = server.name;
          const configPath = containerType === 'lxc'
            ? `/etc/pve/nodes/${nodeName}/lxc/${nextId}.conf`
            : `/etc/pve/nodes/${nodeName}/qemu-server/${nextId}.conf`;

          let configContent = '';
          await new Promise(/** @type {(resolve: (value?: void) => void) => void} */((resolve) => {
            sshService.executeCommand(
              server,
              `cat "${configPath}" 2>/dev/null || echo ""`,
              /** @param {string} data */
              (data) => {
                configContent += data;
              },
              () => resolve(),
              () => resolve()
            );
          }));

          // Parse config for hostname/name
          let finalHostname = hostname;
          if (configContent.trim()) {
            const lines = configContent.split('\n');
            for (const line of lines) {
              const trimmed = line.trim();
              if (containerType === 'lxc' && trimmed.startsWith('hostname:')) {
                finalHostname = trimmed.substring(9).trim();
                break;
              } else if (containerType === 'vm' && trimmed.startsWith('name:')) {
                finalHostname = trimmed.substring(5).trim();
                break;
              }
            }
          }

          if (!finalHostname) {
            finalHostname = `${containerType}-${nextId}`;
          }

          // Create installed script record
          const script = await this.db.createInstalledScript({
            script_name: finalHostname,
            script_path: `cloned/${finalHostname}`,
            container_id: nextId,
            server_id: server.id,
            execution_mode: 'ssh',
            status: 'success',
            output_log: `Cloned ${containerType.toUpperCase()}`
          });

          // For LXC, store config in database
          if (containerType === 'lxc' && configContent.trim()) {
            // Simple config parser
            /** @type {any} */
            const configData = {};
            const lines = configContent.split('\n');
            for (const line of lines) {
              const trimmed = line.trim();
              if (!trimmed || trimmed.startsWith('#')) continue;

              const [key, ...valueParts] = trimmed.split(':');
              const value = valueParts.join(':').trim();

              if (key === 'hostname') configData.hostname = value;
              else if (key === 'arch') configData.arch = value;
              else if (key === 'cores') configData.cores = parseInt(value) || null;
              else if (key === 'memory') configData.memory = parseInt(value) || null;
              else if (key === 'swap') configData.swap = parseInt(value) || null;
              else if (key === 'onboot') configData.onboot = parseInt(value) || null;
              else if (key === 'ostype') configData.ostype = value;
              else if (key === 'unprivileged') configData.unprivileged = parseInt(value) || null;
              else if (key === 'tags') configData.tags = value;
              else if (key === 'rootfs') {
                const match = value.match(/^([^:]+):([^,]+)/);
                if (match) {
                  configData.rootfs_storage = match[1];
                  const sizeMatch = value.match(/size=([^,]+)/);
                  if (sizeMatch) {
                    configData.rootfs_size = sizeMatch[1];
                  }
                }
              }
            }

            await this.db.createLXCConfig(script.id, configData);
          }

          this.sendMessage(ws, {
            type: 'output',
            data: `\nClone ${i + 1} (ID: ${nextId}, Hostname: ${finalHostname}) added to database successfully.\n`,
            timestamp: Date.now()
          });
        } catch (error) {
          this.sendMessage(ws, {
            type: 'error',
            data: `\nError adding clone ${i + 1} (ID: ${nextId}) to database: ${error instanceof Error ? error.message : String(error)}\n`,
            timestamp: Date.now()
          });
        }
      }

      this.sendMessage(ws, {
        type: 'output',
        data: `\n\n[Clone operation completed successfully!]\nCreated ${cloneCount} clone(s) of ${containerType.toUpperCase()} ${containerId}.\n`,
        timestamp: Date.now()
      });

      this.activeExecutions.delete(executionId);
    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `\n\n[Clone operation failed!]\nError: ${error instanceof Error ? error.message : String(error)}\n`,
        timestamp: Date.now()
      });
      this.activeExecutions.delete(executionId);
    }
  }

  /**
   * Start update execution (pct enter + update command)
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {string} mode
   * @param {ServerInfo|undefined} server
   * @param {string} [backupStorage] - Optional storage to backup to before update
   * @param {number} [installedScriptId] - InstalledScript row to persist the final status/output to
   */
  async startUpdateExecution(ws, containerId, executionId, mode = 'local', server = undefined, backupStorage = undefined, envVars = {}, installedScriptId = undefined, cols = 220, rows = 50) {
    try {
      // If backup storage is provided, run backup first
      if (backupStorage && mode === 'ssh' && server) {
        this.sendMessage(ws, {
          type: 'start',
          data: `Starting backup before update for container ${containerId}...`,
          timestamp: Date.now()
        });

        // Create a separate execution ID for backup
        const backupExecutionId = `backup_${executionId}`;

        // Run backup and wait for it to complete
        try {
          const backupResult = await this.startSSHBackupExecution(
            ws,
            containerId,
            backupExecutionId,
            backupStorage,
            server,
            undefined,
            cols,
            rows
          );

          // Backup completed (successfully or not)
          if (!backupResult || !backupResult.success) {
            // Backup failed, but we'll still allow update (per requirement 1b)
            this.sendMessage(ws, {
              type: 'output',
              data: '\n⚠️ Backup failed, but proceeding with update as requested...\n',
              timestamp: Date.now()
            });
          } else {
            // Backup succeeded
            this.sendMessage(ws, {
              type: 'output',
              data: '\n✅ Backup completed successfully. Starting update...\n',
              timestamp: Date.now()
            });
          }
        } catch (error) {
          console.error('Backup error before update:', error);
          // Backup failed to start, but allow update to proceed
          this.sendMessage(ws, {
            type: 'output',
            data: `\n⚠️ Backup error: ${error instanceof Error ? error.message : String(error)}. Proceeding with update...\n`,
            timestamp: Date.now()
          });
        }

        // Small delay before starting update
        await new Promise(resolve => setTimeout(resolve, BACKUP_UPDATE_DELAY_MS));
      }

      // Send start message for update (only if we're actually starting an update)
      this.sendMessage(ws, {
        type: 'start',
        data: `Starting update for container ${containerId}...`,
        timestamp: Date.now()
      });

      if (mode === 'ssh' && server) {
        await this.startSSHUpdateExecution(ws, containerId, executionId, server, envVars, installedScriptId, cols, rows);
      } else {
        await this.startLocalUpdateExecution(ws, containerId, executionId, envVars, installedScriptId, cols, rows);
      }

    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `Failed to start update: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });
    }
  }

  /**
   * Start local update execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {Object} [envVars]
   * @param {number} [installedScriptId]
   */
  async startLocalUpdateExecution(ws, containerId, executionId, envVars = {}, installedScriptId = undefined, cols = 220, rows = 50) {
    const { spawn } = await import('node-pty');

    // Create a shell process that will run pct enter and then update
    const childProcess = spawn('bash', ['-c', `pct enter ${containerId}`], {
      name: 'xterm-color',
      cols,
      rows,
      cwd: process.cwd(),
      env: process.env
    });

    // Store the execution
    this.activeExecutions.set(executionId, {
      process: childProcess,
      ws,
      installationId: installedScriptId ?? null,
      outputBuffer: ''
    });

    // Handle pty data
    childProcess.onData((data) => {
      const execution = this.activeExecutions.get(executionId);
      if (execution) execution.outputBuffer += data.toString();
      this.sendMessage(ws, {
        type: 'output',
        data: data.toString(),
        timestamp: Date.now()
      });
    });

    // Build env export commands (e.g. for PHS_SILENT=1)
    const envExports = Object.entries(envVars)
      .filter(([key]) => (key.startsWith('PHS_') || key.startsWith('var_')) && /^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
      .map(([key, value]) => {
        const safeValue = String(value)
          .replace(/\\/g, '\\\\')
          .replace(/"/g, '\\"');
        return `export ${key}="${safeValue}"`;
      })
      .join('; ');

    // Send the update command after a delay to ensure we're in the container
    setTimeout(() => {
      childProcess.write(`${NEWT_COLORS_EXPORT}; ${envExports ? `${envExports}; ` : ''}update\n`);
    }, 4000);

    // Handle process exit
    childProcess.onExit(async (e) => {
      const execution = this.activeExecutions.get(executionId);
      if (installedScriptId && execution) {
        await this.updateInstallationRecord(installedScriptId, {
          status: e.exitCode === 0 ? 'success' : 'failed',
          output_log: execution.outputBuffer
        });
      }

      this.sendMessage(ws, {
        type: 'end',
        data: `Update completed with exit code: ${e.exitCode}`,
        timestamp: Date.now()
      });

      this.activeExecutions.delete(executionId);
    });
  }

  /**
   * Start SSH update execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {ServerInfo} server
   * @param {Object} [envVars]
   * @param {number} [installedScriptId]
   */
  async startSSHUpdateExecution(ws, containerId, executionId, server, envVars = {}, installedScriptId = undefined, cols = 220, rows = 50) {
    const sshService = getSSHExecutionService();

    try {
      const execution = await sshService.executeCommand(
        server,
        `pct enter ${containerId}`,
        /** @param {string} data */
        (data) => {
          const activeExec = this.activeExecutions.get(executionId);
          if (activeExec) activeExec.outputBuffer += data;
          this.sendMessage(ws, {
            type: 'output',
            data: data,
            timestamp: Date.now()
          });
        },
        /** @param {string} error */
        (error) => {
          this.sendMessage(ws, {
            type: 'error',
            data: error,
            timestamp: Date.now()
          });
        },
        /** @param {number} code */
        async (code) => {
          const activeExec = this.activeExecutions.get(executionId);
          if (installedScriptId && activeExec) {
            await this.updateInstallationRecord(installedScriptId, {
              status: code === 0 ? 'success' : 'failed',
              output_log: activeExec.outputBuffer
            });
          }

          this.sendMessage(ws, {
            type: 'end',
            data: `Update completed with exit code: ${code}`,
            timestamp: Date.now()
          });

          this.activeExecutions.delete(executionId);
        },
        cols,
        rows
      );

      // Store the execution
      this.activeExecutions.set(executionId, {
        process: /** @type {any} */ (execution).process,
        ws,
        installationId: installedScriptId ?? null,
        outputBuffer: ''
      });

      // Build env export commands (e.g. for PHS_SILENT=1)
      const envExports = Object.entries(envVars)
        .filter(([key]) => (key.startsWith('PHS_') || key.startsWith('var_')) && /^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
        .map(([key, value]) => {
          const safeValue = String(value)
            .replace(/\\/g, '\\\\')
            .replace(/"/g, '\\"');
          return `export ${key}="${safeValue}"`;
        })
        .join('; ');

      // Send the update command after a delay to ensure we're in the container
      setTimeout(() => {
        /** @type {any} */ (execution).process.write(`${NEWT_COLORS_EXPORT}; ${envExports ? `${envExports}; ` : ''}update\n`);
      }, 4000);

    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `SSH execution failed: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });
    }
  }

  /**
   * Start shell execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {string} mode
   * @param {ServerInfo|null} server
   * @param {'lxc'|'vm'} [containerType='lxc']
   */
  async startShellExecution(ws, containerId, executionId, mode = 'local', server = null, containerType = 'lxc', cols = 220, rows = 50) {
    try {
      const typeLabel = containerType === 'vm' ? 'VM' : 'container';
      this.sendMessage(ws, {
        type: 'start',
        data: `Starting shell session for ${typeLabel} ${containerId}...`,
        timestamp: Date.now()
      });

      if (mode === 'ssh' && server) {
        await this.startSSHShellExecution(ws, containerId, executionId, server, containerType, cols, rows);
      } else {
        await this.startLocalShellExecution(ws, containerId, executionId, containerType, cols, rows);
      }

    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `Failed to start shell: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });
    }
  }

  /**
   * Start local shell execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {'lxc'|'vm'} [containerType='lxc']
   */
  async startLocalShellExecution(ws, containerId, executionId, containerType = 'lxc', cols = 220, rows = 50) {
    const { spawn } = await import('node-pty');
    const shellCommand = containerType === 'vm' ? `qm terminal ${containerId}` : `pct enter ${containerId}`;
    const childProcess = spawn('bash', ['-c', shellCommand], {
      name: 'xterm-color',
      cols,
      rows,
      cwd: process.cwd(),
      env: process.env
    });

    // Store the execution
    this.activeExecutions.set(executionId, {
      process: childProcess,
      ws
    });

    // Handle pty data
    childProcess.onData((data) => {
      this.sendMessage(ws, {
        type: 'output',
        data: data.toString(),
        timestamp: Date.now()
      });
    });

    // Improve readability of any whiptail/dialog UI the user runs manually
    // (LXC only — a VM's serial console may not have a shell ready yet).
    if (containerType === 'lxc') {
      setTimeout(() => {
        childProcess.write(`${NEWT_COLORS_EXPORT}\n`);
      }, 2000);
    }

    // Handle process exit
    childProcess.onExit((e) => {
      this.sendMessage(ws, {
        type: 'end',
        data: `Shell session ended with exit code: ${e.exitCode}`,
        timestamp: Date.now()
      });

      this.activeExecutions.delete(executionId);
    });
  }

  /**
   * Start SSH shell execution
   * @param {ExtendedWebSocket} ws
   * @param {string} containerId
   * @param {string} executionId
   * @param {ServerInfo} server
   * @param {'lxc'|'vm'} [containerType='lxc']
   */
  async startSSHShellExecution(ws, containerId, executionId, server, containerType = 'lxc', cols = 220, rows = 50) {
    const sshService = getSSHExecutionService();
    const shellCommand = containerType === 'vm' ? `qm terminal ${containerId}` : `pct enter ${containerId}`;
    try {
      const execution = await sshService.executeCommand(
        server,
        shellCommand,
        /** @param {string} data */
        (data) => {
          this.sendMessage(ws, {
            type: 'output',
            data: data,
            timestamp: Date.now()
          });
        },
        /** @param {string} error */
        (error) => {
          this.sendMessage(ws, {
            type: 'error',
            data: error,
            timestamp: Date.now()
          });
        },
        /** @param {number} code */
        (code) => {
          this.sendMessage(ws, {
            type: 'end',
            data: `Shell session ended with exit code: ${code}`,
            timestamp: Date.now()
          });

          this.activeExecutions.delete(executionId);
        },
        cols,
        rows
      );

      // Store the execution
      this.activeExecutions.set(executionId, {
        process: /** @type {any} */ (execution).process,
        ws
      });

      // Improve readability of any whiptail/dialog UI the user runs manually
      // (LXC only — a VM's serial console may not have a shell ready yet).
      if (containerType === 'lxc') {
        setTimeout(() => {
          /** @type {any} */ (execution).process.write(`${NEWT_COLORS_EXPORT}\n`);
        }, 2000);
      }

    } catch (error) {
      this.sendMessage(ws, {
        type: 'error',
        data: `SSH shell execution failed: ${error instanceof Error ? error.message : String(error)}`,
        timestamp: Date.now()
      });
    }
  }
}

// TerminalHandler removed - not used by current application

app.prepare().then(() => {
  console.log('> Next.js app prepared successfully');
  const httpServer = createServer(async (req, res) => {
    try {
      // Be sure to pass `true` as the second argument to `url.parse`.
      // This tells it to parse the query portion of the URL.
      const parsedUrl = parse(req.url || '', true);
      const { pathname, query } = parsedUrl;

      // Check if this is a WebSocket upgrade request
      const isWebSocketUpgrade = req.headers.upgrade === 'websocket';

      // Only intercept WebSocket upgrades for /ws/script-execution
      // Let Next.js handle all other WebSocket upgrades (like HMR) and all HTTP requests
      if (isWebSocketUpgrade && pathname === '/ws/script-execution') {
        // WebSocket upgrade will be handled by the WebSocket server
        // Don't call handle() for this path - let WebSocketServer handle it
        return;
      }

      // Let Next.js handle all other requests including:
      // - HTTP requests to /ws/script-execution (non-WebSocket)
      // - WebSocket upgrades to other paths (like /_next/webpack-hmr)
      // - All static assets (_next routes)
      // - All other routes
      await handle(req, res, parsedUrl);
    } catch (err) {
      console.error('Error occurred handling', req.url, err);
      res.statusCode = 500;
      res.end('internal server error');
    }
  });

  // Create WebSocket handlers
  const scriptHandler = new ScriptExecutionHandler(httpServer);

  // Handle WebSocket upgrades manually to avoid interfering with Next.js HMR
  // We need to preserve Next.js's upgrade handlers and call them for non-matching paths
  // Save any existing upgrade listeners (Next.js might have set them up)
  const existingUpgradeListeners = httpServer.listeners('upgrade').slice();
  httpServer.removeAllListeners('upgrade');

  // Add our upgrade handler that routes based on path
  httpServer.on('upgrade', (request, socket, head) => {
    const parsedUrl = parse(request.url || '', true);
    const { pathname } = parsedUrl;

    if (pathname === '/ws/script-execution') {
      if (!isWebSocketUpgradeAuthorized(request)) {
        socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
        socket.destroy();
        return;
      }

      // Handle our custom WebSocket endpoint
      scriptHandler.handleUpgrade(request, socket, head);
    } else {
      // For all other paths (including Next.js HMR), call existing listeners
      // This allows Next.js to handle its own WebSocket upgrades
      for (const listener of existingUpgradeListeners) {
        try {
          listener.call(httpServer, request, socket, head);
        } catch (err) {
          console.error('Error in upgrade listener:', err);
        }
      }
    }
  });
  // Note: TerminalHandler removed as it's not being used by the current application

  httpServer
    .once('error', (err) => {
      console.error(err);
      process.exit(1);
    })
    .listen(port, hostname, async () => {
      console.log(`> Ready on http://${hostname}:${port}`);
      console.log(`> WebSocket server running on ws://${hostname}:${port}/ws/script-execution`);

      // Initialize auto sync module and run initialization
      if (!autoSyncModule) {
        try {
          console.log('Dynamically importing autoSyncInit...');
          autoSyncModule = await import('./src/server/lib/autoSyncInit.js');
          console.log('autoSyncModule loaded, exports:', Object.keys(autoSyncModule));
        } catch (error) {
          const err = error instanceof Error ? error : new Error(String(error));
          console.error('Failed to import autoSyncInit:', err.message);
          console.error('Stack:', err.stack);
          throw error;
        }
      }

      // Initialize default repositories
      if (typeof autoSyncModule.initializeRepositories === 'function') {
        console.log('Calling initializeRepositories...');
        await autoSyncModule.initializeRepositories();
      } else {
        console.warn('initializeRepositories is not a function, type:', typeof autoSyncModule.initializeRepositories);
      }

      // Initialize auto-sync service
      if (typeof autoSyncModule.initializeAutoSync === 'function') {
        console.log('Calling initializeAutoSync...');
        autoSyncModule.initializeAutoSync();
      }

      // Setup graceful shutdown handlers
      if (typeof autoSyncModule.setupGracefulShutdown === 'function') {
        console.log('Setting up graceful shutdown...');
        autoSyncModule.setupGracefulShutdown();
      }
    });
}).catch((err) => {
  console.error('> Failed to start server:', err.message);
  console.error('> If you see "Could not find a production build", run: npm run build');
  console.error('> Full error:', err);
  process.exit(1);
});