openClaw_agent_dm/agent-monitor.js

#!/usr/bin/env node

/**
 * OpenClaw Agent Health Monitor & Auto-Healing System
 * 
 * Features:
 * - Process crash detection and auto-restart
 * - Memory leak monitoring
 * - Service health checks
 * - Telegram notifications on events
 * - Comprehensive logging
 * - Systemd integration
 */

const fs = require('fs');
const path = require('path');
const { spawn } = require('child_process');
const { exec } = require('child_process');
const util = require('util');
const execAsync = util.promisify(exec);

class AgentHealthMonitor {
  constructor() {
    this.config = this.loadConfig();
    this.logDir = '/root/.openclaw/workspace/logs/agents';
    this.workspaceDir = '/root/.openclaw/workspace';
    this.processes = new Map();
    this.restartCounts = new Map();
    this.maxRestarts = 5;
    this.restartWindow = 300000; // 5 minutes
    this.gracePeriod = 60000; // 60s grace period after first failure (upgrade tolerance)
    this.heartbeatInterval = 600000; // 10 minutes
    this.lastKnownState = { gateway: true, life: true };
    this.firstFailureTime = { gateway: 0, life: 0 };
    
    this.ensureLogDir();
    this.setupSignalHandlers();
    this.log('Agent Health Monitor initialized', 'info');
  }

  loadConfig() {
    try {
      const configPath = '/root/.openclaw/openclaw.json';
      if (fs.existsSync(configPath)) {
        return JSON.parse(fs.readFileSync(configPath, 'utf8'));
      }
    } catch (error) {
      console.error('Failed to load OpenClaw config:', error.message);
    }
    return {};
  }

  ensureLogDir() {
    if (!fs.existsSync(this.logDir)) {
      fs.mkdirSync(this.logDir, { recursive: true });
    }
  }

  setupSignalHandlers() {
    process.on('SIGTERM', () => this.gracefulShutdown());
    process.on('SIGINT', () => this.gracefulShutdown());
  }

  async gracefulShutdown() {
    this.log('Graceful shutdown initiated', 'info');
    
    // Stop all monitored processes
    for (const [name, proc] of this.processes.entries()) {
      try {
        proc.kill('SIGTERM');
        this.log(`Stopped process: ${name}`, 'info');
      } catch (error) {
        this.log(`Error stopping ${name}: ${error.message}`, 'error');
      }
    }
    
    process.exit(0);
  }

  log(message, severity = 'info') {
    const timestamp = new Date().toISOString();
    const logEntry = `[${timestamp}] [${severity.toUpperCase()}] ${message}\n`;
    
    // Console output
    console.log(logEntry.trim());
    
    // File logging
    const logFile = path.join(this.logDir, `health-${new Date().toISOString().split('T')[0]}.log`);
    fs.appendFileSync(logFile, logEntry);
  }

  async sendNotification(message, severity = 'info') {
    this.log(message, severity);
    
    // Send via Telegram if configured
    const telegramConfig = this.config.channels?.telegram;
    if (telegramConfig?.enabled && telegramConfig.botToken) {
      await this.sendTelegramNotification(message, severity);
    }
    
    // Also send via OpenClaw message tool if available
    if (severity === 'critical' || severity === 'error') {
      await this.sendOpenClawNotification(message, severity);
    }
  }

  async sendTelegramNotification(message, severity) {
    const botToken = this.config.channels.telegram.botToken;
    const chatId = '5237946060';
    
    if (!botToken) {
      return;
    }

    try {
      const url = `https://api.telegram.org/bot${botToken}/sendMessage`;
      const emojis = {
        critical: '🚨',
        error: '❌',
        warning: '⚠️',
        info: 'ℹ️'
      };
      
      const payload = {
        chat_id: chatId,
        text: `${emojis[severity] || '📢'} *OpenClaw Alert* (${severity})\n\n${message}`,
        parse_mode: 'Markdown'
      };

      const response = await fetch(url, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(payload)
      });

      if (!response.ok) {
        throw new Error(`Telegram API error: ${response.status}`);
      }
    } catch (error) {
      console.error('Telegram notification error:', error.message);
    }
  }

  async sendOpenClawNotification(message, severity) {
    try {
      // Use OpenClaw's message tool via exec
      const cmd = `openclaw message send --channel telegram --target 5237946060 --message "🚨 OpenClaw Service Alert (${severity})\\n\\n${message}"`;
      await execAsync(cmd);
    } catch (error) {
      console.error('OpenClaw notification error:', error.message);
    }
  }

  checkRestartLimit(processName) {
    const now = Date.now();
    const restarts = this.restartCounts.get(processName) || [];
    
    // Filter restarts within the window
    const recentRestarts = restarts.filter(time => now - time < this.restartWindow);
    
    if (recentRestarts.length >= this.maxRestarts) {
      return false; // Too many restarts
    }
    
    this.restartCounts.set(processName, [...recentRestarts, now]);
    return true;
  }

  async monitorProcess(name, command, args = [], options = {}) {
    const {
      healthCheck,
      healthCheckInterval = 30000,
      env = {},
      cwd = this.workspaceDir
    } = options;

    const startProcess = () => {
      return new Promise((resolve, reject) => {
        const proc = spawn(command, args, {
          cwd,
          env: { ...process.env, ...env },
          stdio: ['ignore', 'pipe', 'pipe']
        });

        proc.stdout.on('data', (data) => {
          this.log(`[${name}] ${data.toString().trim()}`, 'info');
        });

        proc.stderr.on('data', (data) => {
          this.log(`[${name}] ${data.toString().trim()}`, 'error');
        });

        proc.on('error', async (error) => {
          this.log(`[${name}] Process error: ${error.message}`, 'critical');
          await this.sendNotification(`${name} failed to start: ${error.message}`, 'critical');
          reject(error);
        });

        proc.on('close', async (code, signal) => {
          this.processes.delete(name);
          this.log(`[${name}] Process exited with code ${code}, signal ${signal}`, 'warning');
          
          // Auto-restart logic
          if (code !== 0 || signal) {
            if (this.checkRestartLimit(name)) {
              this.log(`[${name}] Auto-restarting...`, 'warning');
              await this.sendNotification(`${name} crashed (code: ${code}, signal: ${signal}). Restarting...`, 'error');
              setTimeout(() => startProcess(), 5000);
            } else {
              await this.sendNotification(
                `${name} crashed ${this.maxRestarts} times in ${this.restartWindow/60000} minutes. Giving up.`,
                'critical'
              );
            }
          }
        });

        this.processes.set(name, proc);
        resolve(proc);
      });
    };

    // Start the process
    await startProcess();

    // Set up health checks
    if (healthCheck) {
      setInterval(async () => {
        try {
          const isHealthy = await healthCheck();
          if (!isHealthy) {
            await this.sendNotification(`${name} health check failed`, 'warning');
            
            // Restart unhealthy process
            const proc = this.processes.get(name);
            if (proc) {
              proc.kill('SIGTERM');
            }
          }
        } catch (error) {
          await this.sendNotification(`${name} health check error: ${error.message}`, 'error');
        }
      }, healthCheckInterval);
    }
  }

  async checkOpenClawGateway() {
    try {
      const { stdout } = await execAsync('openclaw gateway status 2>&1 || echo "not running"');
      return stdout.includes('running') || 
             stdout.includes('active') || 
             stdout.includes('RPC probe: ok') ||
             stdout.includes('Listening:');
    } catch (error) {
      this.log(`Gateway status check error: ${error.message}`, 'error');
      return false;
    }
  }

  async checkLifeAgent() {
    try {
      const env = {
        ...process.env,
        XDG_RUNTIME_DIR: '/run/user/0',
        DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/0/bus'
      };
      const { stdout } = await execAsync(
        'systemctl --user is-active openclaw-gateway-life.service 2>&1 || echo "inactive"',
        { env }
      );
      return stdout.trim() === 'active';
    } catch (error) {
      this.log(`Life agent status check error: ${error.message}`, 'error');
      return false;
    }
  }

  async startOpenClawGateway() {
    const env = {
      ...process.env,
      XDG_RUNTIME_DIR: '/run/user/0',
      DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/0/bus'
    };
    try {
      const { stdout } = await execAsync('openclaw gateway start', { env });
      this.log(`OpenClaw Gateway started: ${stdout}`, 'info');
    } catch (error) {
      this.log(`Failed to start OpenClaw Gateway: ${error.message}`, 'error');
      throw error;
    }
  }

  async startLifeAgent() {
    const env = {
      ...process.env,
      XDG_RUNTIME_DIR: '/run/user/0',
      DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/0/bus'
    };
    try {
      const { stdout } = await execAsync(
        'systemctl --user start openclaw-gateway-life.service', { env }
      );
      this.log(`Life agent started: ${stdout}`, 'info');
    } catch (error) {
      this.log(`Failed to start Life agent: ${error.message}`, 'error');
      throw error;
    }
  }

  async handleServiceDown(serviceName, startFn) {
    const now = Date.now();

    if (this.lastKnownState[serviceName]) {
      this.firstFailureTime[serviceName] = now;
      this.lastKnownState[serviceName] = false;
      this.log(`${serviceName} detected down, entering grace period (${this.gracePeriod / 1000}s)...`, 'warning');
      return;
    }

    if (now - this.firstFailureTime[serviceName] < this.gracePeriod) {
      return;
    }

    if (!this.checkRestartLimit(serviceName)) {
      await this.sendNotification(
        `${serviceName} crashed ${this.maxRestarts} times in ${this.restartWindow / 60000} min. Auto-restart disabled until window resets.`,
        'critical'
      );
      return;
    }

    await this.sendNotification(`${serviceName} is down. Attempting restart...`, 'error');
    try {
      await startFn();
      this.lastKnownState[serviceName] = true;
      this.firstFailureTime[serviceName] = 0;
      await this.sendNotification(`${serviceName} restarted successfully`, 'info');
    } catch (error) {
      await this.sendNotification(`Failed to restart ${serviceName}: ${error.message}`, 'critical');
    }
  }

  async monitorOpenClawService() {
    this.log('Starting service monitoring (gateway + life)...', 'info');
    let heartbeatCounter = 0;

    setInterval(async () => {
      const gatewayOk = await this.checkOpenClawGateway();
      const lifeOk = await this.checkLifeAgent();

      if (gatewayOk) {
        if (!this.lastKnownState.gateway) {
          this.log('Gateway recovered', 'info');
        }
        this.lastKnownState.gateway = true;
        this.firstFailureTime.gateway = 0;
      } else {
        await this.handleServiceDown('gateway', () => this.startOpenClawGateway());
      }

      if (lifeOk) {
        if (!this.lastKnownState.life) {
          this.log('Life agent recovered', 'info');
        }
        this.lastKnownState.life = true;
        this.firstFailureTime.life = 0;
      } else {
        await this.handleServiceDown('life', () => this.startLifeAgent());
      }

      heartbeatCounter++;
      if (heartbeatCounter >= (this.heartbeatInterval / 30000)) {
        this.log(`Heartbeat: gateway=${gatewayOk ? 'OK' : 'DOWN'}, life=${lifeOk ? 'OK' : 'DOWN'}`, 'info');
        heartbeatCounter = 0;
      }
    }, 30000);
  }

  async start() {
    this.log('Agent Health Monitor starting...', 'info');

    const gatewayOk = await this.checkOpenClawGateway();
    const lifeOk = await this.checkLifeAgent();
    this.log(`Initial check: gateway=${gatewayOk ? 'OK' : 'DOWN'}, life=${lifeOk ? 'OK' : 'DOWN'}`, 'info');
    this.lastKnownState.gateway = gatewayOk;
    this.lastKnownState.life = lifeOk;

    await this.monitorOpenClawService();
    this.log('Monitor is now active. Press Ctrl+C to stop.', 'info');
  }
}

// Start the monitor
const monitor = new AgentHealthMonitor();
monitor.start().catch(console.error);