'use strict';

//
// AutoPilot auto-remediation engine. Owns BOTH trigger paths:
//   Path A — service heartbeat (5-min cron in server.js): live-poll EBS
//            infrastructure status ops that never appear in check_results,
//            track consecutive downs, restart when a threshold is crossed.
//   Path B — HC-triggered (services/schedule-runner.js): CM/OPP checks that DO
//            appear in check_results fire remediation off occurrence_count.
//
// Agent contract (verified against oracle-proxy.py /api/ebs-ctrl + the
// routes/ebs-ops.js middleware-run consumer):
//   sendToAgent(connId, { method, path, body }, timeoutMs) → { statusCode, body }
//   body = { success, ok, stdout, stderr, exit_code, duration_ms }
//     success — the handler ran (false/HTTP≠200 = proxy/handler error, NOT a
//               service-down signal; we never restart on it)
//     ok      — exit_code === 0
//     exit_code — the shell exit code (0 = service up)
//

const pool    = require('../db/index');
const channel = require('./agent-channel');
const ebsJobsDb = require('../db/ebs-jobs');
const { decrypt } = require('../crypto-utils');
const oracleClient = require('../oracle-client');

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// ── Status op → start op mapping (Path A — heartbeat services) ──────────────
// Only safe individual-service restarts. NEVER: adcmctl_stop/start (CM full),
// wls_admin (LONG_OP bg-job model), apps_stop_all, DB shutdown, ADOP ops.
const HEARTBEAT_SERVICES = [
  { statusOp: 'adapcctl_status',     startOp: 'adapcctl_start',     label: 'Restart Apache/OHS'    },
  { statusOp: 'adopmnctl_status',    startOp: 'adopmnctl_start',    label: 'Restart OPMN'          },
  { statusOp: 'adalnctl_status',     startOp: 'adalnctl_start',     label: 'Restart Apps Listener' },
  { statusOp: 'adnodemgrctl_status', startOp: 'adnodemgrctl_start', label: 'Restart Node Manager'  },
  { statusOp: 'db_listener_status',  startOp: 'db_listener_start',  label: 'Restart DB Listener'   },
  // DB instance startup. db_listener_status alone is NOT proof the instance is down, so
  // this is gated with a second step (requiresSqlProbe): after the listener is down for
  // `threshold` consecutive checks, a direct SELECT 1 FROM DUAL confirms the instance is
  // genuinely down before we fire. startOp is db_startup_open (plain STARTUP — a no-op on
  // a running instance), never STARTUP FORCE. dbOnly (server_type='db').
  { statusOp: 'db_listener_status',  startOp: 'db_startup_open',    label: 'Start DB Instance', requiresSqlProbe: true, dbOnly: true },
  // WLS managed servers (OACore/Forms/OAFM). No single-op restart: their real server
  // names vary per environment and are DISCOVERED at runtime from the
  // managed_servers_status response (never hardcoded), then each DOWN server is
  // restarted individually via managed_server_start(server_name). groupKey is what the
  // user opts into (stored in remediate_services); `match` selects this group's servers
  // from the discovered set; startOp is the shared restart op.
  { requiresDiscovery: true, groupKey: 'oacore', match: 'oacore', statusOp: 'managed_servers_status', startOp: 'managed_server_start', label: 'Restart OACore managed server' },
  { requiresDiscovery: true, groupKey: 'forms',  match: 'forms',  statusOp: 'managed_servers_status', startOp: 'managed_server_start', label: 'Restart Forms managed server'  },
  { requiresDiscovery: true, groupKey: 'oafm',   match: 'oafm',   statusOp: 'managed_servers_status', startOp: 'managed_server_start', label: 'Restart OAFM managed server'   },
  // WLS Admin Server — LONG_OP (background job model). Poll is slow (75s). Restart is
  // NOT a simple op: fire wls_admin_start_bg (returns {started,pid}), then poll
  // apps_job_status until done/failed. isLongOp routes it through fireWlsAdminRestart.
  { statusOp: 'wls_admin_status', startOp: 'wls_admin_start', label: 'WLS Admin Server', scope: 'ebs_only', isLongOp: true },
];

// ── HC check_id → start op mapping (Path B — check_results) ─────────────────
const HC_REMEDIATION_MAP = {
  'EBS_CM01_INTERNAL_MANAGER': { startOp: 'adcmctl_start', label: 'Restart Concurrent Manager', scope: 'ebs_only' },
  'EBS_CM03_OPP_STATUS':       { startOp: 'adcmctl_start', label: 'Restart OPP via adcmctl',    scope: 'ebs_only' },
};

// The opt-in keys the engine can act on: simple services use their startOp; managed-
// server groups use their groupKey (oacore/forms/oafm) — the shared managed_server_start
// op alone can't distinguish the group. Plus adcmctl_start (Path B). remediate_services
// is validated against this at save time so no non-actionable key can ever be stored.
const RESTARTABLE_OPS = new Set([
  ...HEARTBEAT_SERVICES.map(s => (s.requiresDiscovery ? s.groupKey : s.startOp)),
  ...Object.values(HC_REMEDIATION_MAP).map(m => m.startOp),
]);

// Per-start-op agent timeouts (mirror routes/ebs-ops.js _ctrlTimeoutMap).
const START_TIMEOUTS = {
  adapcctl_start: 65000, adopmnctl_start: 65000, adalnctl_start: 35000,
  adnodemgrctl_start: 65000, db_listener_start: 60000,
  adcmctl_start: 130000, managed_server_start: 130000, db_startup_open: 120000,
  wls_admin_start: 130000, // used only if fired directly; the LONG_OP path uses fireWlsAdminRestart
};
const STATUS_TIMEOUT_MS = 25000;
const WLS_STATUS_TIMEOUT_MS = 75000; // wls_admin_status is a slow check
const MANAGED_STATUS_TIMEOUT_MS = 130000; // managed_servers_status loops admanagedsrvctl per server

// ── 30-minute per-op cooldown ────────────────────────────────────────────────
async function isCoolingDown(connectionId, opKey) {
  const { rows } = await pool.query(
    `SELECT id FROM auto_remediation_log
     WHERE connection_id = $1 AND op_key = $2
       AND outcome IN ('success', 'failed')
       AND executed_at > NOW() - INTERVAL '30 minutes'
     LIMIT 1`,
    [connectionId, opKey]
  );
  return rows.length > 0;
}

// ── ADOP gate ────────────────────────────────────────────────────────────────
// ebs_adop_state only ever has a row for EBS connections; absence → not patching.
async function isAdopActive(connectionId) {
  const { rows } = await pool.query(
    `SELECT patching FROM ebs_adop_state WHERE connection_id = $1`,
    [connectionId]
  );
  return !!(rows[0] && rows[0].patching);
}

// ── Direct SQL liveness probe ────────────────────────────────────────────────
// SELECT 1 FROM DUAL against the connection's Oracle instance via oracle-client.
// true = instance answered (genuinely up); false = unreachable/error. Used as the
// second step of the DB-startup gate so we never fire STARTUP on a live instance.
// NOTE: this dials the DB directly (not via the agent). For a proxy connection whose
// DB host is firewalled from the server it may return false even when the instance is
// up — but the startup op is plain STARTUP (a no-op on a running instance), so a
// false negative is harmless.
async function probeSqlConnection(connectionId) {
  try {
    const { rows } = await pool.query(
      `SELECT host, port, service_name, username, encrypted_password
       FROM oracle_connections WHERE id = $1`,
      [connectionId]
    );
    const c = rows[0];
    if (!c || !c.encrypted_password || !c.username || !c.host || !c.service_name) return false;
    return await oracleClient.sqlPing({
      host: c.host, port: c.port, serviceName: c.service_name,
      username: c.username, password: decrypt(c.encrypted_password),
    });
  } catch (err) {
    console.warn(`[autopilot/heartbeat] SQL probe error conn ${connectionId}: ${err.message}`);
    return false;
  }
}

// ── Log helper (never throws — remediation must not die on a log-write error) ─
async function logAction(fields) {
  await pool.query(
    `INSERT INTO auto_remediation_log
       (connection_id, trigger_path, triggered_by, consecutive_count,
        op_key, op_label, outcome, exit_code, stdout_tail, tuneops_ticket_id)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`,
    [
      fields.connectionId, fields.triggerPath, fields.triggeredBy,
      fields.consecutiveCount, fields.opKey, fields.opLabel,
      fields.outcome, fields.exitCode == null ? null : fields.exitCode,
      fields.stdoutTail == null ? null : fields.stdoutTail,
      fields.tuneopsTicketId == null ? null : fields.tuneopsTicketId,
    ]
  ).catch(err => console.error('[autopilot] log error:', err.message));
}

// ── Fire a start op via agent-channel ────────────────────────────────────────
// ok requires HTTP 200 AND body.success (handler ran) AND body.ok (exit 0) —
// the same gate routes/ebs-ops.js applies before reporting success.
async function fireStartOp(connectionId, startOp, appsPwd, weblogicPwd, serverName) {
  const timeoutMs = START_TIMEOUTS[startOp] || 40000;

  const reqBody = {
    op: startOp,
    apps_pwd: appsPwd || '',
    weblogic_pwd: weblogicPwd || '',
  };
  // managed_server_start targets one WLS managed server by its discovered name.
  if (serverName) reqBody.server_name = serverName;

  const result = await channel.sendToAgent(connectionId, {
    method: 'POST',
    path: '/api/ebs-ctrl',
    body: reqBody,
  }, timeoutMs);

  const body = (result && result.body) || {};
  return {
    exitCode: body.exit_code == null ? null : body.exit_code,
    stdout: body.stdout || '',
    ok: result && result.statusCode === 200 && body.success === true && body.ok === true,
  };
}

// ── Fire a LONG_OP (WLS Admin) via the background-job model ───────────────────
// createJob → wls_admin_start_bg (returns {started,pid}) → poll apps_job_status
// (returns {state,rc,log_tail}) every 15s up to 10× (~150s) → completeJob. We do
// NOT call startJob, so the row stays 'queued' and the separate ebs-jobs poller
// (which only handles status='running') never races this inline poll. Returns the
// same { ok, exitCode, stdout } shape fireStartOp does.
async function fireWlsAdminRestart(connectionId, appsPwd, weblogicPwd) {
  const t0 = Date.now();
  let jobId = null;
  try {
    const created = await ebsJobsDb.createJob(connectionId, 'wls_admin_start', null);
    jobId = created.id;

    const launch = await channel.sendToAgent(connectionId, {
      method: 'POST', path: '/api/ebs-ctrl',
      body: { op: 'wls_admin_start_bg', job_id: String(jobId), weblogic_pwd: weblogicPwd || '', apps_pwd: appsPwd || '' },
    }, 15000);
    const lb = (launch && launch.body) || {};
    if (lb.started !== true) {
      const msg = lb.error || 'Background start not confirmed';
      await ebsJobsDb.completeJob(jobId, { ok: false, stdout: msg, exit_code: 1, duration_ms: Date.now() - t0 }).catch(() => {});
      return { ok: false, exitCode: 1, stdout: msg };
    }

    // Poll apps_job_status (state ∈ done|failed|cancelled|running|unknown).
    let state = 'running', rc = null, logTail = '';
    for (let i = 0; i < 10; i++) {
      await sleep(15000);
      let pr;
      try {
        pr = await channel.sendToAgent(connectionId, {
          method: 'POST', path: '/api/ebs-ctrl',
          body: { op: 'apps_job_status', job_id: String(jobId) },
        }, 20000);
      } catch (e) { continue; } // transient poll error — try again next tick
      const pb = (pr && pr.body) || {};
      if (pb.success !== true) continue;
      if (pb.log_tail) logTail = pb.log_tail;
      if (pb.state) state = pb.state;
      if (pb.rc != null) rc = pb.rc;
      if (state === 'done' || state === 'failed' || state === 'cancelled') break;
    }

    const ok = state === 'done';
    const exitCode = ok ? 0 : (rc != null ? rc : 1);
    await ebsJobsDb.completeJob(jobId, { ok, stdout: logTail, exit_code: exitCode, duration_ms: Date.now() - t0 }).catch(() => {});
    const tail = (state === 'running' || state === 'unknown')
      ? (logTail + '\n[still running after ~150s — may finish on its own]')
      : logTail;
    return { ok, exitCode, stdout: tail };
  } catch (err) {
    if (jobId != null) await ebsJobsDb.completeJob(jobId, { ok: false, stdout: err.message, exit_code: 1, duration_ms: Date.now() - t0 }).catch(() => {});
    return { ok: false, exitCode: 1, stdout: err.message };
  }
}

// Parse a managed_servers_status response into [{ name, state, raw }] by discovering
// server names from the TV_SERVER_NAME / TV_SERVER_EXIT blocks the proxy emits — no
// hardcoded names. Classification mirrors oracle-proxy.py exactly (states RUNNING |
// DOWN | TIMEOUT | NOT_DEPLOYED | UNKNOWN) so we never diverge from what the proxy
// itself considers down.
function parseServerNames(stdout) {
  const out = [];
  let cur = null, lines = [], exit = 0;
  for (const line of String(stdout || '').split('\n')) {
    if (line.indexOf('TV_SERVER_NAME ') === 0) {
      cur = line.slice(15).trim(); lines = []; exit = 0;
    } else if (line.indexOf('TV_SERVER_EXIT ') === 0) {
      exit = parseInt(line.slice(15).trim(), 10) || 0;
      if (cur) {
        const raw = lines.join('\n');
        const escName = cur.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
        let state;
        if (exit === 124) state = 'TIMEOUT';
        else if (/invalid server name|invalid managed server/i.test(raw)) state = 'NOT_DEPLOYED';
        else if (new RegExp('\\b' + escName + '\\s+is\\s+running', 'i').test(raw)) state = 'RUNNING';
        else if (/is shutdown|is not running|not running/i.test(raw)) state = 'DOWN';
        else state = 'UNKNOWN';
        out.push({ name: cur, state, raw });
        cur = null;
      }
    } else if (cur !== null) {
      lines.push(line);
    }
  }
  return out;
}

// Shared down-signal lifecycle for both simple services and individual managed
// servers: upsert consecutive_down (keyed by o.stateKey), gate on threshold / ADOP /
// cooldown, fire the restart, log, reset. The log + cooldown key is startOp[:serverName]
// so managed servers cool down (30 min) per individual server, not as a group.
async function applyDownSignal(o) {
  const opKey = o.serverName ? (o.startOp + ':' + o.serverName) : o.startOp;

  const { rows } = await pool.query(
    `INSERT INTO service_heartbeat_state (connection_id, op_key, consecutive_down, last_checked_at, last_status)
     VALUES ($1, $2, $3, NOW(), $4)
     ON CONFLICT (connection_id, op_key) DO UPDATE
       SET consecutive_down = CASE WHEN $4 = 'down'
             THEN service_heartbeat_state.consecutive_down + 1 ELSE 0 END,
           last_checked_at = NOW(),
           last_status = $4
     RETURNING consecutive_down`,
    [o.connectionId, o.stateKey, o.isDown ? 1 : 0, o.isDown ? 'down' : 'up']
  );
  const consecutiveDown = (rows[0] && rows[0].consecutive_down) || 0;
  if (!o.isDown || consecutiveDown < o.threshold) return;

  if (o.adopActive) {
    await logAction({ connectionId: o.connectionId, triggerPath: 'heartbeat', triggeredBy: o.triggeredBy,
      consecutiveCount: consecutiveDown, opKey, opLabel: o.label, outcome: 'skipped_adop' });
    return;
  }
  if (await isCoolingDown(o.connectionId, opKey)) return; // attempted within 30 min — spam guard

  // Optional pre-fire gate (e.g. the DB SQL probe): if it returns {skip}, log the given
  // outcome, reset the counter (so we re-accumulate before probing again), and do NOT fire.
  if (o.preFire) {
    const pf = await o.preFire();
    if (pf && pf.skip) {
      await logAction({ connectionId: o.connectionId, triggerPath: 'heartbeat', triggeredBy: o.triggeredBy,
        consecutiveCount: consecutiveDown, opKey, opLabel: o.label, outcome: pf.outcome });
      await pool.query(
        `UPDATE service_heartbeat_state SET consecutive_down = 0 WHERE connection_id = $1 AND op_key = $2`,
        [o.connectionId, o.stateKey]
      ).catch(() => {});
      return;
    }
  }

  console.log(`[autopilot] Firing ${opKey} on conn ${o.connectionId} — ${o.triggeredBy} down ${consecutiveDown}x`);
  let outcome = 'failed', exitCodeR = null, stdoutTail = null;
  try {
    // o.fireFn (LONG_OP path, e.g. WLS Admin bg job) overrides the simple fireStartOp.
    const r = o.fireFn
      ? await o.fireFn()
      : await fireStartOp(o.connectionId, o.startOp, o.appsPwd, o.weblogicPwd, o.serverName);
    exitCodeR = r.exitCode;
    stdoutTail = (r.stdout || '').slice(-500);
    outcome = r.ok ? 'success' : 'failed';
  } catch (err) {
    stdoutTail = err.message;
  }
  await logAction({ connectionId: o.connectionId, triggerPath: 'heartbeat', triggeredBy: o.triggeredBy,
    consecutiveCount: consecutiveDown, opKey, opLabel: o.label, outcome, exitCode: exitCodeR, stdoutTail });

  // Reset after any attempt so the next window must re-accumulate.
  await pool.query(
    `UPDATE service_heartbeat_state SET consecutive_down = 0 WHERE connection_id = $1 AND op_key = $2`,
    [o.connectionId, o.stateKey]
  ).catch(() => {});
  console.log(`[autopilot] ${opKey} → ${outcome} (exit ${exitCodeR}) conn ${o.connectionId}`);
}

// State-only heartbeat update (last_status/last_checked_at, no consecutive_down
// bump) — for services polled purely for the heartbeat table when NOT opted into
// auto-restart. Never throws.
async function updateHeartbeatStatusOnly(connectionId, opKey, isUp) {
  await pool.query(
    `INSERT INTO service_heartbeat_state (connection_id, op_key, consecutive_down, last_checked_at, last_status)
     VALUES ($1, $2, 0, NOW(), $3)
     ON CONFLICT (connection_id, op_key) DO UPDATE
       SET last_checked_at = NOW(), last_status = $3`,
    [connectionId, opKey, isUp ? 'up' : 'down']
  ).catch(() => {});
}

// ── PATH A: Heartbeat poll for one connection ────────────────────────────────
// Called from the 5-min cron for EVERY proxy connection. Always polls + updates
// service_heartbeat_state for visibility; only opted-in services (remediate_services)
// go through the restart engine.
async function runHeartbeatForConnection(conn) {
  const connectionId = conn.id;
  const threshold    = conn.remediate_threshold || 3;
  const serverType   = conn.server_type || null;
  // Restart only fires when auto_remediate is ON *and* the service is opted in.
  // auto_remediate OFF (or nothing opted in) → we still poll + update state below
  // for the heartbeat table, but restart nothing (fail-closed).
  const autoRemediate = conn.auto_remediate === true;
  const services      = Array.isArray(conn.remediate_services) ? conn.remediate_services : [];
  const appsPwd      = conn.apps_pwd_enc     ? decrypt(conn.apps_pwd_enc)     : '';
  const weblogicPwd  = conn.weblogic_pwd_enc ? decrypt(conn.weblogic_pwd_enc) : '';

  // Agent must be online before we poll.
  const online = await channel.isAgentConnected(connectionId).catch(() => false);
  if (!online) return;

  const adopActive = await isAdopActive(connectionId);

  // ── Simple services (one status op each; exit-code down signal) ────────────
  for (const svc of HEARTBEAT_SERVICES) {
    if (svc.requiresDiscovery) continue;                 // managed servers handled below
    if (svc.dbOnly && serverType !== 'db') continue;     // db_startup_open: db-tier only

    // Poll status. A non-success/timeout response is an UNKNOWN, never a "down"
    // signal — treating a proxy error as down would trigger spurious restarts.
    // wls_admin_status is slow, so it gets a longer poll timeout.
    let ok;
    try {
      const result = await channel.sendToAgent(connectionId, {
        method: 'POST',
        path: '/api/ebs-ctrl',
        body: { op: svc.statusOp, apps_pwd: appsPwd, weblogic_pwd: weblogicPwd },
      }, svc.isLongOp ? WLS_STATUS_TIMEOUT_MS : STATUS_TIMEOUT_MS);
      const body = (result && result.body) || {};
      if (!result || result.statusCode !== 200 || body.success !== true) {
        console.warn(`[autopilot/heartbeat] non-success poll conn ${connectionId} ${svc.statusOp}: http=${result && result.statusCode} success=${body.success}`);
        continue; // leave heartbeat state untouched
      }
      // Trust the proxy's ok determination (it uses the exact same logic as the HC,
      // oracle-proxy.py); never re-implement it in Node.
      ok = body.ok;
    } catch (pollErr) {
      console.warn(`[autopilot/heartbeat] poll error conn ${connectionId} ${svc.statusOp}: ${pollErr.message}`);
      continue;
    }

    // db_startup_open uses its OWN stateKey (svc.startOp) so it never shares the
    // db_listener_status counter with db_listener_start. A requiresSqlProbe service
    // adds a second-step gate: after the listener has been down for `threshold`
    // consecutive checks, SELECT 1 FROM DUAL decides — instance answers => skip
    // (skipped_instance_up), instance unreachable => fire the (no-op-if-up) STARTUP.
    // db_startup_open uses its OWN stateKey (svc.startOp) so it never shares the
    // db_listener_status counter with db_listener_start.
    const stateKey = svc.requiresSqlProbe ? svc.startOp : svc.statusOp;

    // The proxy is the source of truth: body.ok=false => down. (last_status becomes
    // body.ok ? 'up' : 'down' via updateHeartbeatStatusOnly(..., !isDown).)
    const isDown = !ok;

    // auto_remediate off OR not opted in → keep heartbeat state current (visibility)
    // but skip the restart engine. Keyed by stateKey (NOT statusOp) to avoid the
    // db_startup_open collision.
    if (!autoRemediate || !services.includes(svc.startOp)) {
      await updateHeartbeatStatusOnly(connectionId, stateKey, !isDown);
      continue;
    }

    const preFire  = svc.requiresSqlProbe
      ? (async () => (await probeSqlConnection(connectionId)) ? { skip: true, outcome: 'skipped_instance_up' } : null)
      : undefined;
    // LONG_OP services (WLS Admin) restart via the background-job model, not fireStartOp.
    const fireFn = svc.isLongOp
      ? (async () => fireWlsAdminRestart(connectionId, appsPwd, weblogicPwd))
      : undefined;

    await applyDownSignal({
      connectionId, stateKey, isDown, threshold, adopActive,
      startOp: svc.startOp, label: svc.label, triggeredBy: svc.statusOp, appsPwd, weblogicPwd, preFire, fireFn,
    });
  }

  // ── Managed servers (OACore/Forms/OAFM) — dynamic per-server discovery ─────
  await runManagedServerHeartbeat({ connectionId, services, threshold, adopActive, appsPwd, weblogicPwd, serverType, autoRemediate });
}

// One shared managed_servers_status poll covers every opted-in managed-server group;
// server names are discovered from the response (never hardcoded), each DOWN server
// restarted individually via managed_server_start(server_name).
async function runManagedServerHeartbeat({ connectionId, services, threshold, adopActive, appsPwd, weblogicPwd, serverType, autoRemediate }) {
  // Managed servers are app-tier only. Always poll (for the heartbeat table); the
  // restart is gated per group below.
  if (serverType !== 'apps' && serverType !== 'both') return;
  // managed_servers_status AND managed_server_start both need the WebLogic admin pw.
  if (!weblogicPwd) return;

  let stdout;
  try {
    const result = await channel.sendToAgent(connectionId, {
      method: 'POST',
      path: '/api/ebs-ctrl',
      body: { op: 'managed_servers_status', apps_pwd: appsPwd, weblogic_pwd: weblogicPwd },
    }, MANAGED_STATUS_TIMEOUT_MS);
    const body = (result && result.body) || {};
    if (!result || result.statusCode !== 200 || body.success !== true) {
      console.warn(`[autopilot/heartbeat] managed_servers_status non-success conn ${connectionId}: http=${result && result.statusCode}`);
      return;
    }
    stdout = body.stdout || '';
    // Can't evaluate → skip (never restart on doubt).
    if (/NO_WLS_PWD|NO_SERVERS_FOUND/.test(stdout)) return;
  } catch (pollErr) {
    console.warn(`[autopilot/heartbeat] managed_servers_status error conn ${connectionId}: ${pollErr.message}`);
    return;
  }

  const servers = parseServerNames(stdout);
  const groups = HEARTBEAT_SERVICES.filter(s => s.requiresDiscovery);
  for (const grp of groups) {
    const optedIn = autoRemediate && services.includes(grp.groupKey);
    for (const srv of servers) {
      const lc = srv.name.toLowerCase();
      if (lc.indexOf(grp.match) === -1) continue;           // not this group's server
      if (lc.indexOf('c4ws') !== -1 || lc.indexOf('oaea') !== -1) continue; // optional — never auto-restart
      // Only a clear DOWN/RUNNING is a signal; TIMEOUT / UNKNOWN / NOT_DEPLOYED are
      // ambiguous — leave state untouched, never act on doubt.
      if (srv.state !== 'DOWN' && srv.state !== 'RUNNING') continue;
      const isDown = srv.state === 'DOWN';
      // Not opted in → update heartbeat state for visibility, but don't restart.
      if (!optedIn) {
        await updateHeartbeatStatusOnly(connectionId, srv.name, !isDown);
        continue;
      }
      await applyDownSignal({
        connectionId, stateKey: srv.name, isDown, threshold, adopActive,
        startOp: grp.startOp, serverName: srv.name,
        label: 'Restart ' + srv.name + ' (' + grp.groupKey + ')',
        triggeredBy: 'managed_servers_status:' + srv.name, appsPwd, weblogicPwd,
      });
    }
  }
}

// ── PATH B: HC finding trigger ───────────────────────────────────────────────
// Called from services/schedule-runner.js after finding_history upsert. Only
// fires for check_ids in HC_REMEDIATION_MAP once occurrence_count ≥ threshold.
async function maybeRemediateFromHC({ connectionId, checkId, consecutiveCount }) {
  try {
    const mapping = HC_REMEDIATION_MAP[checkId];
    if (!mapping) return;

    const { rows: schedRows } = await pool.query(
      `SELECT cs.auto_remediate, cs.remediate_threshold, cs.remediate_services,
              oc.apps_pwd_enc, oc.weblogic_pwd_enc
       FROM connection_schedules cs
       JOIN oracle_connections oc ON oc.id = cs.connection_id
       WHERE cs.connection_id = $1`,
      [connectionId]
    );
    const sched = schedRows[0];
    if (!sched || !sched.auto_remediate) return;

    const threshold = sched.remediate_threshold || 3;
    if (consecutiveCount < threshold) return;

    // Only fire if the user opted this connection's CM (adcmctl_start) in.
    const services = Array.isArray(sched.remediate_services) ? sched.remediate_services : [];
    if (!services.includes(mapping.startOp)) return;

    if (await isAdopActive(connectionId)) {
      await logAction({ connectionId, triggerPath: 'hc_finding', triggeredBy: checkId,
        consecutiveCount, opKey: mapping.startOp, opLabel: mapping.label, outcome: 'skipped_adop' });
      return;
    }

    if (await isCoolingDown(connectionId, mapping.startOp)) return;

    const online = await channel.isAgentConnected(connectionId).catch(() => false);
    if (!online) {
      await logAction({ connectionId, triggerPath: 'hc_finding', triggeredBy: checkId,
        consecutiveCount, opKey: mapping.startOp, opLabel: mapping.label, outcome: 'skipped_no_agent' });
      return;
    }

    const appsPwd     = sched.apps_pwd_enc     ? decrypt(sched.apps_pwd_enc)     : '';
    const weblogicPwd = sched.weblogic_pwd_enc ? decrypt(sched.weblogic_pwd_enc) : '';

    console.log(`[autopilot/hc] Firing ${mapping.startOp} on conn ${connectionId} — ${checkId} down ${consecutiveCount}x`);

    let outcome = 'failed', exitCode = null, stdoutTail = null;
    try {
      const r = await fireStartOp(connectionId, mapping.startOp, appsPwd, weblogicPwd);
      exitCode = r.exitCode;
      stdoutTail = (r.stdout || '').slice(-500);
      outcome = r.ok ? 'success' : 'failed';
    } catch (err) {
      stdoutTail = err.message;
    }

    await logAction({ connectionId, triggerPath: 'hc_finding', triggeredBy: checkId,
      consecutiveCount, opKey: mapping.startOp, opLabel: mapping.label,
      outcome, exitCode, stdoutTail });

    console.log(`[autopilot/hc] ${mapping.startOp} → ${outcome} (exit ${exitCode}) conn ${connectionId}`);
  } catch (err) {
    console.error(`[autopilot/hc] error conn ${connectionId}:`, err.message);
  }
}

module.exports = { runHeartbeatForConnection, maybeRemediateFromHC, HEARTBEAT_SERVICES, HC_REMEDIATION_MAP, RESTARTABLE_OPS, parseServerNames };
