fix(ha): orchestration Keepalived root, support unicast VRRP, preservation identite backup et synchro HTTPS
This commit is contained in:
1 parent
93f842a363
commit
da0805fd5f
7 files changed
+302
-82
No files matched your search
+192
-69
@@ -16,6 +16,7 @@ class ClusterManager
|
||||
private PDO $db;
|
||||
public const KEEPALIVED_CONF_PATH = '/etc/keepalived/keepalived.conf';
|
||||
public const CLUSTER_LOG_PATH = '/var/log/nexus_cluster.log';
|
||||
public const CLUSTER_HELPER_PATH = '/var/www/nexus-pbx/scripts/nexus-cluster';
|
||||
|
||||
public function __construct(?PDO $db = null)
|
||||
{
|
||||
@@ -75,6 +76,22 @@ class ClusterManager
|
||||
throw new InvalidArgumentException("L'adresse IP Virtuelle partagée (VIP) est requise pour activer le cluster HA.");
|
||||
}
|
||||
|
||||
// Empêcher l'utilisateur de configurer l'IP locale de ce serveur comme IP du pair
|
||||
if ($peerIp !== '') {
|
||||
$localInterfaces = $this->detectNetworkInterfaces();
|
||||
foreach ($localInterfaces as $ifaceInfo) {
|
||||
if (($ifaceInfo['primary'] ?? '') === $peerIp) {
|
||||
throw new InvalidArgumentException("L'adresse IP du pair ({$peerIp}) correspond à l'adresse locale de cette machine ! Veuillez renseigner l'IP du second nœud.");
|
||||
}
|
||||
foreach ($ifaceInfo['ips'] ?? [] as $cidr) {
|
||||
$ipOnly = explode('/', $cidr)[0] ?? '';
|
||||
if ($ipOnly === $peerIp) {
|
||||
throw new InvalidArgumentException("L'adresse IP du pair ({$peerIp}) correspond à l'adresse locale de cette machine ! Veuillez renseigner l'IP du second nœud.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$stmt = $this->db->prepare('
|
||||
UPDATE cluster_settings SET
|
||||
enabled = :enabled,
|
||||
@@ -106,12 +123,12 @@ class ClusterManager
|
||||
':auto_sync' => $autoSync,
|
||||
]);
|
||||
|
||||
// Application automatique de la configuration Keepalived si demandé
|
||||
// Application automatique de la configuration Keepalived
|
||||
$applyResult = $this->applyKeepalivedConfig();
|
||||
|
||||
return [
|
||||
'success' => true,
|
||||
'message' => 'Configuration Haute Disponibilité enregistrée.',
|
||||
'message' => 'Configuration Haute Disponibilité enregistrée et appliquée.',
|
||||
'apply_result' => $applyResult,
|
||||
];
|
||||
}
|
||||
@@ -129,14 +146,13 @@ class ClusterManager
|
||||
if ($line === '') {
|
||||
continue;
|
||||
}
|
||||
// Format typique ip -o: "2: eth0 inet 10.10.0.4/16 brd 10.10.255.255 scope global eth0\..."
|
||||
if (preg_match('/^\d+:\s+([a-zA-Z0-9_\-\.]+)\s+inet\s+([0-9.]+)\/(\d+)/', $line, $matches)) {
|
||||
$iface = $matches[1];
|
||||
$ip = $matches[2];
|
||||
$mask = $matches[3];
|
||||
|
||||
if ($iface === 'lo') {
|
||||
continue; // Ignorer loopback
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isset($interfaces[$iface])) {
|
||||
@@ -162,9 +178,13 @@ class ClusterManager
|
||||
if ($vip === '') {
|
||||
return false;
|
||||
}
|
||||
$escapedVip = escapeshellarg($vip);
|
||||
$output = shell_exec("ip addr show 2>/dev/null | grep -F " . $escapedVip) ?: '';
|
||||
return trim($output) !== '';
|
||||
$output = shell_exec('ip -o -4 addr show 2>/dev/null') ?: '';
|
||||
foreach (preg_split('/\r?\n/', $output) ?: [] as $line) {
|
||||
if (preg_match('/\sinet\s+' . preg_quote($vip, '/') . '\//', $line)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -177,19 +197,23 @@ class ClusterManager
|
||||
$peerIp = $settings['peer_ip'] ?? '';
|
||||
$isVipActive = $this->isVirtualIpActive($vip);
|
||||
|
||||
// 1. Vérification de Keepalived
|
||||
// 1. Statut Keepalived
|
||||
$keepalivedInstalled = !empty(shell_exec('command -v keepalived 2>/dev/null'));
|
||||
$keepalivedRunning = false;
|
||||
if ($keepalivedInstalled) {
|
||||
$psOut = shell_exec('pidof keepalived 2>/dev/null') ?: '';
|
||||
$keepalivedRunning = trim($psOut) !== '';
|
||||
$helperStatus = trim((string)shell_exec('sudo -n ' . escapeshellcmd(self::CLUSTER_HELPER_PATH) . ' status 2>/dev/null'));
|
||||
if ($helperStatus === 'RUNNING' || trim((string)shell_exec('pidof keepalived 2>/dev/null')) !== '') {
|
||||
$keepalivedRunning = true;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Vérification d'Asterisk
|
||||
// 2. Statut Asterisk
|
||||
$asteriskRunning = trim((string)shell_exec('pidof asterisk 2>/dev/null')) !== '';
|
||||
$asteriskPing = false;
|
||||
if ($asteriskRunning) {
|
||||
$pingOut = shell_exec("asterisk -rx 'core ping' 2>/dev/null") ?: '';
|
||||
$pingOut = shell_exec("asterisk -rx 'core ping' 2>/dev/null")
|
||||
?: shell_exec("sudo -n /usr/sbin/asterisk -rx 'core ping' 2>/dev/null")
|
||||
?: '';
|
||||
$asteriskPing = str_contains($pingOut, 'Pong');
|
||||
}
|
||||
|
||||
@@ -207,7 +231,7 @@ class ClusterManager
|
||||
}
|
||||
}
|
||||
|
||||
// Rôle réel effectif
|
||||
// 4. Rôle effectif
|
||||
$effectiveState = 'STANDALONE';
|
||||
if (!empty($settings['enabled'])) {
|
||||
if ($isVipActive) {
|
||||
@@ -217,7 +241,7 @@ class ClusterManager
|
||||
}
|
||||
}
|
||||
|
||||
// Dernières lignes du log cluster
|
||||
// 5. Dernières lignes de journalisation HA
|
||||
$recentLogs = [];
|
||||
if (file_exists(self::CLUSTER_LOG_PATH)) {
|
||||
$lines = file(self::CLUSTER_LOG_PATH, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [];
|
||||
@@ -256,10 +280,20 @@ class ClusterManager
|
||||
$iface = $settings['interface'] ?? 'eth0';
|
||||
$vip = $settings['virtual_ip'] ?? '';
|
||||
$vmask = $settings['virtual_netmask'] ?? '24';
|
||||
$peerIp = trim((string)($settings['peer_ip'] ?? ''));
|
||||
$vrid = (int)($settings['vrid'] ?? 51);
|
||||
$prio = (int)($settings['priority'] ?? ($role === 'MASTER' ? 100 : 90));
|
||||
$secret = substr($settings['cluster_secret'] ?? 'NexusPBX', 0, 8); // Keepalived max 8 chars auth_pass
|
||||
|
||||
// Détection de l'IP locale sur l'interface pour unicast VRRP
|
||||
$localIp = '';
|
||||
foreach ($this->detectNetworkInterfaces() as $net) {
|
||||
if ($net['name'] === $iface) {
|
||||
$localIp = $net['primary'] ?? '';
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
$conf = [];
|
||||
$conf[] = "! ==============================================================================";
|
||||
$conf[] = "! Configuration Keepalived générée automatiquement par NexusPBX HA";
|
||||
@@ -273,7 +307,7 @@ class ClusterManager
|
||||
$conf[] = "}";
|
||||
$conf[] = "";
|
||||
$conf[] = "vrrp_script chk_asterisk {";
|
||||
$conf[] = " script \"/var/www/nexus-pbx/scripts/cluster/check_asterisk.sh\"";
|
||||
$conf[] = " script \"/etc/keepalived/scripts/check_asterisk.sh\"";
|
||||
$conf[] = " interval 2";
|
||||
$conf[] = " weight 2";
|
||||
$conf[] = " fall 2";
|
||||
@@ -287,6 +321,16 @@ class ClusterManager
|
||||
$conf[] = " priority {$prio}";
|
||||
$conf[] = " advert_int 1";
|
||||
$conf[] = "";
|
||||
|
||||
// Unicast VRRP si l'IP du nœud pair est configurée (indispensable en environnement virtualisé type Proxmox/VMware)
|
||||
if ($peerIp !== '' && $localIp !== '') {
|
||||
$conf[] = " unicast_src_ip {$localIp}";
|
||||
$conf[] = " unicast_peer {";
|
||||
$conf[] = " {$peerIp}";
|
||||
$conf[] = " }";
|
||||
$conf[] = "";
|
||||
}
|
||||
|
||||
$conf[] = " authentication {";
|
||||
$conf[] = " auth_type PASS";
|
||||
$conf[] = " auth_pass {$secret}";
|
||||
@@ -300,9 +344,9 @@ class ClusterManager
|
||||
$conf[] = " chk_asterisk";
|
||||
$conf[] = " }";
|
||||
$conf[] = "";
|
||||
$conf[] = " notify_master \"/var/www/nexus-pbx/scripts/cluster/notify_master.sh\"";
|
||||
$conf[] = " notify_backup \"/var/www/nexus-pbx/scripts/cluster/notify_backup.sh\"";
|
||||
$conf[] = " notify_fault \"/var/www/nexus-pbx/scripts/cluster/notify_fault.sh\"";
|
||||
$conf[] = " notify_master \"/etc/keepalived/scripts/notify_master.sh\"";
|
||||
$conf[] = " notify_backup \"/etc/keepalived/scripts/notify_backup.sh\"";
|
||||
$conf[] = " notify_fault \"/etc/keepalived/scripts/notify_fault.sh\"";
|
||||
$conf[] = "}";
|
||||
$conf[] = "";
|
||||
|
||||
@@ -310,7 +354,7 @@ class ClusterManager
|
||||
}
|
||||
|
||||
/**
|
||||
* Applique la configuration Keepalived sur le système Linux
|
||||
* Applique la configuration Keepalived sur le système Linux via le helper root
|
||||
*/
|
||||
public function applyKeepalivedConfig(): array
|
||||
{
|
||||
@@ -318,9 +362,7 @@ class ClusterManager
|
||||
$enabled = !empty($settings['enabled']);
|
||||
|
||||
if (!$enabled) {
|
||||
// Désactivation du service Keepalived si le cluster est désactivé
|
||||
shell_exec('systemctl stop keepalived 2>/dev/null || true');
|
||||
shell_exec('systemctl disable keepalived 2>/dev/null || true');
|
||||
shell_exec('sudo -n ' . escapeshellcmd(self::CLUSTER_HELPER_PATH) . ' stop 2>&1');
|
||||
return [
|
||||
'status' => 'disabled',
|
||||
'message' => 'Cluster HA désactivé (service Keepalived stoppé).',
|
||||
@@ -335,23 +377,21 @@ class ClusterManager
|
||||
}
|
||||
|
||||
$confContent = $this->generateKeepalivedConfig();
|
||||
if (!is_dir('/etc/keepalived')) {
|
||||
mkdir('/etc/keepalived', 0755, true);
|
||||
file_put_contents('/tmp/keepalived.conf.tmp', $confContent);
|
||||
chmod('/tmp/keepalived.conf.tmp', 0644);
|
||||
|
||||
$out = trim((string)shell_exec('sudo -n ' . escapeshellcmd(self::CLUSTER_HELPER_PATH) . ' apply 2>&1'));
|
||||
|
||||
if (!str_contains($out, 'KEEPALIVED_APPLIED_OK')) {
|
||||
return [
|
||||
'status' => 'warning',
|
||||
'message' => 'Configuration transmise mais le démarrage a retourné : ' . ($out ?: 'Erreur inconnue'),
|
||||
];
|
||||
}
|
||||
|
||||
file_put_contents(self::KEEPALIVED_CONF_PATH, $confContent);
|
||||
chmod(self::KEEPALIVED_CONF_PATH, 0600);
|
||||
|
||||
// S'assure que les scripts de cluster sont bien exécutables
|
||||
shell_exec('chmod +x /var/www/nexus-pbx/scripts/cluster/*.sh 2>/dev/null || true');
|
||||
|
||||
// Activation et rechargement / démarrage de Keepalived
|
||||
shell_exec('systemctl enable keepalived 2>/dev/null || true');
|
||||
shell_exec('systemctl restart keepalived 2>/dev/null || true');
|
||||
|
||||
return [
|
||||
'status' => 'active',
|
||||
'message' => 'Configuration Keepalived générée et service redémarré avec succès.',
|
||||
'message' => 'Configuration Keepalived générée et service activé avec succès.',
|
||||
];
|
||||
}
|
||||
|
||||
@@ -368,13 +408,24 @@ class ClusterManager
|
||||
throw new RuntimeException("L'adresse IP du Nœud pair (Peer IP) n'est pas configurée.");
|
||||
}
|
||||
|
||||
// Création du paquet de données en mémoire ou fichier temporaire
|
||||
// Port d'administration HTTPS configuré
|
||||
$adminPort = 8443;
|
||||
try {
|
||||
$st = $this->db->query("SELECT admin_https_port FROM settings WHERE id = 1")->fetch(PDO::FETCH_ASSOC);
|
||||
if (!empty($st['admin_https_port'])) {
|
||||
$adminPort = (int)$st['admin_https_port'];
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$adminPort = 8443;
|
||||
}
|
||||
|
||||
// Création de l'archive de réplication
|
||||
$dbPath = Database::getDatabasePath();
|
||||
if (!file_exists($dbPath)) {
|
||||
throw new RuntimeException("Fichier de base de données introuvable : {$dbPath}");
|
||||
}
|
||||
|
||||
// Forcer le checkpoint SQLite WAL pour avoir un dump 100% à jour
|
||||
// Forcer le checkpoint SQLite WAL pour un dump intègre
|
||||
$this->db->exec('PRAGMA wal_checkpoint(FULL);');
|
||||
|
||||
$tmpArchive = '/tmp/nexus_cluster_sync_' . time() . '.tar.gz';
|
||||
@@ -393,39 +444,63 @@ class ClusterManager
|
||||
@unlink($tmpArchive);
|
||||
|
||||
$payload = [
|
||||
'timestamp' => time(),
|
||||
'timestamp' => time(),
|
||||
'archive_b64' => base64_encode($archiveData),
|
||||
];
|
||||
|
||||
$signature = hash_hmac('sha256', json_encode($payload), $secret);
|
||||
$jsonPayload = json_encode($payload);
|
||||
$signature = hash_hmac('sha256', $jsonPayload, $secret);
|
||||
|
||||
$ch = curl_init("http://{$peerIp}/api.php?action=cluster_receive_sync");
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
||||
'Content-Type: application/json',
|
||||
'X-Nexus-Cluster-Signature: ' . $signature,
|
||||
]);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
|
||||
// URLs d'accès candidates vers l'API du second nœud
|
||||
$candidateUrls = [
|
||||
"https://{$peerIp}:{$adminPort}/api.php?action=cluster_receive_sync",
|
||||
];
|
||||
if ($adminPort !== 443) {
|
||||
$candidateUrls[] = "https://{$peerIp}/api.php?action=cluster_receive_sync";
|
||||
}
|
||||
if ($adminPort !== 8443) {
|
||||
$candidateUrls[] = "https://{$peerIp}:8443/api.php?action=cluster_receive_sync";
|
||||
}
|
||||
$candidateUrls[] = "http://{$peerIp}/api.php?action=cluster_receive_sync";
|
||||
$candidateUrls = array_values(array_unique($candidateUrls));
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$err = curl_error($ch);
|
||||
curl_close($ch);
|
||||
$lastError = '';
|
||||
$success = false;
|
||||
|
||||
if ($httpCode !== 200 || !empty($err)) {
|
||||
$errorMsg = "Échec d'envoi vers le pair {$peerIp} (HTTP {$httpCode}) : {$err}";
|
||||
$this->updateSyncStatus('error', $errorMsg);
|
||||
throw new RuntimeException($errorMsg);
|
||||
foreach ($candidateUrls as $targetUrl) {
|
||||
$ch = curl_init($targetUrl);
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, $jsonPayload);
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
||||
'Content-Type: application/json',
|
||||
'X-Nexus-Cluster-Signature: ' . $signature,
|
||||
]);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
|
||||
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 4);
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$err = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($httpCode === 200 && !empty($response)) {
|
||||
$resJson = json_decode((string)$response, true);
|
||||
if (!empty($resJson['success'])) {
|
||||
$success = true;
|
||||
break;
|
||||
}
|
||||
$lastError = "Rejet par le pair ({$targetUrl}) : " . ($resJson['message'] ?? $resJson['error'] ?? 'Erreur inconnue');
|
||||
} else {
|
||||
$lastError = "Échec d'envoi vers {$targetUrl} (HTTP {$httpCode})" . ($err ? " : {$err}" : "");
|
||||
}
|
||||
}
|
||||
|
||||
$resJson = json_decode((string)$response, true);
|
||||
if (empty($resJson['success'])) {
|
||||
$errorMsg = "Le pair a rejeté la réplication : " . ($resJson['error'] ?? 'Erreur inconnue');
|
||||
$this->updateSyncStatus('error', $errorMsg);
|
||||
throw new RuntimeException($errorMsg);
|
||||
if (!$success) {
|
||||
$this->updateSyncStatus('error', $lastError);
|
||||
throw new RuntimeException($lastError);
|
||||
}
|
||||
|
||||
$this->updateSyncStatus('success', "Synchronisation réussie avec le nœud {$peerIp}");
|
||||
@@ -463,13 +538,20 @@ class ClusterManager
|
||||
$tmpArchive = '/tmp/nexus_cluster_recv_' . time() . '.tar.gz';
|
||||
file_put_contents($tmpArchive, $archiveRaw);
|
||||
|
||||
// Sauvegarde d'urgence de la DB actuelle avant écrasement
|
||||
// Sauvegarde d'urgence de la base actuelle avant écrasement
|
||||
$dbPath = Database::getDatabasePath();
|
||||
if (file_exists($dbPath)) {
|
||||
copy($dbPath, $dbPath . '.pre_cluster_sync.bak');
|
||||
}
|
||||
|
||||
// Fermeture de la connexion PDO pour libérer les verrous SQLite
|
||||
// Sauvegarder la configuration locale cluster et réseau pour ne pas l'écraser par celle du master
|
||||
$localClusterSettings = $settings;
|
||||
$localSettingsRow = null;
|
||||
try {
|
||||
$localSettingsRow = $this->db->query("SELECT local_net, local_networks_json, admin_https_port FROM settings WHERE id = 1")->fetch(PDO::FETCH_ASSOC);
|
||||
} catch (\Throwable $e) {}
|
||||
|
||||
// Libérer les verrous PDO SQLite
|
||||
Database::resetConnection();
|
||||
|
||||
// Extraction de l'archive
|
||||
@@ -479,18 +561,60 @@ class ClusterManager
|
||||
}
|
||||
@unlink($tmpArchive);
|
||||
|
||||
// Réouverture de la connexion DB et régénération des configs Asterisk
|
||||
// Réouverture de la connexion DB
|
||||
$newDb = Database::getConnection();
|
||||
$this->db = $newDb;
|
||||
|
||||
// Préserver le rôle local configuré (pour ne pas qu'un master force l'esclave à se croire master)
|
||||
$localRole = $settings['role'] ?? 'backup';
|
||||
$newDb->exec("UPDATE cluster_settings SET role = '{$localRole}', last_sync_at = CURRENT_TIMESTAMP, last_sync_status = 'synced', last_sync_message = 'Base et sons synchronisés depuis le Master' WHERE id = 1");
|
||||
// Restauration de l'identité cluster et réseau propre à CE serveur
|
||||
$updateStmt = $newDb->prepare('
|
||||
UPDATE cluster_settings SET
|
||||
role = :role,
|
||||
interface = :iface,
|
||||
peer_ip = :peer_ip,
|
||||
priority = :priority,
|
||||
virtual_ip = :virtual_ip,
|
||||
virtual_netmask = :virtual_netmask,
|
||||
vrid = :vrid,
|
||||
last_sync_at = CURRENT_TIMESTAMP,
|
||||
last_sync_status = :status,
|
||||
last_sync_message = :msg
|
||||
WHERE id = 1
|
||||
');
|
||||
$updateStmt->execute([
|
||||
':role' => $localClusterSettings['role'] ?? 'backup',
|
||||
':iface' => $localClusterSettings['interface'] ?? 'eth0',
|
||||
':peer_ip' => $localClusterSettings['peer_ip'] ?? '',
|
||||
':priority' => $localClusterSettings['priority'] ?? 90,
|
||||
':virtual_ip' => $localClusterSettings['virtual_ip'] ?? '',
|
||||
':virtual_netmask' => $localClusterSettings['virtual_netmask'] ?? '24',
|
||||
':vrid' => $localClusterSettings['vrid'] ?? 51,
|
||||
':status' => 'synced',
|
||||
':msg' => 'Base et sons synchronisés depuis le Master',
|
||||
]);
|
||||
|
||||
if ($localSettingsRow) {
|
||||
$newDb->prepare('
|
||||
UPDATE settings SET
|
||||
local_net = :net,
|
||||
local_networks_json = :nets,
|
||||
admin_https_port = :port
|
||||
WHERE id = 1
|
||||
')->execute([
|
||||
':net' => $localSettingsRow['local_net'] ?? '192.168.1.0/24',
|
||||
':nets' => $localSettingsRow['local_networks_json'] ?? '["192.168.1.0/24"]',
|
||||
':port' => $localSettingsRow['admin_https_port'] ?? 8443,
|
||||
]);
|
||||
}
|
||||
|
||||
// Régénérer les configurations PJSIP et Asterisk
|
||||
$generator = new ConfigGenerator($newDb);
|
||||
$generator->generateAndApply();
|
||||
|
||||
// Si le cluster est actif, actualiser le service Keepalived
|
||||
if (!empty($localClusterSettings['enabled'])) {
|
||||
$this->applyKeepalivedConfig();
|
||||
}
|
||||
|
||||
return [
|
||||
'success' => true,
|
||||
'message' => 'Base de données et fichiers audio appliqués avec succès.',
|
||||
@@ -525,4 +649,3 @@ class ClusterManager
|
||||
$stmt->execute([':status' => $status, ':msg' => $message]);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user