214 lines
5.8 KiB
PHP
214 lines
5.8 KiB
PHP
#!/usr/local/bin/php
|
|
<?php
|
|
/**
|
|
* vmm-jobd - executes exactly one queued VM job, then exits.
|
|
*
|
|
* Started detached by vmm-ctl. Owns the per-VM lock while it runs, streams
|
|
* stdout/stderr into state/jobs/<id>.log and records the final state.
|
|
*/
|
|
|
|
declare(strict_types=1);
|
|
|
|
require_once __DIR__ . '/vmm-lib.php';
|
|
|
|
$id = $argv[1] ?? '';
|
|
$job = vmm_job_get($id);
|
|
if ($job === null) {
|
|
fwrite(STDERR, "job not found: {$id}\n");
|
|
exit(2);
|
|
}
|
|
|
|
$verb = (string)$job['verb'];
|
|
$name = (string)$job['vm'];
|
|
if (!isset(VMM_ACTIONS[$verb])) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = 'unknown verb';
|
|
$job['ended'] = time();
|
|
vmm_job_write($job);
|
|
exit(2);
|
|
}
|
|
|
|
$logFile = VMM_JOBDIR . '/' . $id . '.log';
|
|
$log = fopen($logFile, 'ab');
|
|
if ($log === false) {
|
|
exit(2);
|
|
}
|
|
|
|
function joblog($log, string $line): void
|
|
{
|
|
fwrite($log, '[' . date('H:i:s') . '] ' . $line . "\n");
|
|
fflush($log);
|
|
}
|
|
|
|
// one lock per VM (destroy/start/stop of the same guest never overlap)
|
|
vmm_ensure_state_dirs();
|
|
$lockPath = VMM_LOCKDIR . '/' . preg_replace('/[^A-Za-z0-9._-]/', '_', $name) . '.lock';
|
|
$lock = fopen($lockPath, 'c');
|
|
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = 'another job holds the lock for this vm';
|
|
$job['ended'] = time();
|
|
vmm_job_write($job);
|
|
joblog($log, 'lock busy: ' . $lockPath);
|
|
fclose($log);
|
|
exit(3);
|
|
}
|
|
|
|
$job['state'] = 'running';
|
|
$job['pid'] = getmypid();
|
|
$job['started'] = time();
|
|
vmm_job_write($job);
|
|
|
|
$argvVm = isset($job['argv']) && is_array($job['argv']) && $job['argv'] !== []
|
|
? $job['argv']
|
|
: array_merge(VMM_ACTIONS[$verb]['argv'], [$name]);
|
|
joblog($log, 'start: vm ' . implode(' ', $argvVm));
|
|
|
|
$vm = vmm_resolve('vm');
|
|
if (!$vm['available']) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = 'vm-bhyve not available';
|
|
$job['ended'] = time();
|
|
vmm_job_write($job);
|
|
joblog($log, 'fatal: vm-bhyve not available');
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
fclose($log);
|
|
exit(4);
|
|
}
|
|
|
|
$env = getenv() ?: [];
|
|
$env['LC_ALL'] = 'C';
|
|
$grub = vmm_resolve('grub');
|
|
if ($grub['source'] === 'bundle') {
|
|
$env['PATH'] = VMM_ROOT . '/vendor/grub2-bhyve:' . ($env['PATH'] ?? '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin');
|
|
}
|
|
|
|
// destroying a VM fails when backup tools (znapzend/zrepl) hold its snapshots
|
|
if ($verb === 'destroy') {
|
|
$ds = vmm_dataset_for_vm($name);
|
|
if ($ds !== '') {
|
|
joblog($log, 'checking zfs holds on ' . $ds);
|
|
$released = vmm_release_holds($ds);
|
|
foreach ($released as $r) {
|
|
joblog($log, 'released hold: ' . $r);
|
|
}
|
|
if ($released === []) {
|
|
joblog($log, 'no holds to release');
|
|
}
|
|
}
|
|
}
|
|
|
|
$cmd = array_merge([$vm['path']], $argvVm);
|
|
$spec = [
|
|
0 => ['pipe', 'r'],
|
|
1 => ['pipe', 'w'],
|
|
2 => ['pipe', 'w'],
|
|
];
|
|
$proc = proc_open($cmd, $spec, $pipes, null, $env);
|
|
if (!is_resource($proc)) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = 'proc_open failed';
|
|
$job['ended'] = time();
|
|
vmm_job_write($job);
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
fclose($log);
|
|
exit(5);
|
|
}
|
|
fclose($pipes[0]);
|
|
stream_set_blocking($pipes[1], false);
|
|
stream_set_blocking($pipes[2], false);
|
|
|
|
$timeout = (int)($job['timeout'] ?? (VMM_ACTIONS[$verb]['timeout'] ?? 300));
|
|
$start = time();
|
|
$timedOut = false;
|
|
$cancelFile = VMM_JOBDIR . '/' . $id . '.cancel';
|
|
|
|
while (true) {
|
|
$status = proc_get_status($proc);
|
|
$out = (string)stream_get_contents($pipes[1]);
|
|
$err = (string)stream_get_contents($pipes[2]);
|
|
if ($out !== '') {
|
|
fwrite($log, $out);
|
|
fflush($log);
|
|
}
|
|
if ($err !== '') {
|
|
fwrite($log, $err);
|
|
fflush($log);
|
|
}
|
|
if (!$status['running']) {
|
|
break;
|
|
}
|
|
if (is_file($cancelFile)) {
|
|
joblog($log, 'cancel requested, sending TERM');
|
|
proc_terminate($proc, 15);
|
|
usleep(300000);
|
|
if (proc_get_status($proc)['running']) {
|
|
proc_terminate($proc, 9);
|
|
}
|
|
@unlink($cancelFile);
|
|
break;
|
|
}
|
|
if (time() - $start > $timeout) {
|
|
$timedOut = true;
|
|
joblog($log, "timeout after {$timeout}s, sending TERM");
|
|
proc_terminate($proc, 15);
|
|
usleep(500000);
|
|
if (proc_get_status($proc)['running']) {
|
|
proc_terminate($proc, 9);
|
|
}
|
|
break;
|
|
}
|
|
usleep(200000);
|
|
}
|
|
|
|
fwrite($log, (string)stream_get_contents($pipes[1]));
|
|
fwrite($log, (string)stream_get_contents($pipes[2]));
|
|
fclose($pipes[1]);
|
|
fclose($pipes[2]);
|
|
$rc = proc_close($proc);
|
|
|
|
// vm-bhyve often returns 0 even when it refused the operation, so the
|
|
// resulting state is what counts. Some states are transitional (a guest that
|
|
// just booted reports "bootloader" for a moment), so wait a little.
|
|
$accept = [
|
|
'start' => ['running', 'bootloader', 'starting'],
|
|
'stop' => ['stopped'],
|
|
'restart' => ['running', 'bootloader', 'starting'],
|
|
'reset' => ['running', 'bootloader'],
|
|
'poweroff' => ['stopped'],
|
|
'destroy' => ['absent'],
|
|
][$verb] ?? [];
|
|
|
|
$stateAfter = vmm_vm_state($name);
|
|
$deadline = time() + 30;
|
|
while ($accept !== [] && !in_array($stateAfter, $accept, true) && time() < $deadline) {
|
|
usleep(500000);
|
|
$stateAfter = vmm_vm_state($name);
|
|
}
|
|
joblog($log, 'state after action: ' . $stateAfter);
|
|
|
|
$job['rc'] = $rc;
|
|
$job['ended'] = time();
|
|
$job['state_after'] = $stateAfter;
|
|
if ($timedOut) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = 'timeout';
|
|
} elseif ($rc !== 0) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = 'vm exited with code ' . $rc;
|
|
} elseif ($accept !== [] && !in_array($stateAfter, $accept, true)) {
|
|
$job['state'] = 'failed';
|
|
$job['error'] = sprintf('unexpected state after %s: %s (expected %s)', $verb, $stateAfter, implode('/', $accept));
|
|
} else {
|
|
$job['state'] = 'done';
|
|
}
|
|
vmm_job_write($job);
|
|
joblog($log, sprintf('finished: rc=%d, state=%s', $rc, $stateAfter));
|
|
|
|
flock($lock, LOCK_UN);
|
|
fclose($lock);
|
|
fclose($log);
|
|
exit(0);
|