0.2.0: job engine, VM actions, theme-aware styling
This commit is contained in:
1 parent
c62de24cc0
commit
485a267ffe
14 files changed
+901
-71
No files matched your search
+69
@@ -122,6 +122,75 @@ switch ($cmd) {
|
||||
}
|
||||
);
|
||||
|
||||
case 'action':
|
||||
$verb = $argv[1] ?? '';
|
||||
$name = $argv[2] ?? '';
|
||||
$confirm = '';
|
||||
foreach ($argv as $a) {
|
||||
if (str_starts_with($a, '--confirm=')) {
|
||||
$confirm = substr($a, 10);
|
||||
}
|
||||
}
|
||||
$check = vmm_action_check($verb, $name, $confirm);
|
||||
if (!$check['ok']) {
|
||||
if ($json) {
|
||||
vmm_json(['ok' => false, 'error' => $check['error']], 1);
|
||||
}
|
||||
fwrite(STDERR, 'refused: ' . $check['error'] . "\n");
|
||||
exit(1);
|
||||
}
|
||||
$job = vmm_job_new($verb, $name);
|
||||
out(
|
||||
['ok' => true, 'job' => $job],
|
||||
$json,
|
||||
static function (array $d): void {
|
||||
printf("job %s queued: %s %s\n", $d['job']['id'], $d['job']['verb'], $d['job']['vm']);
|
||||
}
|
||||
);
|
||||
|
||||
case 'job':
|
||||
$job = vmm_job_get($arg);
|
||||
if ($job === null) {
|
||||
if ($json) {
|
||||
vmm_json(['ok' => false, 'error' => 'job not found'], 404);
|
||||
}
|
||||
fwrite(STDERR, "job not found\n");
|
||||
exit(1);
|
||||
}
|
||||
$job['log'] = vmm_job_log($arg);
|
||||
out(
|
||||
['ok' => true, 'job' => $job],
|
||||
$json,
|
||||
static function (array $d): void {
|
||||
printf("job %s %s %s state=%s rc=%s\n", $d['job']['id'], $d['job']['verb'], $d['job']['vm'], $d['job']['state'], $d['job']['rc'] ?? '-');
|
||||
echo $d['job']['log'];
|
||||
}
|
||||
);
|
||||
|
||||
case 'jobs':
|
||||
$jobs = vmm_job_list(50, $arg);
|
||||
out(
|
||||
['ok' => true, 'jobs' => $jobs],
|
||||
$json,
|
||||
static function (array $d): void {
|
||||
foreach ($d['jobs'] as $j) {
|
||||
printf("%-32s %-8s %-8s %-10s rc=%s\n", $j['id'], $j['verb'], $j['vm'], $j['state'], $j['rc'] ?? '-');
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
case 'cancel':
|
||||
$res = vmm_job_cancel($arg);
|
||||
out($res, $json, static function (array $d): void {
|
||||
echo $d['ok'] ? "cancel requested\n" : ('error: ' . $d['error'] . "\n");
|
||||
});
|
||||
|
||||
case 'recover':
|
||||
$fixed = vmm_job_recover();
|
||||
out(['ok' => true, 'fixed' => $fixed], $json, static function (array $d): void {
|
||||
printf("%d interrupted job(s) marked failed\n", $d['fixed']);
|
||||
});
|
||||
|
||||
case 'json':
|
||||
vmm_json(vmm_selfcheck());
|
||||
// no break
|
||||
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
#!/usr/local/bin/php
|
||||
<?php
|
||||
/**
|
||||
* vmm-jobd - executes exactly one queued VM job, then exits.
|
||||
*
|
||||
* Started detached by vmm-ctl. Owns the per-VM lock while it runs, streams
|
||||
* stdout/stderr into state/jobs/<id>.log and records the final state.
|
||||
*/
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
require_once __DIR__ . '/vmm-lib.php';
|
||||
|
||||
$id = $argv[1] ?? '';
|
||||
$job = vmm_job_get($id);
|
||||
if ($job === null) {
|
||||
fwrite(STDERR, "job not found: {$id}\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
$verb = (string)$job['verb'];
|
||||
$name = (string)$job['vm'];
|
||||
if (!isset(VMM_ACTIONS[$verb])) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'unknown verb';
|
||||
$job['ended'] = time();
|
||||
vmm_job_write($job);
|
||||
exit(2);
|
||||
}
|
||||
|
||||
$logFile = VMM_JOBDIR . '/' . $id . '.log';
|
||||
$log = fopen($logFile, 'ab');
|
||||
if ($log === false) {
|
||||
exit(2);
|
||||
}
|
||||
|
||||
function joblog($log, string $line): void
|
||||
{
|
||||
fwrite($log, '[' . date('H:i:s') . '] ' . $line . "\n");
|
||||
fflush($log);
|
||||
}
|
||||
|
||||
// one lock per VM (destroy/start/stop of the same guest never overlap)
|
||||
vmm_ensure_state_dirs();
|
||||
$lockPath = VMM_LOCKDIR . '/' . preg_replace('/[^A-Za-z0-9._-]/', '_', $name) . '.lock';
|
||||
$lock = fopen($lockPath, 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'another job holds the lock for this vm';
|
||||
$job['ended'] = time();
|
||||
vmm_job_write($job);
|
||||
joblog($log, 'lock busy: ' . $lockPath);
|
||||
fclose($log);
|
||||
exit(3);
|
||||
}
|
||||
|
||||
$job['state'] = 'running';
|
||||
$job['pid'] = getmypid();
|
||||
$job['started'] = time();
|
||||
vmm_job_write($job);
|
||||
|
||||
$argvVm = array_merge(VMM_ACTIONS[$verb]['argv'], [$name]);
|
||||
joblog($log, 'start: vm ' . implode(' ', $argvVm));
|
||||
|
||||
$vm = vmm_resolve('vm');
|
||||
if (!$vm['available']) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'vm-bhyve not available';
|
||||
$job['ended'] = time();
|
||||
vmm_job_write($job);
|
||||
joblog($log, 'fatal: vm-bhyve not available');
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
fclose($log);
|
||||
exit(4);
|
||||
}
|
||||
|
||||
$env = getenv() ?: [];
|
||||
$env['LC_ALL'] = 'C';
|
||||
$grub = vmm_resolve('grub');
|
||||
if ($grub['source'] === 'bundle') {
|
||||
$env['PATH'] = VMM_ROOT . '/vendor/grub2-bhyve:' . ($env['PATH'] ?? '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin');
|
||||
}
|
||||
|
||||
$cmd = array_merge([$vm['path']], $argvVm);
|
||||
$spec = [
|
||||
0 => ['pipe', 'r'],
|
||||
1 => ['pipe', 'w'],
|
||||
2 => ['pipe', 'w'],
|
||||
];
|
||||
$proc = proc_open($cmd, $spec, $pipes, null, $env);
|
||||
if (!is_resource($proc)) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'proc_open failed';
|
||||
$job['ended'] = time();
|
||||
vmm_job_write($job);
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
fclose($log);
|
||||
exit(5);
|
||||
}
|
||||
fclose($pipes[0]);
|
||||
stream_set_blocking($pipes[1], false);
|
||||
stream_set_blocking($pipes[2], false);
|
||||
|
||||
$timeout = (int)(VMM_ACTIONS[$verb]['timeout'] ?? 300);
|
||||
$start = time();
|
||||
$timedOut = false;
|
||||
$cancelFile = VMM_JOBDIR . '/' . $id . '.cancel';
|
||||
|
||||
while (true) {
|
||||
$status = proc_get_status($proc);
|
||||
$out = (string)stream_get_contents($pipes[1]);
|
||||
$err = (string)stream_get_contents($pipes[2]);
|
||||
if ($out !== '') {
|
||||
fwrite($log, $out);
|
||||
fflush($log);
|
||||
}
|
||||
if ($err !== '') {
|
||||
fwrite($log, $err);
|
||||
fflush($log);
|
||||
}
|
||||
if (!$status['running']) {
|
||||
break;
|
||||
}
|
||||
if (is_file($cancelFile)) {
|
||||
joblog($log, 'cancel requested, sending TERM');
|
||||
proc_terminate($proc, 15);
|
||||
usleep(300000);
|
||||
if (proc_get_status($proc)['running']) {
|
||||
proc_terminate($proc, 9);
|
||||
}
|
||||
@unlink($cancelFile);
|
||||
break;
|
||||
}
|
||||
if (time() - $start > $timeout) {
|
||||
$timedOut = true;
|
||||
joblog($log, "timeout after {$timeout}s, sending TERM");
|
||||
proc_terminate($proc, 15);
|
||||
usleep(500000);
|
||||
if (proc_get_status($proc)['running']) {
|
||||
proc_terminate($proc, 9);
|
||||
}
|
||||
break;
|
||||
}
|
||||
usleep(200000);
|
||||
}
|
||||
|
||||
fwrite($log, (string)stream_get_contents($pipes[1]));
|
||||
fwrite($log, (string)stream_get_contents($pipes[2]));
|
||||
fclose($pipes[1]);
|
||||
fclose($pipes[2]);
|
||||
$rc = proc_close($proc);
|
||||
|
||||
$stateAfter = vmm_vm_state($name);
|
||||
|
||||
// vm-bhyve often returns 0 even when it refused the operation, so the
|
||||
// resulting state is what counts.
|
||||
$expected = [
|
||||
'start' => 'running',
|
||||
'stop' => 'stopped',
|
||||
'restart' => 'running',
|
||||
'reset' => 'running',
|
||||
'poweroff' => 'stopped',
|
||||
'destroy' => 'absent',
|
||||
][$verb] ?? '';
|
||||
|
||||
$job['rc'] = $rc;
|
||||
$job['ended'] = time();
|
||||
$job['state_after'] = $stateAfter;
|
||||
if ($timedOut) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'timeout';
|
||||
} elseif ($rc !== 0) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'vm exited with code ' . $rc;
|
||||
} elseif ($expected !== '' && $stateAfter !== $expected) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = sprintf('unexpected state after %s: %s (expected %s)', $verb, $stateAfter, $expected);
|
||||
} else {
|
||||
$job['state'] = 'done';
|
||||
}
|
||||
vmm_job_write($job);
|
||||
joblog($log, sprintf('finished: rc=%d, state=%s', $rc, $stateAfter));
|
||||
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
fclose($log);
|
||||
exit(0);
|
||||
+249
@@ -472,3 +472,252 @@ function vmm_json(array $data, int $code = 0): never
|
||||
echo json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
exit;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* job engine */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const VMM_JOBDIR = VMM_ROOT . '/state/jobs';
|
||||
const VMM_LOCKDIR = VMM_ROOT . '/state/locks';
|
||||
const VMM_ACTIONS = [
|
||||
'start' => ['argv' => ['start'], 'timeout' => 300, 'needs' => 'stopped'],
|
||||
'stop' => ['argv' => ['stop'], 'timeout' => 180, 'needs' => 'running'],
|
||||
'restart' => ['argv' => ['restart'], 'timeout' => 300, 'needs' => 'running'],
|
||||
'reset' => ['argv' => ['reset', '-f'], 'timeout' => 120, 'needs' => 'running'],
|
||||
'poweroff' => ['argv' => ['poweroff', '-f'], 'timeout' => 120, 'needs' => 'running'],
|
||||
'destroy' => ['argv' => ['destroy', '-f'], 'timeout' => 300, 'needs' => 'stopped'],
|
||||
];
|
||||
|
||||
function vmm_ensure_state_dirs(): void
|
||||
{
|
||||
foreach ([VMM_JOBDIR, VMM_LOCKDIR] as $d) {
|
||||
if (!is_dir($d)) {
|
||||
@mkdir($d, 0750, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function vmm_vm_state(string $name): string
|
||||
{
|
||||
foreach (vmm_list() as $v) {
|
||||
if (($v['name'] ?? '') === $name) {
|
||||
return (string)$v['state'];
|
||||
}
|
||||
}
|
||||
return 'absent';
|
||||
}
|
||||
|
||||
function vmm_ram_budget(): array
|
||||
{
|
||||
$conf = vmm_config();
|
||||
$limitPct = (int)($conf['ram_budget_pct'] ?? 85);
|
||||
$total = 0;
|
||||
$out = trim((string)@shell_exec('/sbin/sysctl -n hw.physmem 2>/dev/null'));
|
||||
if ($out !== '' && ctype_digit($out)) {
|
||||
$total = (int)$out;
|
||||
}
|
||||
if ($total <= 0) {
|
||||
$total = 8 * 1024 ** 3;
|
||||
}
|
||||
return ['total' => $total, 'limit' => (int)($total * $limitPct / 100), 'pct' => $limitPct];
|
||||
}
|
||||
|
||||
function vmm_ram_used(): int
|
||||
{
|
||||
$used = 0;
|
||||
foreach (vmm_list() as $v) {
|
||||
if (($v['state'] ?? '') === 'running') {
|
||||
$used += vmm_size_to_bytes((string)($v['memory'] ?? '0'));
|
||||
}
|
||||
}
|
||||
return $used;
|
||||
}
|
||||
|
||||
function vmm_action_check(string $verb, string $name, string $confirm = ''): array
|
||||
{
|
||||
if (!isset(VMM_ACTIONS[$verb])) {
|
||||
return ['ok' => false, 'error' => 'unknown action'];
|
||||
}
|
||||
if (!vmm_valid_name($name)) {
|
||||
return ['ok' => false, 'error' => 'invalid vm name'];
|
||||
}
|
||||
$state = vmm_vm_state($name);
|
||||
if ($state === 'absent') {
|
||||
return ['ok' => false, 'error' => 'vm not found'];
|
||||
}
|
||||
$needs = VMM_ACTIONS[$verb]['needs'];
|
||||
if ($needs === 'running' && $state !== 'running') {
|
||||
return ['ok' => false, 'error' => "vm is {$state}, not running"];
|
||||
}
|
||||
if ($needs === 'stopped' && $state === 'running') {
|
||||
return ['ok' => false, 'error' => 'vm is running; stop it first'];
|
||||
}
|
||||
if ($verb === 'start') {
|
||||
$budget = vmm_ram_budget();
|
||||
$used = vmm_ram_used();
|
||||
$need = 0;
|
||||
foreach (vmm_list() as $v) {
|
||||
if (($v['name'] ?? '') === $name) {
|
||||
$need = vmm_size_to_bytes((string)($v['memory'] ?? '0'));
|
||||
}
|
||||
}
|
||||
if ($need > 0 && $used + $need > $budget['limit']) {
|
||||
return [
|
||||
'ok' => false,
|
||||
'error' => sprintf(
|
||||
'RAM budget: %s in use + %s requested exceeds limit of %s (%d%%)',
|
||||
vmm_human_bytes($used),
|
||||
vmm_human_bytes($need),
|
||||
vmm_human_bytes($budget['limit']),
|
||||
$budget['pct']
|
||||
),
|
||||
];
|
||||
}
|
||||
}
|
||||
if ($verb === 'destroy') {
|
||||
if ($confirm !== $name) {
|
||||
return ['ok' => false, 'error' => 'confirmation mismatch: type the vm name exactly'];
|
||||
}
|
||||
}
|
||||
return ['ok' => true, 'state' => $state];
|
||||
}
|
||||
|
||||
function vmm_job_path(string $id): string
|
||||
{
|
||||
return VMM_JOBDIR . '/' . $id . '.json';
|
||||
}
|
||||
|
||||
function vmm_job_write(array $job): void
|
||||
{
|
||||
$job['updated'] = time();
|
||||
$tmp = vmm_job_path($job['id']) . '.tmp';
|
||||
file_put_contents($tmp, json_encode($job, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
|
||||
rename($tmp, vmm_job_path($job['id']));
|
||||
}
|
||||
|
||||
function vmm_job_get(string $id): ?array
|
||||
{
|
||||
if (!preg_match('/^[0-9]{8}-[0-9]{6}-[a-f0-9]{8}$/', $id)) {
|
||||
return null;
|
||||
}
|
||||
$f = vmm_job_path($id);
|
||||
if (!is_file($f)) {
|
||||
return null;
|
||||
}
|
||||
$job = json_decode((string)file_get_contents($f), true);
|
||||
return is_array($job) ? $job : null;
|
||||
}
|
||||
|
||||
function vmm_job_list(int $limit = 50, string $vm = ''): array
|
||||
{
|
||||
$files = glob(VMM_JOBDIR . '/*.json') ?: [];
|
||||
rsort($files);
|
||||
$out = [];
|
||||
foreach ($files as $f) {
|
||||
$job = json_decode((string)file_get_contents($f), true);
|
||||
if (!is_array($job)) {
|
||||
continue;
|
||||
}
|
||||
if ($vm !== '' && ($job['vm'] ?? '') !== $vm) {
|
||||
continue;
|
||||
}
|
||||
$out[] = $job;
|
||||
if (count($out) >= $limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vmm_job_log(string $id, int $maxBytes = 65536): string
|
||||
{
|
||||
$f = VMM_JOBDIR . '/' . $id . '.log';
|
||||
if (!is_file($f)) {
|
||||
return '';
|
||||
}
|
||||
$size = filesize($f);
|
||||
if ($size <= $maxBytes) {
|
||||
return (string)file_get_contents($f);
|
||||
}
|
||||
$fh = fopen($f, 'rb');
|
||||
fseek($fh, $maxBytes * -1, SEEK_END);
|
||||
$data = (string)stream_get_contents($fh);
|
||||
fclose($fh);
|
||||
return "... (truncated)\n" . $data;
|
||||
}
|
||||
|
||||
function vmm_job_new(string $verb, string $name): array
|
||||
{
|
||||
vmm_ensure_state_dirs();
|
||||
$id = date('Ymd-His') . '-' . bin2hex(random_bytes(4));
|
||||
$job = [
|
||||
'id' => $id,
|
||||
'verb' => $verb,
|
||||
'vm' => $name,
|
||||
'state' => 'queued',
|
||||
'created' => time(),
|
||||
'started' => null,
|
||||
'ended' => null,
|
||||
'pid' => null,
|
||||
'rc' => null,
|
||||
'error' => '',
|
||||
'by' => (string)($_SERVER['REMOTE_ADDR'] ?? 'cli'),
|
||||
];
|
||||
vmm_job_write($job);
|
||||
@touch(VMM_JOBDIR . '/' . $id . '.log');
|
||||
|
||||
$php = PHP_BINARY !== '' ? PHP_BINARY : '/usr/local/bin/php';
|
||||
$cmd = sprintf(
|
||||
'/usr/bin/nohup %s %s %s > /dev/null 2>&1 &',
|
||||
escapeshellarg($php),
|
||||
escapeshellarg(VMM_ROOT . '/bin/vmm-jobd'),
|
||||
escapeshellarg($id)
|
||||
);
|
||||
exec($cmd);
|
||||
return $job;
|
||||
}
|
||||
|
||||
function vmm_job_recover(): int
|
||||
{
|
||||
$fixed = 0;
|
||||
foreach (vmm_job_list(200) as $job) {
|
||||
if (($job['state'] ?? '') !== 'running') {
|
||||
continue;
|
||||
}
|
||||
$pid = (int)($job['pid'] ?? 0);
|
||||
$alive = vmm_pid_alive($pid);
|
||||
if (!$alive) {
|
||||
$job['state'] = 'failed';
|
||||
$job['error'] = 'interrupted (process gone)';
|
||||
$job['ended'] = time();
|
||||
vmm_job_write($job);
|
||||
$fixed++;
|
||||
}
|
||||
}
|
||||
return $fixed;
|
||||
}
|
||||
|
||||
function vmm_pid_alive(int $pid): bool
|
||||
{
|
||||
if ($pid <= 0) {
|
||||
return false;
|
||||
}
|
||||
if (function_exists('posix_kill')) {
|
||||
return @posix_kill($pid, 0);
|
||||
}
|
||||
exec('/bin/kill -0 ' . $pid . ' > /dev/null 2>&1', $o, $rc);
|
||||
return $rc === 0;
|
||||
}
|
||||
|
||||
function vmm_job_cancel(string $id): array
|
||||
{
|
||||
$job = vmm_job_get($id);
|
||||
if ($job === null) {
|
||||
return ['ok' => false, 'error' => 'job not found'];
|
||||
}
|
||||
if (!in_array($job['state'], ['queued', 'running'], true)) {
|
||||
return ['ok' => false, 'error' => 'job already finished'];
|
||||
}
|
||||
@touch(VMM_JOBDIR . '/' . $id . '.cancel');
|
||||
return ['ok' => true, 'id' => $id];
|
||||
}
|
||||
Reference in new issue
Block a user