0.2.1: fix job spawn (CLI php), transitional states, zfs hold release before destroy, API error logging
This commit is contained in:
1 parent
485a267ffe
commit
364ab42ffe
6 files changed
+183
-19
No files matched your search
+35
-13
@@ -82,6 +82,21 @@ if ($grub['source'] === 'bundle') {
|
|||||||
$env['PATH'] = VMM_ROOT . '/vendor/grub2-bhyve:' . ($env['PATH'] ?? '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin');
|
$env['PATH'] = VMM_ROOT . '/vendor/grub2-bhyve:' . ($env['PATH'] ?? '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// destroying a VM fails when backup tools (znapzend/zrepl) hold its snapshots
|
||||||
|
if ($verb === 'destroy') {
|
||||||
|
$ds = vmm_dataset_for_vm($name);
|
||||||
|
if ($ds !== '') {
|
||||||
|
joblog($log, 'checking zfs holds on ' . $ds);
|
||||||
|
$released = vmm_release_holds($ds);
|
||||||
|
foreach ($released as $r) {
|
||||||
|
joblog($log, 'released hold: ' . $r);
|
||||||
|
}
|
||||||
|
if ($released === []) {
|
||||||
|
joblog($log, 'no holds to release');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
$cmd = array_merge([$vm['path']], $argvVm);
|
$cmd = array_merge([$vm['path']], $argvVm);
|
||||||
$spec = [
|
$spec = [
|
||||||
0 => ['pipe', 'r'],
|
0 => ['pipe', 'r'],
|
||||||
@@ -152,18 +167,25 @@ fclose($pipes[1]);
|
|||||||
fclose($pipes[2]);
|
fclose($pipes[2]);
|
||||||
$rc = proc_close($proc);
|
$rc = proc_close($proc);
|
||||||
|
|
||||||
$stateAfter = vmm_vm_state($name);
|
|
||||||
|
|
||||||
// vm-bhyve often returns 0 even when it refused the operation, so the
|
// vm-bhyve often returns 0 even when it refused the operation, so the
|
||||||
// resulting state is what counts.
|
// resulting state is what counts. Some states are transitional (a guest that
|
||||||
$expected = [
|
// just booted reports "bootloader" for a moment), so wait a little.
|
||||||
'start' => 'running',
|
$accept = [
|
||||||
'stop' => 'stopped',
|
'start' => ['running', 'bootloader', 'starting'],
|
||||||
'restart' => 'running',
|
'stop' => ['stopped'],
|
||||||
'reset' => 'running',
|
'restart' => ['running', 'bootloader', 'starting'],
|
||||||
'poweroff' => 'stopped',
|
'reset' => ['running', 'bootloader'],
|
||||||
'destroy' => 'absent',
|
'poweroff' => ['stopped'],
|
||||||
][$verb] ?? '';
|
'destroy' => ['absent'],
|
||||||
|
][$verb] ?? [];
|
||||||
|
|
||||||
|
$stateAfter = vmm_vm_state($name);
|
||||||
|
$deadline = time() + 30;
|
||||||
|
while ($accept !== [] && !in_array($stateAfter, $accept, true) && time() < $deadline) {
|
||||||
|
usleep(500000);
|
||||||
|
$stateAfter = vmm_vm_state($name);
|
||||||
|
}
|
||||||
|
joblog($log, 'state after action: ' . $stateAfter);
|
||||||
|
|
||||||
$job['rc'] = $rc;
|
$job['rc'] = $rc;
|
||||||
$job['ended'] = time();
|
$job['ended'] = time();
|
||||||
@@ -174,9 +196,9 @@ if ($timedOut) {
|
|||||||
} elseif ($rc !== 0) {
|
} elseif ($rc !== 0) {
|
||||||
$job['state'] = 'failed';
|
$job['state'] = 'failed';
|
||||||
$job['error'] = 'vm exited with code ' . $rc;
|
$job['error'] = 'vm exited with code ' . $rc;
|
||||||
} elseif ($expected !== '' && $stateAfter !== $expected) {
|
} elseif ($accept !== [] && !in_array($stateAfter, $accept, true)) {
|
||||||
$job['state'] = 'failed';
|
$job['state'] = 'failed';
|
||||||
$job['error'] = sprintf('unexpected state after %s: %s (expected %s)', $verb, $stateAfter, $expected);
|
$job['error'] = sprintf('unexpected state after %s: %s (expected %s)', $verb, $stateAfter, implode('/', $accept));
|
||||||
} else {
|
} else {
|
||||||
$job['state'] = 'done';
|
$job['state'] = 'done';
|
||||||
}
|
}
|
||||||
|
|||||||
+61
-1
@@ -666,14 +666,20 @@ function vmm_job_new(string $verb, string $name): array
|
|||||||
vmm_job_write($job);
|
vmm_job_write($job);
|
||||||
@touch(VMM_JOBDIR . '/' . $id . '.log');
|
@touch(VMM_JOBDIR . '/' . $id . '.log');
|
||||||
|
|
||||||
$php = PHP_BINARY !== '' ? PHP_BINARY : '/usr/local/bin/php';
|
// the web SAPI is php-cgi; jobs must be started with the CLI binary
|
||||||
|
$php = is_file('/usr/local/bin/php') ? '/usr/local/bin/php' : PHP_BINARY;
|
||||||
$cmd = sprintf(
|
$cmd = sprintf(
|
||||||
'/usr/bin/nohup %s %s %s > /dev/null 2>&1 &',
|
'/usr/bin/nohup %s %s %s > /dev/null 2>&1 &',
|
||||||
escapeshellarg($php),
|
escapeshellarg($php),
|
||||||
escapeshellarg(VMM_ROOT . '/bin/vmm-jobd'),
|
escapeshellarg(VMM_ROOT . '/bin/vmm-jobd'),
|
||||||
escapeshellarg($id)
|
escapeshellarg($id)
|
||||||
);
|
);
|
||||||
|
if (function_exists('exec')) {
|
||||||
exec($cmd);
|
exec($cmd);
|
||||||
|
} else {
|
||||||
|
// exec() can be disabled in hardened PHP builds; shell_exec is enough here
|
||||||
|
@shell_exec($cmd);
|
||||||
|
}
|
||||||
return $job;
|
return $job;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -697,6 +703,60 @@ function vmm_job_recover(): int
|
|||||||
return $fixed;
|
return $fixed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function vmm_dataset_for_vm(string $name): string
|
||||||
|
{
|
||||||
|
$vmDir = vmm_vm_dir();
|
||||||
|
if ($vmDir === '') {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
$base = preg_replace('/^zfs:/', '', $vmDir);
|
||||||
|
$base = rtrim($base, '/');
|
||||||
|
return $base === '' ? '' : $base . '/' . $name;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Release ZFS holds that would block destroying a VM's datasets.
|
||||||
|
* Returns the list of released "snapshot tag" pairs.
|
||||||
|
*/
|
||||||
|
function vmm_release_holds(string $dataset): array
|
||||||
|
{
|
||||||
|
$released = [];
|
||||||
|
if ($dataset === '' || !preg_match('/^[A-Za-z0-9._:\/-]+$/', $dataset)) {
|
||||||
|
return $released;
|
||||||
|
}
|
||||||
|
|
||||||
|
// "zfs holds" only works on snapshots, so walk the snapshot tree first.
|
||||||
|
exec('/sbin/zfs list -H -o name -t snapshot -r ' . escapeshellarg($dataset) . ' 2>/dev/null', $snaps, $rc);
|
||||||
|
if ($rc !== 0) {
|
||||||
|
return $released;
|
||||||
|
}
|
||||||
|
$snaps = array_slice(array_filter(array_map('trim', $snaps)), 0, 2000);
|
||||||
|
|
||||||
|
foreach ($snaps as $snap) {
|
||||||
|
exec('/sbin/zfs holds ' . escapeshellarg($snap) . ' 2>/dev/null', $lines, $hrc);
|
||||||
|
if ($hrc !== 0 || $lines === []) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
foreach ($lines as $line) {
|
||||||
|
$line = trim($line);
|
||||||
|
if ($line === '' || stripos($line, 'NAME') === 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$parts = preg_split('/\s+/', $line);
|
||||||
|
if (count($parts) < 2) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
[$holdSnap, $tag] = $parts;
|
||||||
|
exec('/sbin/zfs release ' . escapeshellarg($tag) . ' ' . escapeshellarg($holdSnap) . ' 2>&1', $out, $r);
|
||||||
|
if ($r === 0) {
|
||||||
|
$released[] = $holdSnap . ' ' . $tag;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$lines = [];
|
||||||
|
}
|
||||||
|
return $released;
|
||||||
|
}
|
||||||
|
|
||||||
function vmm_pid_alive(int $pid): bool
|
function vmm_pid_alive(int $pid): bool
|
||||||
{
|
{
|
||||||
if ($pid <= 0) {
|
if ($pid <= 0) {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"appname": "VM Manager",
|
"appname": "VM Manager",
|
||||||
"version": "0.2.0",
|
"version": "0.2.1",
|
||||||
"rootfolder": "/mnt/Daten/extensions/vmmngr",
|
"rootfolder": "/mnt/Daten/extensions/vmmngr",
|
||||||
"vm_source": "auto",
|
"vm_source": "auto",
|
||||||
"bridges": ["ix0bridge"],
|
"bridges": ["ix0bridge"],
|
||||||
|
|||||||
+20
-2
@@ -11,7 +11,25 @@
|
|||||||
credentials: 'same-origin',
|
credentials: 'same-origin',
|
||||||
headers: {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'},
|
headers: {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'},
|
||||||
body: new URLSearchParams(data).toString()
|
body: new URLSearchParams(data).toString()
|
||||||
}).then(function (r) { return r.json(); });
|
}).then(function (r) { return readJson(r); });
|
||||||
|
}
|
||||||
|
|
||||||
|
function readJson(r) {
|
||||||
|
// a redirected request means the XigmaNAS session expired; the final
|
||||||
|
// response is usually an empty login page, not JSON
|
||||||
|
if (r.redirected || r.status === 0) {
|
||||||
|
return {ok: false, error: 'session expired - please reload the page and log in again'};
|
||||||
|
}
|
||||||
|
return r.text().then(function (t) {
|
||||||
|
if (!t) {
|
||||||
|
return {ok: false, error: 'empty response (HTTP ' + r.status + ')'};
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return JSON.parse(t);
|
||||||
|
} catch (e) {
|
||||||
|
return {ok: false, error: 'HTTP ' + r.status + ': ' + t.slice(0, 300)};
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function poll(id, box) {
|
function poll(id, box) {
|
||||||
@@ -20,7 +38,7 @@
|
|||||||
var logEl = box.querySelector('pre');
|
var logEl = box.querySelector('pre');
|
||||||
var timer = setInterval(function () {
|
var timer = setInterval(function () {
|
||||||
fetch('vmmngr_api.php?action=job&id=' + encodeURIComponent(id), {credentials: 'same-origin'})
|
fetch('vmmngr_api.php?action=job&id=' + encodeURIComponent(id), {credentials: 'same-origin'})
|
||||||
.then(function (r) { return r.json(); })
|
.then(function (r) { return readJson(r); })
|
||||||
.then(function (d) {
|
.then(function (d) {
|
||||||
if (!d.ok) { return; }
|
if (!d.ok) { return; }
|
||||||
var j = d.job;
|
var j = d.job;
|
||||||
|
|||||||
+65
-1
@@ -7,10 +7,44 @@
|
|||||||
* POST ?action=cancel (id)
|
* POST ?action=cancel (id)
|
||||||
*
|
*
|
||||||
* Requires an authenticated XigmaNAS session with admin rights.
|
* Requires an authenticated XigmaNAS session with admin rights.
|
||||||
|
* Any failure is reported as JSON and written to log/api-error.log, because
|
||||||
|
* the web SAPI runs with display_errors=Off and log_errors=Off.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
require_once 'vmmngr-boot.inc';
|
require_once 'vmmngr-boot.inc';
|
||||||
|
|
||||||
|
error_reporting(E_ALL);
|
||||||
|
ini_set('display_errors', '0');
|
||||||
|
ini_set('log_errors', '0');
|
||||||
|
|
||||||
|
function vmm_api_log(string $msg): void
|
||||||
|
{
|
||||||
|
@file_put_contents(VMM_ROOT . '/log/api-error.log', date('c') . ' ' . $msg . "\n", FILE_APPEND);
|
||||||
|
}
|
||||||
|
|
||||||
|
set_error_handler(static function (int $no, string $str, string $file = '', int $line = 0): bool {
|
||||||
|
if (error_reporting() & $no) {
|
||||||
|
vmm_api_log(sprintf('warning: %s in %s:%d', $str, $file, $line));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
|
||||||
|
register_shutdown_function(static function (): void {
|
||||||
|
$e = error_get_last();
|
||||||
|
if ($e === null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!in_array($e['type'], [E_ERROR, E_PARSE, E_CORE_ERROR, E_COMPILE_ERROR, E_USER_ERROR], true)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vmm_api_log(sprintf('fatal: %s in %s:%d', $e['message'], $e['file'], $e['line']));
|
||||||
|
if (!headers_sent()) {
|
||||||
|
http_response_code(500);
|
||||||
|
header('Content-Type: application/json; charset=utf-8');
|
||||||
|
}
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'internal error: ' . $e['message']]);
|
||||||
|
});
|
||||||
|
|
||||||
if (!session::is_admin()) {
|
if (!session::is_admin()) {
|
||||||
vmm_json(['ok' => false, 'error' => 'forbidden'], 403);
|
vmm_json(['ok' => false, 'error' => 'forbidden'], 403);
|
||||||
}
|
}
|
||||||
@@ -21,7 +55,8 @@ $name = (string)($_REQUEST['vm'] ?? $_REQUEST['name'] ?? '');
|
|||||||
$confirm = (string)($_REQUEST['confirm'] ?? '');
|
$confirm = (string)($_REQUEST['confirm'] ?? '');
|
||||||
$id = (string)($_REQUEST['id'] ?? '');
|
$id = (string)($_REQUEST['id'] ?? '');
|
||||||
|
|
||||||
switch ($action) {
|
try {
|
||||||
|
switch ($action) {
|
||||||
case 'list':
|
case 'list':
|
||||||
$vms = vmm_list();
|
$vms = vmm_list();
|
||||||
vmm_json(['ok' => true, 'count' => count($vms), 'vms' => $vms]);
|
vmm_json(['ok' => true, 'count' => count($vms), 'vms' => $vms]);
|
||||||
@@ -76,6 +111,35 @@ switch ($action) {
|
|||||||
vmm_json(['ok' => true, 'jobs' => vmm_job_list(30, $name)]);
|
vmm_json(['ok' => true, 'jobs' => vmm_job_list(30, $name)]);
|
||||||
// no break
|
// no break
|
||||||
|
|
||||||
|
case 'diag':
|
||||||
|
vmm_json([
|
||||||
|
'ok' => true,
|
||||||
|
'diag' => [
|
||||||
|
'php_binary' => PHP_BINARY,
|
||||||
|
'cli_binary' => is_file('/usr/local/bin/php') ? '/usr/local/bin/php' : '',
|
||||||
|
'sapi' => PHP_SAPI,
|
||||||
|
'exec' => function_exists('exec'),
|
||||||
|
'shell_exec' => function_exists('shell_exec'),
|
||||||
|
'proc_open' => function_exists('proc_open'),
|
||||||
|
'disable_functions' => (string)ini_get('disable_functions'),
|
||||||
|
'open_basedir' => (string)ini_get('open_basedir'),
|
||||||
|
'root' => VMM_ROOT,
|
||||||
|
'writable_state' => is_writable(VMM_ROOT . '/state'),
|
||||||
|
'writable_log' => is_writable(VMM_ROOT . '/log'),
|
||||||
|
'uid' => function_exists('posix_getuid') ? posix_getuid() : -1,
|
||||||
|
'vm' => vmm_resolve('vm'),
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
// no break
|
||||||
|
|
||||||
default:
|
default:
|
||||||
vmm_json(['ok' => false, 'error' => 'unknown action'], 400);
|
vmm_json(['ok' => false, 'error' => 'unknown action'], 400);
|
||||||
|
}
|
||||||
|
} catch (Throwable $e) {
|
||||||
|
vmm_api_log(sprintf('exception: %s in %s:%d', $e->getMessage(), $e->getFile(), $e->getLine()));
|
||||||
|
if (!headers_sent()) {
|
||||||
|
http_response_code(500);
|
||||||
|
header('Content-Type: application/json; charset=utf-8');
|
||||||
|
}
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'exception: ' . $e->getMessage()]);
|
||||||
}
|
}
|
||||||
+1
-1
@@ -1 +1 @@
|
|||||||
0.2.0
|
0.2.1
|
||||||
Reference in new issue
Block a user