Merge develop branch, see detail commit

This commit containers the following changes:
- Fix fd forwarding in exec
- Fix exists_exec algorithm for path to executable contains symlinks parent
- Fix mount destination when using as alias
- Add Device directive for changing devfs rules

ipc/src/packet/codec/mod.rs

@@ -153,6 +153,7 @@ impl<T: FromPacket> FromPacket for List<T> {
 }
 
 /// Like Option<T> but without Serialize/Deserialize trait
+#[derive(Debug)]
 pub enum Maybe<T: FromPacket> {
     Some(T),
     None,

xc-bin/src/jailfile/directives/mod.rs

@@ -44,7 +44,7 @@ pub(crate) trait Directive: Sized {
     fn up_to_date(&self) -> bool;
 }
 
-#[allow(dead_code)]
+//#[allow(dead_code)]
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) enum ConfigMod {
     Allow(Vec<String>),
@@ -61,6 +61,7 @@ pub(crate) enum ConfigMod {
     Mount(String, String),
     SysV(Vec<String>),
     AddEnv(Var, EnvSpec),
+    Device(InterpolatedString),
 }
 
 impl ConfigMod {
@@ -198,6 +199,9 @@ impl ConfigMod {
                     }
                 }
             }
+            Self::Device(device) => {
+                config.devfs_rules.push(device.clone());
+            }
             _ => {}
         }
     }
@@ -212,6 +216,7 @@ impl ConfigMod {
             "WORKDIR",
             "ENTRYPOINT",
             "CMD",
+            "DEVICE",
         ]
     }
 }
@@ -283,6 +288,12 @@ impl Directive for ConfigMod {
                 let mountpoint = action.args.get(1).context("cannot get mountpoint")?;
                 Ok(ConfigMod::Mount(fstype.to_string(), mountpoint.to_string()))
             }
+            "DEVICE" => {
+                let joined = action.args.join(" ");
+                let interpolated = InterpolatedString::new(&joined)
+                    .context("invalid value")?;
+                Ok(ConfigMod::Device(interpolated))
+            }
             _ => unreachable!(),
         }
     }

xc-bin/src/main.rs

@@ -607,6 +607,27 @@ fn main() -> Result<(), ActionError> {
             };
 
             if let Ok(res) = res {
+                eprintln!("required_cleanerce: {:?}", res.require_clearence.clone());
+
+                if !res.require_clearence.is_empty() {
+                    println!("this container require exposing these additional device nodes (y/n):");
+                    for dev in res.require_clearence.iter() {
+                        println!("    {dev}");
+                    }
+
+                    let mut s = String::new();
+                    std::io::stdin().read_line(&mut s).expect("cannot read user input");
+                    if s.to_lowercase().starts_with('y') {
+                        let req = ContinueInstantiateRequest {
+                            id: res.id.to_string(),
+                            clearences: res.require_clearence.clone()
+                        };
+                        _ = do_continue_instantiate(&mut conn, req)?;
+                    } else {
+                        std::process::exit(0)
+                    }
+                }
+
                 for publish in publish.iter() {
                     let redirection = publish.to_host_spec();
                     let req = DoRdr {
@@ -770,10 +791,8 @@ fn main() -> Result<(), ActionError> {
                 if let Some(socket) = response.terminal_socket {
                     _ = attach::run(socket);
                 }
-                /*
                 let exit = n.notified_sync_take_value();
                 std::process::exit((exit.unwrap_or(2) - 1) as i32)
-                */
             }
         }
         Action::Volume(action) => {

xc/Cargo.toml

@@ -15,6 +15,7 @@ chrono = "0.4.23"
 futures = "0.3.25"
 freebsd = { path = "../freebsd", features = ["tokio"] }
 ipc = { path = "../ipc" }
+ipc-macro = { path = "../ipc-macro" }
 ipcidr = { path = "../ipcidr" }
 jail = "*"
 nix = { version = "0.25.0", features = ["term", "process", "event"] }

xc/src/container/runner/mod.rs

@@ -33,13 +33,14 @@ use crate::container::process::*;
 use crate::container::running::RunningContainer;
 use crate::container::{ContainerManifest, ProcessStat};
 use crate::elf::{brand_elf_if_unsupported, ElfBrand};
-use crate::models::exec::{Jexec, StdioMode};
+use crate::models::exec::{Jexec, StdioMode, IpcJexec};
 use crate::models::network::HostEntry;
 use crate::util::{epoch_now_nano, exists_exec};
 
 use anyhow::Context;
 use freebsd::event::{EventFdNotify, KEventExt};
 use freebsd::FreeBSDCommandExt;
+use ipc::packet::codec::FromPacket;
 use ipc::packet::codec::json::JsonPacket;
 use jail::process::Jailed;
 use nix::libc::intptr_t;
@@ -71,15 +72,8 @@ pub struct ProcessRunner {
 
     control_streams: HashMap<i32, ControlStream>,
 
-    //    created: Option<u64>,
-    //
-    //    /// This field records the epoch seconds when the container is "started", which defined by a
-    //    /// container that has completed its init-routine
-    //    started: Option<u64>,
-    //
-    //    finished_at: Option<u64>,
     /// If `auto_start` is true, the container executes its init routine automatically after
-    /// creation
+    /// create
     auto_start: bool,
 
     container: RunningContainer,
@@ -177,13 +171,7 @@ impl ProcessRunner {
         let exec_path = Path::new(&exec);
 
         if exec_path.is_absolute() {
-            let mut path = root.clone();
-            for component in exec_path.components() {
-                if component != Component::RootDir {
-                    path.push(component);
-                }
-            }
-            exists_exec(root, path, 64).unwrap()
+            exists_exec(root, exec_path, 64).unwrap()
         } else {
             env_path
                 .split(':')
@@ -393,12 +381,11 @@ impl ProcessRunner {
         use ipc::transport::PacketTransport;
 
         let packet = if method == "exec" {
-            let jexec: Jexec = serde_json::from_value(request.data.clone()).with_context(|| {
-                format!(
-                    "cannot deserialize request data, expected Jexec, got {}",
-                    request.data
-                )
-            })?;
+
+            let jexec = IpcJexec::from_packet_failable(request, |value| serde_json::from_value(value.clone()))
+                .context("cannot deserialize jexec")?;
+
+            let jexec = jexec.to_local();
 
             let notify = Arc::new(EventFdNotify::from_fd(jexec.notify.unwrap()));
             let result = self.spawn_process(&crate::util::gen_id(), &jexec, Some(notify), None);

xc/src/models/exec.rs

@@ -21,7 +21,10 @@
 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.
+
 use super::resolve_environ_order;
+use ipc::packet::codec::{Maybe, Fd, FromPacket};
+use ipc_macro::FromPacket;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::os::fd::RawFd;
@@ -48,6 +51,88 @@ pub enum StdioMode {
     },
 }
 
+pub enum IpcStdioMode {
+    Terminal,
+    Inherit,
+    Files {
+        stdout: Option<PathBuf>,
+        stderr: Option<PathBuf>,
+    },
+    Forward {
+        stdin: Maybe<Fd>,
+        stdout: Maybe<Fd>,
+        stderr: Maybe<Fd>
+    }
+}
+
+impl IpcStdioMode {
+    pub fn to_local(self) -> StdioMode {
+        match self {
+            Self::Terminal => StdioMode::Terminal,
+            Self::Inherit => StdioMode::Inherit,
+            Self::Files { stdout, stderr } => StdioMode::Files { stdout, stderr },
+            Self::Forward { stdin, stdout, stderr } => {
+                StdioMode::Forward {
+                    stdin: stdin.to_option().map(|fd| fd.0),
+                    stdout: stdout.to_option().map(|fd| fd.0),
+                    stderr: stderr.to_option().map(|fd| fd.0),
+                }
+            }
+        }
+    }
+}
+
+impl FromPacket for IpcStdioMode {
+    // we are abusing the `StdioMode` struct, in this mode, we store offset to the raw fd instead
+    // of the raw fds
+    type Dual = StdioMode;
+    fn encode_to_dual(self, fds: &mut Vec<i32>) -> Self::Dual {
+        match self {
+            Self::Terminal => StdioMode::Terminal,
+            Self::Inherit => StdioMode::Inherit,
+            Self::Files { stdout, stderr } => StdioMode::Files { stdout, stderr },
+            Self::Forward { stdin, stdout, stderr } => {
+                let stdin = if let Maybe::Some(fd) = stdin {
+                    let idx = fds.len();
+                    fds.push(fd.0);
+                    Some(idx as i32)
+                } else {
+                    None
+                };
+                let stdout = if let Maybe::Some(fd) = stdout {
+                    let idx = fds.len();
+                    fds.push(fd.0);
+                    Some(idx as i32)
+                } else {
+                    None
+                };
+                let stderr = if let Maybe::Some(fd) = stderr {
+                    let idx = fds.len();
+                    fds.push(fd.0);
+                    Some(idx as i32)
+                } else {
+                    None
+                };
+                StdioMode::Forward { stdin, stdout, stderr }
+            }
+        }
+    }
+    fn decode_from_dual(value: Self::Dual, fds: &[RawFd]) -> Self {
+        match value {
+            StdioMode::Terminal => Self::Terminal,
+            StdioMode::Inherit => Self::Inherit,
+            StdioMode::Files { stdout, stderr } => Self::Files { stdout, stderr },
+            StdioMode::Forward { stdin, stdout, stderr } => {
+                Self::Forward {
+                    stdin: Maybe::from_option(stdin.map(|idx| Fd(*fds.get(idx as usize).unwrap()))),
+                    stdout: Maybe::from_option(stdout.map(|idx| Fd(*fds.get(idx as usize).unwrap()))),
+                    stderr: Maybe::from_option(stderr.map(|idx| Fd(*fds.get(idx as usize).unwrap()))),
+                }
+            }
+        }
+    }
+}
+
 /// Executable parameters to be executed in container
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Jexec {
@@ -63,6 +148,37 @@ pub struct Jexec {
     pub work_dir: Option<String>,
 }
 
+#[derive(FromPacket)]
+pub struct IpcJexec {
+    pub arg0: String,
+    pub args: Vec<String>,
+    pub envs: std::collections::HashMap<String, String>,
+    pub uid: Option<u32>,
+    pub gid: Option<u32>,
+    pub user: Option<String>,
+    pub group: Option<String>,
+    pub output_mode: IpcStdioMode,
+    pub notify: Maybe<Fd>,
+    pub work_dir: Option<String>,
+}
+
+impl IpcJexec {
+    pub fn to_local(self) -> Jexec {
+        Jexec {
+            arg0: self.arg0,
+            args: self.args,
+            envs: self.envs,
+            uid: self.uid,
+            gid: self.gid,
+            user: self.user,
+            group: self.group,
+            work_dir: self.work_dir,
+            notify: self.notify.to_option().map(|fd| fd.0),
+            output_mode: self.output_mode.to_local()
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 pub struct Exec {
     pub exec: String,

xc/src/util.rs

@@ -153,60 +153,47 @@ fn _realpath(
     if path.as_ref().is_relative() {
         Err(std::io::Error::new(
             std::io::ErrorKind::Other,
-            "path connot be relative path",
+            format!("path cannot be relative path: {:?}", path.as_ref()).as_str(),
         ))?;
     }
 
-    let mut path = path.as_ref().to_path_buf();
+    let path = path.as_ref().to_path_buf();
     let mut current = PathBuf::new();
     let mut real_path = root.clone();
     let mut directed = 0;
 
-    'p: while max_redirect > directed {
-        let mut components: VecDeque<_> = path.components().map(PathComp::from).collect();
+    let mut components: VecDeque<_> = path.components().map(PathComp::from).collect();
 
-        while let Some(head) = components.pop_front() {
-            if max_redirect <= directed {
-                return Ok(None);
+    while let Some(head) = components.pop_front() {
+        if max_redirect <= directed {
+            return Ok(None);
+        }
+        match head {
+            PathComp::RootDir => {
+                current.push(head);
+                real_path = root.clone();
             }
-            match head {
-                PathComp::RootDir => {
-                    current.push(head);
-                    real_path = root.clone();
-                }
-                PathComp::CurDir => continue,
-                PathComp::ParentDir => {
-                    if !current.pop() {
-                        return Ok(None);
-                    }
-                    real_path.pop();
-                }
-                PathComp::Normal(ent) => {
-                    current.push(&ent);
-                    real_path.push(&ent);
+            PathComp::CurDir => continue,
+            PathComp::ParentDir => {
+                if !current.pop() {
+                    return Ok(None);
                 }
+                real_path.pop();
             }
-
-            if real_path.is_symlink() {
-                directed += 1;
-                let link = real_path.read_link()?;
-                let link_components = link.components();
-                if link.is_relative() {
-                    for component in link_components.rev() {
-                        components.push_front(PathComp::from(component));
-                    }
-                } else {
-                    path = link.to_path_buf();
-                    current = PathBuf::new();
-                    real_path = root.clone();
-                    continue 'p;
-                }
-            } else {
-                continue;
+            PathComp::Normal(ent) => {
+                current.push(&ent);
+                real_path.push(&ent);
             }
         }
 
-        break;
+        if real_path.is_symlink() {
+            directed += 1;
+            let link = real_path.read_link()?;
+            let link_components = link.components();
+            for component in link_components.rev() {
+                components.push_front(PathComp::from(component));
+            }
+        }
     }
     Ok(Some(real_path))
 }
@@ -224,41 +211,17 @@ pub fn exists_exec(
     path: impl AsRef<Path>,
     max_redirect: usize,
 ) -> Result<Option<PathBuf>, std::io::Error> {
-    let root = root.as_ref();
-    let mut path = path.as_ref().to_path_buf();
-    let mut redirected = 0usize;
+    let file = _realpath(root, path, max_redirect).and_then(|path| {
+        path.ok_or(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "invalid path",
+        ))
+    })?;
 
-    if path.is_relative() {
-        panic!("path cannot be relative path");
-    }
-
-    while path.is_symlink() {
-        if max_redirect == redirected {
-            break;
-        }
-        let link = path.read_link()?;
-        if link.is_relative() {
-            let mut old_path = path.to_path_buf();
-            for component in link.components() {
-                old_path.push(component);
-            }
-            path = old_path;
-        } else {
-            let mut old_path = root.to_path_buf();
-            for component in link.components() {
-                if component != Component::RootDir {
-                    old_path.push(component);
-                }
-            }
-            path = old_path;
-        }
-        redirected += 1;
-    }
-
-    if path.exists() && path.is_file() && path.starts_with(root) {
-        Ok(Some(path))
-    } else {
+    if !file.exists() || !file.is_file() {
         Ok(None)
+    } else {
+        Ok(Some(file))
     }
 }
 

xcd/src/context.rs

@@ -81,6 +81,8 @@ pub struct ServerContext {
     pub(crate) jail2ngs: HashMap<String, Vec<String>>,
 
     pub(crate) resources: Arc<RwLock<Resources>>,
+
+    pub(crate) ins_queue: HashMap<String, AppliedInstantiateRequest>,
 }
 
 impl ServerContext {
@@ -122,6 +124,7 @@ impl ServerContext {
             ng2jails: HashMap::new(),
             jail2ngs: HashMap::new(),
             resources,
+            ins_queue: HashMap::new(),
         }
     }
 
@@ -532,45 +535,28 @@ impl ServerContext {
         Ok(())
     }
 
-    /// Create a new site with id and instantiate a container in the site
-    pub(crate) async fn instantiate(
+    pub(crate) async fn continue_instantiate(
         this: Arc<RwLock<Self>>,
         id: &str,
-        image: &JailImage,
-        request: InstantiateRequest,
+        applied: AppliedInstantiateRequest,
         cred: Credential,
     ) -> anyhow::Result<()> {
-        let no_clean = request.no_clean;
-
-        context_provider::enter_instantiate!(|| &request);
+        let no_clean = applied.request.no_clean;
+        let name = applied.request.name.clone();
 
         let (site, notify) = {
             let this = this.clone();
             let mut this = this.write().await;
+            let res_ref = this.resources.clone();
+            let mut res = res_ref.write().await;
+
             let mut site = Site::new(id, this.config());
-
-            site.stage(image)?;
-            let name = request.name.clone();
-
-            let resources_ref = this.resources.clone();
-            let mut resources = resources_ref.write().await;
-
-            let applied =
-                { AppliedInstantiateRequest::new(request, image, &cred, &mut resources)? };
-
-            let blueprint = {
-                InstantiateBlueprint::new(
-                    id,
-                    image,
-                    applied,
-                    &mut this.devfs_store,
-                    &cred,
-                    &mut resources,
-                )?
-            };
+            site.stage(&applied.image)?;
+            let blueprint =
+                InstantiateBlueprint::new(id, applied, &mut this.devfs_store, &cred, &mut res)?;
 
             if pf::is_pf_enabled().unwrap_or_default() {
-                if let Some(map) = resources.get_allocated_addresses(id) {
+                if let Some(map) = res.get_allocated_addresses(id) {
                     for (network, addresses) in map.iter() {
                         let table = format!("xc:network:{network}");
                         let result = pf::table_add_addresses(None, &table, addresses);
@@ -625,9 +611,38 @@ impl ServerContext {
                 }
             }
         });
+
         Ok(())
     }
 
+    /// Create a new site with id and instantiate a container in the site
+    pub(crate) async fn instantiate(
+        this: Arc<RwLock<Self>>,
+        id: &str,
+        image: &JailImage,
+        request: InstantiateRequest,
+        cred: Credential,
+    ) -> anyhow::Result<Vec<String>> {
+        context_provider::enter_instantiate!(|| &request);
+
+        let applied = {
+            let this = this.clone();
+            let this = this.write().await;
+            let resources_ref = this.resources.clone();
+            let mut resources = resources_ref.write().await;
+
+            AppliedInstantiateRequest::new(request, image, &cred, &mut resources)?
+        };
+        if !applied.devfs_rules.is_empty() {
+            let rules = applied.devfs_rules.iter().map(|r| r.to_string()).collect();
+            this.write().await.ins_queue.insert(id.to_string(), applied);
+            Ok(rules)
+        } else {
+            Self::continue_instantiate(this, id, applied, cred).await?;
+            Ok(Vec::new())
+        }
+    }
+
     pub(crate) async fn push_image(
         &self,
         reference: ImageReference,

xcd/src/instantiate.rs

@@ -46,8 +46,8 @@ use xc::models::network::{DnsSetting, IpAssign};
 use xc::models::EnforceStatfs;
 
 pub struct AppliedInstantiateRequest {
-    base: InstantiateRequest,
-    devfs_rules: Vec<DevfsRule>,
+    pub(crate) request: InstantiateRequest,
+    pub(crate) devfs_rules: Vec<DevfsRule>,
     init: Vec<Jexec>,
     deinit: Vec<Jexec>,
     main: Option<Jexec>,
@@ -55,6 +55,7 @@ pub struct AppliedInstantiateRequest {
     allowing: Vec<String>,
     copies: Vec<xc::container::request::CopyFileReq>,
     enforce_statfs: EnforceStatfs,
+    pub(crate) image: JailImage,
 }
 
 impl AppliedInstantiateRequest {
@@ -238,7 +239,7 @@ impl AppliedInstantiateRequest {
         }
 
         Ok(AppliedInstantiateRequest {
-            base: request,
+            request,
             copies,
             devfs_rules,
             init,
@@ -247,6 +248,7 @@ impl AppliedInstantiateRequest {
             envs,
             allowing,
             enforce_statfs,
+            image: oci_config.clone(),
         })
     }
 }
@@ -293,20 +295,15 @@ pub struct InstantiateBlueprint {
 impl InstantiateBlueprint {
     pub(crate) fn new(
         id: &str,
-        oci_config: &JailImage,
         request: AppliedInstantiateRequest,
         devfs_store: &mut DevfsRulesetStore,
         cred: &Credential,
-        /*
-        network_manager: &mut NetworkManager,
-        volume_manager: &VolumeManager,
-        dataset_tracker: &mut JailedDatasetTracker,
-        */
         resources: &mut Resources,
     ) -> anyhow::Result<InstantiateBlueprint> {
+        let oci_config = &request.image;
         let existing_ifaces = freebsd::net::ifconfig::interfaces()?;
         let config = oci_config.jail_config();
-        let name = match request.base.name {
+        let name = match request.request.name {
             None => format!("xc-{id}"),
             Some(name) => {
                 if name.parse::<isize>().is_ok() {
@@ -318,8 +315,8 @@ impl InstantiateBlueprint {
                 }
             }
         };
-        let hostname = request.base.hostname.unwrap_or_else(|| name.to_string());
-        let vnet = request.base.vnet || config.vnet;
+        let hostname = request.request.hostname.unwrap_or_else(|| name.to_string());
+        let vnet = request.request.vnet || config.vnet;
         let envs = request.envs.clone();
 
         if config.linux {
@@ -333,16 +330,16 @@ impl InstantiateBlueprint {
             }
         }
 
-        let main_started_notify = match request.base.main_started_notify {
+        let main_started_notify = match request.request.main_started_notify {
             ipc::packet::codec::Maybe::None => None,
             ipc::packet::codec::Maybe::Some(x) => Some(EventFdNotify::from_fd(x.as_raw_fd())),
         };
 
-        let mut ip_alloc = request.base.ips.clone();
+        let mut ip_alloc = request.request.ips.clone();
 
         let mut default_router = None;
 
-        for req in request.base.ipreq.iter() {
+        for req in request.request.ipreq.iter() {
             match resources.allocate(vnet, req, id) {
                 Ok((alloc, router)) => {
                     if !existing_ifaces.contains(&alloc.interface) {
@@ -391,7 +388,7 @@ impl InstantiateBlueprint {
         let mut mount_specs = oci_config.jail_config().mounts;
         let mut added_mount_specs = HashMap::new();
 
-        for req in request.base.mount_req.clone().to_vec().iter() {
+        for req in request.request.mount_req.clone().to_vec().iter() {
             let source_path = std::path::Path::new(&req.source);
 
             let volume = if !source_path.is_absolute() {
@@ -412,20 +409,15 @@ impl InstantiateBlueprint {
                 match &req.evid {
                     Maybe::None => errx!(ENOENT, "missing evidence"),
                     Maybe::Some(fd) => {
-                        println!("process to check evidence");
                         let Ok(stat) = freebsd::nix::sys::stat::fstat(fd.as_raw_fd()) else {
                             println!("cannot stat evidence");
                             errx!(ENOENT, "cannot stat evidence")
                         };
                         let check_stat = freebsd::nix::sys::stat::stat(source_path).unwrap();
-                        println!("c: {}", stat.st_ino);
-                        println!("n: {}", check_stat.st_ino);
                         if stat.st_ino != check_stat.st_ino {
                             errx!(ENOENT, "evidence inode mismatch")
                         }
-
                         freebsd::nix::unistd::close(fd.as_raw_fd());
-
                         Volume::adhoc(source_path)
                     }
                 }
@@ -441,7 +433,7 @@ impl InstantiateBlueprint {
             mount_req.push(mount);
         }
 
-        for dataset in request.base.jail_datasets.iter() {
+        for dataset in request.request.jail_datasets.iter() {
             if resources.dataset_tracker.is_jailed(dataset) {
                 errx!(
                     EPERM,
@@ -469,8 +461,10 @@ impl InstantiateBlueprint {
 
         let devfs_ruleset_id = devfs_store.get_ruleset_id(&devfs_rules);
 
+        println!("devfs_ruleset_id: {devfs_ruleset_id}");
+
         let extra_layers = request
-            .base
+            .request
             .extra_layers
             .to_vec()
             .into_iter()
@@ -486,34 +480,34 @@ impl InstantiateBlueprint {
             deinit: request.deinit,
             extra_layers,
             main: request.main,
-            ips: request.base.ips,
-            ipreq: request.base.ipreq,
+            ips: request.request.ips,
+            ipreq: request.request.ipreq,
             mount_req,
             linux: config.linux,
-            deinit_norun: request.base.deinit_norun,
-            init_norun: request.base.init_norun,
-            main_norun: request.base.main_norun,
-            persist: request.base.persist,
-            no_clean: request.base.no_clean,
-            dns: request.base.dns,
+            deinit_norun: request.request.deinit_norun,
+            init_norun: request.request.init_norun,
+            main_norun: request.request.main_norun,
+            persist: request.request.persist,
+            no_clean: request.request.no_clean,
+            dns: request.request.dns,
             origin_image: Some(oci_config.clone()),
             allowing: request.allowing,
-            image_reference: Some(request.base.image_reference),
+            image_reference: Some(request.request.image_reference),
             copies: request.copies,
             envs,
             ip_alloc,
             devfs_ruleset_id,
             default_router,
             main_started_notify,
-            create_only: request.base.create_only,
-            linux_no_create_sys_dir: request.base.linux_no_create_sys_dir,
-            linux_no_create_proc_dir: request.base.linux_no_create_proc_dir,
-            linux_no_mount_sys: request.base.linux_no_mount_sys,
-            linux_no_mount_proc: request.base.linux_no_mount_proc,
-            override_props: request.base.override_props,
+            create_only: request.request.create_only,
+            linux_no_create_sys_dir: request.request.linux_no_create_sys_dir,
+            linux_no_create_proc_dir: request.request.linux_no_create_proc_dir,
+            linux_no_mount_sys: request.request.linux_no_mount_sys,
+            linux_no_mount_proc: request.request.linux_no_mount_proc,
+            override_props: request.request.override_props,
             enforce_statfs: request.enforce_statfs,
-            jailed_datasets: request.base.jail_datasets,
-            children_max: request.base.children_max,
+            jailed_datasets: request.request.jail_datasets,
+            children_max: request.request.children_max,
         })
     }
 }

xcd/src/ipc.rs

@@ -51,7 +51,7 @@ use tokio::sync::RwLock;
 use tracing::*;
 use xc::container::request::NetworkAllocRequest;
 use xc::image_store::ImageStoreError;
-use xc::models::exec::{Jexec, StdioMode};
+use xc::models::exec::{Jexec, StdioMode, IpcJexec, IpcStdioMode};
 use xc::models::jail_image::JailConfig;
 use xc::models::network::{DnsSetting, IpAssign, PortRedirection};
 use xc::util::{gen_id, CompressionFormat, CompressionFormatExt};
@@ -299,6 +299,7 @@ impl Default for InstantiateRequest {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct InstantiateResponse {
     pub id: String,
+    pub require_clearence: Vec<String>,
 }
 
 #[ipc_method(method = "instantiate")]
@@ -331,20 +332,52 @@ async fn instantiate(
             let instantiate_result =
                 ServerContext::instantiate(context, &id, &image_row.manifest, request, credential)
                     .await;
-            if let Err(error) = instantiate_result {
-                tracing::error!("instantiate error: {error:#?}");
-                if let Some(err) = error.downcast_ref::<xc::container::error::Error>() {
-                    ipc_err(err.errno(), &err.error_message())
-                } else {
-                    enoent(error.to_string().as_str())
+            match instantiate_result {
+                Ok(cleanerces) => {
+                    Ok(InstantiateResponse { id, require_clearence: cleanerces })
+                },
+                Err(error) => {
+                    tracing::error!("instantiate error: {error:#?}");
+                    if let Some(err) = error.downcast_ref::<xc::container::error::Error>() {
+                        ipc_err(err.errno(), &err.error_message())
+                    } else {
+                        enoent(error.to_string().as_str())
+                    }
                 }
-            } else {
-                Ok(InstantiateResponse { id })
             }
         }
     }
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ContinueInstantiateRequest {
+    pub id: String,
+    pub clearences: Vec<String>,
+}
+
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ContinueInstantiateResponse {
+    pub id: String,
+}
+
+#[ipc_method(method = "continue_instantiate")]
+async fn continue_instantiate(
+    context: Arc<RwLock<ServerContext>>,
+    load_context: &mut ConnectionContext<Variables>,
+    request: ContinueInstantiateRequest,
+) -> GenericResult<ContinueInstantiateResponse> {
+    let Some(applied) = ({
+        context.clone().write().await.ins_queue.remove(&request.id)
+    }) else {
+        return enoent("no such instantiate request")
+    };
+    let credential = Credential::from_conn_ctx(local_context);
+    ServerContext::continue_instantiate(context, &request.id, applied, credential).await
+        .expect("todo");
+    Ok(ContinueInstantiateResponse { id: request.id })
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 pub struct UploadStat {
     pub image_reference: ImageReference,
@@ -999,7 +1032,7 @@ async fn exec(
         .clone()
         .and_then(|group| group.parse::<u32>().ok());
 
-    let jexec = Jexec {
+    let jexec = IpcJexec {
         arg0: request.arg0,
         args: request.args,
         envs: request.envs,
@@ -1008,15 +1041,15 @@ async fn exec(
         user: request.user.clone(),
         group: request.group.clone(),
         output_mode: if request.use_tty {
-            StdioMode::Terminal
+            IpcStdioMode::Terminal
         } else {
-            StdioMode::Forward {
-                stdin: request.stdin.to_option().map(|fd| fd.0),
-                stdout: request.stdout.to_option().map(|fd| fd.0),
-                stderr: request.stderr.to_option().map(|fd| fd.0),
+            IpcStdioMode::Forward {
+                stdin: request.stdin,
+                stdout: request.stdout,
+                stderr: request.stderr,
             }
         },
-        notify: request.notify.to_option().map(|fd| fd.0),
+        notify: request.notify,
         work_dir: None,
     };
     if let Some(arc_site) = context.write().await.get_site(&request.name) {
@@ -1247,6 +1280,7 @@ pub(crate) async fn register_to_service(
     service: &mut Service<tokio::sync::RwLock<ServerContext>, Variables>,
 ) {
     service.register_event_delegate(on_channel_closed).await;
+    service.register(continue_instantiate).await;
     service.register(list_rdr_rules).await;
     service.register(create_volume).await;
     service.register(list_volumes).await;

xcd/src/resources/volume/drivers/local.rs

@@ -117,9 +117,15 @@ impl VolumeDriver for LocalDriver {
             mount_options.insert(option.to_string());
         }
 
+        let real_dest = match mount_spec {
+            None => mount_req.dest.to_os_string(),
+            Some(spec) => spec.destination.as_os_str().to_os_string()
+        };
+
+
         Ok(Mount {
             options: Vec::from_iter(mount_options),
-            ..Mount::nullfs(source_path, &mount_req.dest)
+            ..Mount::nullfs(source_path, &real_dest)
         })
     }
 }

xcd/src/resources/volume/drivers/zfs.rs

@@ -137,9 +137,14 @@ impl VolumeDriver for ZfsDriver {
             mount_options.insert(option.to_string());
         }
 
+        let real_dest = match mount_spec {
+            None => mount_req.dest.to_os_string(),
+            Some(spec) => spec.destination.as_os_str().to_os_string()
+        };
+
         Ok(Mount {
             options: Vec::from_iter(mount_options),
-            ..Mount::nullfs(&mount_point, &mount_req.dest)
+            ..Mount::nullfs(&mount_point, &real_dest)
         })
     }
 }

xcd/src/site.rs

@@ -45,7 +45,7 @@ use tokio::sync::watch::Receiver;
 use tracing::{error, info};
 use xc::container::effect::UndoStack;
 use xc::container::{ContainerManifest, CreateContainer};
-use xc::models::exec::Jexec;
+use xc::models::exec::{Jexec, StdioMode, IpcJexec};
 use xc::models::jail_image::JailImage;
 use xc::models::network::HostEntry;
 
@@ -267,24 +267,20 @@ impl Site {
 
     pub fn exec(
         &mut self,
-        jexec: Jexec,
+        jexec: IpcJexec,
     ) -> ipc::proto::GenericResult<xc::container::process::SpawnInfo> {
         use ipc::packet::codec::FromPacket;
         use ipc::proto::ipc_err;
 
-        let value = serde_json::to_value(jexec).unwrap();
-        let request = Request {
-            method: "exec".to_string(),
-            value,
-        };
-        let encoded = serde_json::to_vec(&request).unwrap();
-        let packet = Packet {
-            data: encoded,
-            fds: Vec::new(),
-        };
+        let packet = jexec.to_packet(|dual| serde_json::to_value(dual).unwrap()).map(|value| {
+            Request {
+                method: "exec".to_string(),
+                value: value.clone()
+            }
+        }).map(|p| serde_json::to_vec(&p).unwrap());
 
         let Some(stream) = self.control_stream.as_mut() else {
-            return ipc_err(freebsd::libc::ENOENT, "no much control stream");
+            return ipc_err(freebsd::libc::ENOENT, "no such control stream");
         };
 
         let _result = stream.send_packet(&packet);