add jailed dataset support

2026-03-20 15:55:36 +01:00 · 2023-09-05 08:06:58 -04:00
parent 4955b8a821
commit b40562d28e
30 changed files with 667 additions and 211 deletions
--- a/freebsd/src/fs/zfs.rs
+++ b/freebsd/src/fs/zfs.rs
@@ -84,7 +84,7 @@ impl ZfsCreate {
            dataset: dataset.as_ref().to_path_buf(),
            create_ancestors,
            no_mount,
-            properties: HashMap::new()
+            properties: HashMap::new(),
        }
    }

@@ -292,10 +292,7 @@ impl ZfsHandle {
        })
    }

-    pub fn create(
-        &self,
-        arg: ZfsCreate
-    ) -> Result<()> {
+    pub fn create(&self, arg: ZfsCreate) -> Result<()> {
        self.use_command(|c| {
            c.arg("create");
            if arg.no_mount {
@@ -314,6 +311,21 @@ impl ZfsHandle {
        })
    }

+    pub fn jail(&self, jail: &str, dataset: impl AsRef<Path>) -> Result<()> {
+        self.use_command(|c| {
+            c.arg("jail");
+            c.arg(jail);
+            c.arg(dataset.as_ref());
+        })
+    }
+
+    pub fn unjail(&self, jail: &str, dataset: impl AsRef<Path>) -> Result<()> {
+        self.use_command(|c| {
+            c.arg("unjail");
+            c.arg(jail);
+            c.arg(dataset.as_ref());
+        })
+    }

    /// # Arguments
    /// * `dataset`
--- a/xc-bin/src/format/env_pair.rs
+++ b/xc-bin/src/format/env_pair.rs
@@ -21,3 +21,40 @@
 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.
+use std::io::{Error, ErrorKind};
+use std::path::PathBuf;
+use std::str::FromStr;
+use varutil::string_interpolation::Var;
+
+#[derive(Clone, Debug)]
+pub struct DatasetParam {
+    pub key: Option<Var>,
+    pub dataset: PathBuf,
+}
+
+impl FromStr for DatasetParam {
+    type Err = std::io::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.split_once(':') {
+            None => {
+                if s.starts_with('/') {
+                    return Err(Error::new(ErrorKind::Other, "invalid dataset path"));
+                }
+                let dataset = s
+                    .parse::<PathBuf>()
+                    .map_err(|_| Error::new(ErrorKind::Other, "invalid dataset path"))?;
+                Ok(DatasetParam { key: None, dataset })
+            }
+            Some((env, dataset)) => {
+                if dataset.starts_with('/') {
+                    return Err(Error::new(ErrorKind::Other, "invalid dataset path"));
+                }
+                let key = varutil::string_interpolation::Var::from_str(env)?;
+                Ok(DatasetParam {
+                    key: Some(key),
+                    dataset: dataset.into(),
+                })
+            }
+        }
+    }
+}
--- a/xc-bin/src/format/mod.rs
+++ b/xc-bin/src/format/mod.rs
@@ -21,7 +21,7 @@
 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.
-mod env_pair;
+pub(crate) mod dataset;

 use ipcidr::IpCidr;
 use pest::Parser;
--- a/xc-bin/src/jailfile/directives/mod.rs
+++ b/xc-bin/src/jailfile/directives/mod.rs
@@ -43,6 +43,7 @@ pub(crate) trait Directive: Sized {
    fn up_to_date(&self) -> bool;
 }

+#[allow(dead_code)]
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) enum ConfigMod {
    Allow(Vec<String>),
--- a/xc-bin/src/jailfile/directives/run.rs
+++ b/xc-bin/src/jailfile/directives/run.rs
@@ -40,6 +40,7 @@ use std::os::fd::AsRawFd;
 use tracing::{error, info, warn};
 use xcd::ipc::*;

+#[allow(dead_code)]
 #[derive(Debug)]
 enum Input {
    // XXX: File as input has not yet implemented
@@ -161,7 +162,7 @@ impl Directive for RunDirective {
                    Input::File(_file) => {
                        error!("using file as stdin has not been implemented");
                        todo!()
-                    },
+                    }
                    Input::Content(content) => Box::new(VecSlice::new(content.as_bytes())),
                };

@@ -194,7 +195,8 @@ impl Directive for RunDirective {
                                            }
                                            Ok(bytes) => {
                                                available -= bytes;
-                                                _ = std::io::stdout().write_all(&stdout_buf[..bytes]);
+                                                _ = std::io::stdout()
+                                                    .write_all(&stdout_buf[..bytes]);
                                            }
                                        }
                                    }
@@ -212,7 +214,8 @@ impl Directive for RunDirective {
                                            }
                                            Ok(bytes) => {
                                                available -= bytes;
-                                                _ = std::io::stderr().write_all(&stderr_buf[..bytes]);
+                                                _ = std::io::stderr()
+                                                    .write_all(&stderr_buf[..bytes]);
                                            }
                                        }
                                    }
--- a/xc-bin/src/jailfile/mod.rs
+++ b/xc-bin/src/jailfile/mod.rs
@@ -123,8 +123,9 @@ impl JailContext {
            }
            image.set_config(&config);

-            std::fs::rename(local_id, &response.commit_id);
-            std::fs::write("jail.json", serde_json::to_string_pretty(&image).unwrap());
+            // XXX: handle effect error
+            _ = std::fs::rename(local_id, &response.commit_id);
+            _ = std::fs::write("jail.json", serde_json::to_string_pretty(&image).unwrap());
        } else {
            crate::image::patch_image(&mut conn, &image_reference, |config| {
                for config_mod in config_mods.iter() {
--- a/xc-bin/src/jailfile/statefile.rs
+++ b/xc-bin/src/jailfile/statefile.rs
@@ -1,3 +1,4 @@
+#![allow(unused)]
 // Copyright (c) 2023 Yan Ka, Chiu.
 // All rights reserved.
 //
--- a/xc-bin/src/main.rs
+++ b/xc-bin/src/main.rs
@@ -553,6 +553,8 @@ fn main() -> Result<(), ActionError> {
            extra_layers,
            publish,
            image_reference,
+            props,
+            jail_dataset,
        }) => {
            let hostname = hostname.or_else(|| name.clone());

@@ -568,6 +570,7 @@ fn main() -> Result<(), ActionError> {
                        mount.source.to_string()
                    };
                    MountReq {
+                        read_only: false,
                        source,
                        dest: mount.destination.clone(),
                    }
@@ -589,7 +592,7 @@ fn main() -> Result<(), ActionError> {
                })
                .collect();

-            let envs = {
+            let mut envs = {
                let mut map = std::collections::HashMap::new();
                for env in envs.into_iter() {
                    map.insert(env.key, env.value);
@@ -597,6 +600,16 @@ fn main() -> Result<(), ActionError> {
                map
            };

+            let mut jail_datasets = Vec::new();
+
+            for spec in jail_dataset.into_iter() {
+                if let Some(key) = &spec.key {
+                    let path_str = spec.dataset.to_string_lossy().to_string();
+                    envs.insert(key.to_string(), path_str);
+                }
+                jail_datasets.push(spec.dataset);
+            }
+
            let mut extra_layer_files = Vec::new();

            for layer in extra_layers.iter() {
@@ -606,6 +619,14 @@ fn main() -> Result<(), ActionError> {
            let extra_layers =
                List::from_iter(extra_layer_files.iter().map(|file| Fd(file.as_raw_fd())));

+            let mut override_props = std::collections::HashMap::new();
+
+            for prop in props.iter() {
+                if let Some((key, value)) = prop.split_once('=') {
+                    override_props.insert(key.to_string(), value.to_string());
+                }
+            }
+
            let res = {
                let reqt = InstantiateRequest {
                    create_only: true,
@@ -625,6 +646,8 @@ fn main() -> Result<(), ActionError> {
                    main_norun: true,
                    init_norun: true,
                    deinit_norun: true,
+                    override_props,
+                    jail_datasets,
                    ..InstantiateRequest::default()
                };

@@ -669,6 +692,8 @@ fn main() -> Result<(), ActionError> {
            ips,
            user,
            group,
+            props,
+            jail_dataset,
        }) => {
            if detach && link {
                panic!("detach and link flags are mutually exclusive");
@@ -678,7 +703,7 @@ fn main() -> Result<(), ActionError> {
                panic!("--dns-nop and --empty-dns are mutually exclusive");
            }

-            let envs = {
+            let mut envs = {
                let mut map = std::collections::HashMap::new();
                for env in envs.into_iter() {
                    map.insert(env.key, env.value);
@@ -686,6 +711,16 @@ fn main() -> Result<(), ActionError> {
                map
            };

+            let mut jail_datasets = Vec::new();
+
+            for spec in jail_dataset.into_iter() {
+                if let Some(key) = &spec.key {
+                    let path_str = spec.dataset.to_string_lossy().to_string();
+                    envs.insert(key.to_string(), path_str);
+                }
+                jail_datasets.push(spec.dataset);
+            }
+
            let dns = if empty_dns {
                DnsSetting::Specified {
                    servers: Vec::new(),
@@ -717,6 +752,7 @@ fn main() -> Result<(), ActionError> {
                    };
                    MountReq {
                        source,
+                        read_only: false,
                        dest: mount.destination.clone(),
                    }
                })
@@ -745,6 +781,13 @@ fn main() -> Result<(), ActionError> {

            let extra_layers =
                List::from_iter(extra_layer_files.iter().map(|file| Fd(file.as_raw_fd())));
+            let mut override_props = std::collections::HashMap::new();
+
+            for prop in props.iter() {
+                if let Some((key, value)) = prop.split_once('=') {
+                    override_props.insert(key.to_string(), value.to_string());
+                }
+            }

            let (res, notify) = {
                let main_started_notify = if detach {
@@ -774,6 +817,8 @@ fn main() -> Result<(), ActionError> {
                    group,
                    ips: ips.into_iter().map(|v| v.0).collect(),
                    main_started_notify: main_started_notify.clone(),
+                    override_props,
+                    jail_datasets,
                    ..InstantiateRequest::default()
                };

@@ -844,7 +889,7 @@ fn main() -> Result<(), ActionError> {
                notify: notify.clone(),
            };

-            if let Ok(res) = do_run_main(&mut conn, req)? {
+            if let Ok(_res) = do_run_main(&mut conn, req)? {
                if !detach {
                    if let Maybe::Some(notify) = notify {
                        EventFdNotify::from_fd(notify.as_raw_fd()).notified_sync();
--- a/xc-bin/src/run.rs
+++ b/xc-bin/src/run.rs
@@ -22,6 +22,7 @@
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.

+use crate::format::dataset::DatasetParam;
 use crate::format::{BindMount, EnvPair, IpWant, PublishSpec};

 use clap::Parser;
@@ -68,6 +69,12 @@ pub(crate) struct CreateArgs {
    pub(crate) publish: Vec<PublishSpec>,

    pub(crate) image_reference: ImageReference,
+
+    #[arg(short = 'z')]
+    pub(crate) jail_dataset: Vec<DatasetParam>,
+
+    #[arg(short = 'o')]
+    pub(crate) props: Vec<String>,
 }

 #[derive(Parser, Debug)]
@@ -142,4 +149,10 @@ pub(crate) struct RunArg {
    pub(crate) entry_point: Option<String>,

    pub(crate) entry_point_args: Vec<String>,
+
+    #[arg(short = 'z')]
+    pub(crate) jail_dataset: Vec<DatasetParam>,
+
+    #[arg(short = 'o')]
+    pub(crate) props: Vec<String>,
 }
--- a/xc-bin/src/volume.rs
+++ b/xc-bin/src/volume.rs
@@ -48,29 +48,37 @@ pub(crate) enum VolumeAction {

        driver: VolumeDriverKind,
    },
-    List
+    List,
 }

-pub(crate) fn use_volume_action(conn: &mut UnixStream, action: VolumeAction)
-    -> Result<(), crate::ActionError>
-{
+pub(crate) fn use_volume_action(
+    conn: &mut UnixStream,
+    action: VolumeAction,
+) -> Result<(), crate::ActionError> {
    match action {
        VolumeAction::List => {
            if let Ok(volumes) = do_list_volumes(conn, ())? {
                println!("{volumes:#?}")
            }
-        },
-        VolumeAction::Create { name, image_reference, volume, device, zfs_props, driver } => {
-            let template = image_reference.and_then(|ir| {
-                volume.map(|v| (ir, v))
-            });
+        }
+        VolumeAction::Create {
+            name,
+            image_reference,
+            volume,
+            device,
+            zfs_props,
+            driver,
+        } => {
+            let template = image_reference.and_then(|ir| volume.map(|v| (ir, v)));
            let zfs_props = {
                let mut props = HashMap::new();
                for value in zfs_props.into_iter() {
                    if let Some((key, value)) = value.split_once('=') {
                        props.insert(key.to_string(), value.to_string());
                    } else {
-                        Err(anyhow!("invalid zfs option, accepted formats are $key=$value"))?;
+                        Err(anyhow!(
+                            "invalid zfs option, accepted formats are $key=$value"
+                        ))?;
                    }
                }
                props
@@ -80,7 +88,7 @@ pub(crate) fn use_volume_action(conn: &mut UnixStream, action: VolumeAction)
                template,
                device,
                zfs_props,
-                kind: driver
+                kind: driver,
            };
            if let Err(err) = do_create_volume(conn, request)? {
                eprintln!("error occurred: {err:#?}")
--- a/xc/src/container/effect.rs
+++ b/xc/src/container/effect.rs
@@ -85,6 +85,7 @@ macro_rules! impl_undos {
                match self {
                    $(Undo::$name { result, $($arg),* } => {
                        debug!("cleaning up {}", stringify!($name));
+                        #[allow(clippy::redundant_closure_call)]
                        {
                            $unwind_code(result)
                        }?;
@@ -261,5 +262,15 @@ impl_undos! {
        |_| {
            freebsd::net::pf::set_rules(Some(anchor.to_string()), &["\n"])
        }
+    };
+
+    JailDataset(zfs_handle: ZfsHandle, jail: String, dataset: PathBuf) {
+        "jail a zfs dataset to `jail`",
+        zfs_handle.jail(jail.as_str(), &dataset),
+        |_| {
+            // by the time the undo stack rewind, the jail has already been destroyed and therefore
+            // unjail will not work.
+            Ok::<(), anyhow::Error>(())
+        }
    }
 }
--- a/xc/src/container/mod.rs
+++ b/xc/src/container/mod.rs
@@ -34,6 +34,7 @@ use crate::container::running::RunningContainer;
 use crate::models::exec::Jexec;
 use crate::models::jail_image::JailImage;
 use crate::models::network::IpAssign;
+use crate::models::EnforceStatfs;
 use crate::util::realpath;

 use anyhow::Context;
@@ -92,6 +93,12 @@ pub struct CreateContainer {
    pub default_router: Option<IpAddr>,

    pub log_directory: Option<PathBuf>,
+
+    pub override_props: HashMap<String, String>,
+
+    pub enforce_statfs: EnforceStatfs,
+
+    pub jailed_datasets: Vec<PathBuf>,
 }

 impl CreateContainer {
@@ -254,6 +261,36 @@ impl CreateContainer {
            }
        }

+        proto = proto.param(
+            "enforce_statfs",
+            match self.enforce_statfs {
+                EnforceStatfs::Strict => Value::Int(2),
+                EnforceStatfs::BelowRoot => Value::Int(1),
+                EnforceStatfs::ExposeAll => Value::Int(0),
+            },
+        );
+
+        for (key, value) in self.override_props.iter() {
+            const STR_KEYS: [&str; 6] = [
+                "host.hostuuid",
+                "host.domainname",
+                "host.hostname",
+                "osrelease",
+                "path",
+                "name",
+            ];
+
+            let v = if STR_KEYS.contains(&key.as_str()) {
+                Value::String(value.to_string())
+            } else if let Ok(num) = value.parse::<i32>() {
+                Value::Int(num)
+            } else {
+                Value::String(value.to_string())
+            };
+
+            proto = proto.param(key, v);
+        }
+
        let jail = proto.start()?;

        if self.vnet {
@@ -317,6 +354,18 @@ impl CreateContainer {
                .status();
        }

+        println!("datasets: {:?}", self.jailed_datasets);
+
+        for dataset in self.jailed_datasets.iter() {
+            // XXX: allow to use a non-default zfs handle?
+            undo.jail_dataset(
+                freebsd::fs::zfs::ZfsHandle::default(),
+                jail.jid.to_string(),
+                dataset.to_path_buf(),
+            )
+            .with_context(|| "jail dataset: {dataset:?}")?;
+        }
+
        let notify = Arc::new(EventFdNotify::new());

        Ok(RunningContainer {
--- a/xc/src/container/request.rs
+++ b/xc/src/container/request.rs
@@ -97,7 +97,10 @@ impl Mount {
            options: Vec::new(),
        }
    }
-    pub fn nullfs(source: impl AsRef<std::path::Path>, mountpoint: impl AsRef<std::path::Path>) -> Mount {
+    pub fn nullfs(
+        source: impl AsRef<std::path::Path>,
+        mountpoint: impl AsRef<std::path::Path>,
+    ) -> Mount {
        Mount {
            source: source.as_ref().to_string_lossy().to_string(),
            dest: mountpoint.as_ref().to_string_lossy().to_string(),
@@ -111,4 +114,5 @@ impl Mount {
 pub struct MountReq {
    pub source: String,
    pub dest: String,
+    pub read_only: bool,
 }
--- a/xc/src/models/jail_image.rs
+++ b/xc/src/models/jail_image.rs
@@ -23,6 +23,7 @@
 // SUCH DAMAGE.

 use crate::format::docker_compat::Expose;
+use crate::models::DatasetSpec;
 use crate::util::default_on_missing;

 use super::exec::Exec;
@@ -184,6 +185,9 @@ pub struct JailConfig {

    pub mounts: HashMap<String, MountSpec>,

+    #[serde(default)]
+    pub datasets: HashMap<String, DatasetSpec>,
+
    #[serde(default)]
    pub init: Vec<Exec>,

--- a/xc/src/models/mod.rs
+++ b/xc/src/models/mod.rs
@@ -33,6 +33,20 @@ use serde_json::Value;
 use std::collections::{HashMap, HashSet};
 use varutil::string_interpolation::{InterpolatedString, Var};

+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub enum EnforceStatfs {
+    ExposeAll,
+    Strict,
+    BelowRoot,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct DatasetSpec {
+    name: String,
+    required: bool,
+    required_props: HashMap<String, String>,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
 pub struct MountSpec {
    pub description: String,
--- a/xcd/src/context.rs
+++ b/xcd/src/context.rs
@@ -27,17 +27,18 @@ use crate::config::config_manager::InventoryManager;
 use crate::config::inventory::Inventory;
 use crate::config::XcConfig;
 use crate::database::Database;
-use crate::instantiate::{InstantiateBlueprint, AppliedInstantiateRequest};
+use crate::dataset::JailedDatasetTracker;
 use crate::devfs_store::DevfsRulesetStore;
 use crate::image::pull::PullImageError;
 use crate::image::ImageManager;
+use crate::instantiate::{AppliedInstantiateRequest, InstantiateBlueprint};
 use crate::ipc::InstantiateRequest;
 use crate::network_manager::NetworkManager;
 use crate::port::PortForwardTable;
 use crate::registry::JsonRegistryProvider;
 use crate::site::Site;
 use crate::util::TwoWayMap;
-use crate::volume::{VolumeManager, Volume, VolumeDriverKind};
+use crate::volume::{Volume, VolumeDriverKind, VolumeManager};

 use anyhow::Context;
 use freebsd::fs::zfs::ZfsHandle;
@@ -45,20 +46,19 @@ use freebsd::net::pf;
 use oci_util::digest::OciDigest;
 use oci_util::image_reference::{ImageReference, ImageTag};
 use oci_util::layer::ChainId;
-use xc::container::error::PreconditionFailure;
 use std::collections::HashMap;
 use std::os::fd::{FromRawFd, RawFd};
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock};
 use tracing::{debug, error, info, warn};
+use xc::container::error::PreconditionFailure;
 use xc::container::ContainerManifest;
 use xc::image_store::sqlite::SqliteImageStore;
 use xc::image_store::ImageRecord;
 use xc::models::jail_image::{JailConfig, JailImage};
 use xc::models::{network::*, MountSpec};

-
 #[usdt::provider]
 mod context_provider {
    use crate::ipc::InstantiateRequest;
@@ -85,6 +85,7 @@ pub struct ServerContext {
    pub(crate) inventory: Arc<std::sync::Mutex<InventoryManager>>,

    pub(crate) volume_manager: Arc<Mutex<VolumeManager>>,
+    pub(crate) dataset_tracker: Arc<RwLock<JailedDatasetTracker>>,
 }

 impl ServerContext {
@@ -96,10 +97,13 @@ impl ServerContext {

        let image_store: Box<SqliteImageStore> = Box::new(image_store_db);

-        let db = Arc::new(Database::from(rusqlite::Connection::open(&config.database_store)
-            .expect("cannot open sqlite database")));
+        let db = Arc::new(Database::from(
+            rusqlite::Connection::open(&config.database_store)
+                .expect("cannot open sqlite database"),
+        ));

-        db.perform(xc::res::create_tables).expect("cannot create tables");
+        db.perform(xc::res::create_tables)
+            .expect("cannot create tables");

        let inventory = Arc::new(std::sync::Mutex::new(
            InventoryManager::load_from_path(&config.inventory).expect("cannot write inventory"),
@@ -117,12 +121,11 @@ impl ServerContext {
            Arc::new(Mutex::new(Box::new(provider))),
        );

-        let volume_manager = Arc::new(Mutex::new(
-                VolumeManager::new(
-                    inventory.clone(),
-                    config.default_volume_dataset.clone(),
-                    None,
-                )));
+        let volume_manager = Arc::new(Mutex::new(VolumeManager::new(
+            inventory.clone(),
+            config.default_volume_dataset.clone(),
+            None,
+        )));

        ServerContext {
            network_manager,
@@ -137,6 +140,7 @@ impl ServerContext {
            jail2ngs: HashMap::new(),
            inventory,
            volume_manager,
+            dataset_tracker: Arc::new(RwLock::new(JailedDatasetTracker::default())),
        }
    }

@@ -425,6 +429,7 @@ impl ServerContext {
            self.port_forward_table.remove_rules(id);
            self.reload_pf_rdr_anchor()?;
            self.alias_map.remove_all_referenced(id);
+            self.dataset_tracker.write().await.release_container(id);

            if let Some(networks) = self.jail2ngs.get(id) {
                for network in networks.iter() {
@@ -550,6 +555,7 @@ impl ServerContext {
            let this = this.clone();
            let mut this = this.write().await;
            let mut site = Site::new(id, this.config());
+
            site.stage(image)?;
            let name = request.name.clone();

@@ -558,12 +564,23 @@ impl ServerContext {
                let network_manager = network_manager.lock().await;
                let volume_manager = this.volume_manager.clone();
                let volume_manager = volume_manager.lock().await;
-                AppliedInstantiateRequest::new(request, image, &cred, &network_manager, &volume_manager)?
+                AppliedInstantiateRequest::new(
+                    request,
+                    image,
+                    &cred,
+                    &network_manager,
+                    &volume_manager,
+                )?
            };

            let blueprint = {
                let network_manager = this.network_manager.clone();
                let mut network_manager = network_manager.lock().await;
+                let volume_manager = this.volume_manager.clone();
+                let volume_manager = volume_manager.lock().await;
+                let dataset_tracker = this.dataset_tracker.clone();
+                let mut dataset_tracker = dataset_tracker.write().await;
+
                InstantiateBlueprint::new(
                    id,
                    image,
@@ -571,6 +588,8 @@ impl ServerContext {
                    &mut this.devfs_store,
                    &cred,
                    &mut network_manager,
+                    &volume_manager,
+                    &mut dataset_tracker,
                )?
            };

@@ -668,8 +687,7 @@ impl ServerContext {
        result.map(|_| ())
    }

-    pub(crate) async fn list_volumes(&self) -> HashMap<String, Volume>
-    {
+    pub(crate) async fn list_volumes(&self) -> HashMap<String, Volume> {
        self.volume_manager.lock().await.list_volumes()
    }

@@ -679,8 +697,11 @@ impl ServerContext {
        template: Option<MountSpec>,
        kind: VolumeDriverKind,
        source: Option<std::path::PathBuf>,
-        zfs_props: HashMap<String, String>) -> Result<(), PreconditionFailure>
-    {
-        self.volume_manager.lock().await.create_volume(kind, name, template, source, zfs_props)
+        zfs_props: HashMap<String, String>,
+    ) -> Result<(), PreconditionFailure> {
+        self.volume_manager
+            .lock()
+            .await
+            .create_volume(kind, name, template, source, zfs_props)
    }
 }
--- a/xcd/src/database.rs
+++ b/xcd/src/database.rs
@@ -21,9 +21,9 @@
 // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.
-use std::sync::Mutex;
+use rusqlite::{Connection, OptionalExtension, Params, Row};
 use std::net::IpAddr;
-use rusqlite::{Connection, Params, Row, OptionalExtension};
+use std::sync::Mutex;

 use crate::network::AddressStore;

@@ -39,7 +39,8 @@ impl From<Connection> for Database {

 impl Database {
    pub fn perform<F, T>(&self, func: F) -> T
-        where F: FnOnce(&Connection) -> T
+    where
+        F: FnOnce(&Connection) -> T,
    {
        let conn = self.db.lock().unwrap();
        func(&conn)
@@ -51,9 +52,9 @@ impl Database {
    }

    pub fn query_row<T, P, F>(&self, sql: &str, params: P, f: F) -> rusqlite::Result<T>
-        where
-            P: Params,
-            F: FnOnce(&Row<'_>) -> rusqlite::Result<T>
+    where
+        P: Params,
+        F: FnOnce(&Row<'_>) -> rusqlite::Result<T>,
    {
        let conn = self.db.lock().unwrap();
        conn.query_row(sql, params, f)
@@ -63,7 +64,8 @@ impl Database {
 impl AddressStore for Database {
    fn all_allocated_addresses(&self, network: &str) -> rusqlite::Result<Vec<IpAddr>> {
        self.perform(|conn| {
-            let mut stmt = conn.prepare("select address from address_allocation where network=?")?;
+            let mut stmt =
+                conn.prepare("select address from address_allocation where network=?")?;
            let rows = stmt.query_map([network], |row| {
                let column: String = row.get(0)?;
                let ipaddr: IpAddr = column.parse().unwrap();
@@ -81,11 +83,14 @@ impl AddressStore for Database {
    fn last_allocated_adddress(&self, network: &str) -> rusqlite::Result<Option<IpAddr>> {
        self.query_row(
            "select addr from network_last_addrs where network=?",
-            [network], |row| {
+            [network],
+            |row| {
                let addr_text: String = row.get(0)?;
                let addr = addr_text.parse().unwrap();
                Ok(addr)
-            }).optional()
+            },
+        )
+        .optional()
    }
    fn is_address_allocated(&self, network: &str, addr: &IpAddr) -> rusqlite::Result<bool> {
        self.perform(|conn| {
@@ -114,7 +119,9 @@ impl AddressStore for Database {
            "
            insert into network_last_addrs (network, addr) values (?, ?)
                on conflict (network) do update set addr=?
-            ", [network, addr.as_str(), addr.as_str()])?;
+            ",
+            [network, addr.as_str(), addr.as_str()],
+        )?;
        Ok(())
    }
 }
--- a/xcd/src/dataset.rs
+++ b/xcd/src/dataset.rs
@@ -0,0 +1,89 @@
+//! Utility to keep track of jailed dataset so we can alert the user before actually calling ZFS
+//! unjail and rip the dataset from containers still using them
+
+// Copyright (c) 2023 Yan Ka, Chiu.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions, and the following disclaimer,
+//    without modification, immediately at the beginning of the file.
+// 2. The name of the author may not be used to endorse or promote products
+//    derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+// SUCH DAMAGE.
+
+use crate::util::TwoWayMap;
+use std::path::{Path, PathBuf};
+
+/// A tracker keep tracks of mapping between containers and jailed dataset. This construct does not
+/// perform any actual effects (jailing and un-jailing datasets).
+#[derive(Default)]
+pub(crate) struct JailedDatasetTracker {
+    jailed: TwoWayMap<String, PathBuf>,
+}
+
+impl JailedDatasetTracker {
+    pub(crate) fn set_jailed(&mut self, token: &str, dataset: impl AsRef<Path>) {
+        if self.is_jailed(&dataset) {
+            panic!("dataset is jailed by other container")
+        } else {
+            self.jailed
+                .insert(token.to_string(), dataset.as_ref().to_path_buf());
+        }
+    }
+
+    pub(crate) fn is_jailed(&self, dataset: impl AsRef<Path>) -> bool {
+        self.jailed.contains_value(dataset.as_ref())
+    }
+
+    #[allow(dead_code)]
+    pub(crate) fn unjail(&mut self, dataset: impl AsRef<Path>) {
+        self.jailed.remove_all_referenced(dataset.as_ref());
+    }
+
+    pub(crate) fn release_container(&mut self, token: &str) {
+        self.jailed.remove(token);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_jailed_dataset_allocation_detection() {
+        let mut tracker = JailedDatasetTracker::default();
+        let dataset = "zroot/test/dataset";
+        let token = "abc";
+        assert!(!tracker.is_jailed(dataset));
+        tracker.set_jailed(token, dataset);
+        assert!(tracker.is_jailed(dataset));
+        tracker.release_container(token);
+        assert!(!tracker.is_jailed(dataset));
+    }
+
+    #[test]
+    fn test_jailed_dataset_unjail() {
+        let mut tracker = JailedDatasetTracker::default();
+        let dataset = "zroot/test/dataset";
+        let token = "abc";
+        assert!(!tracker.is_jailed(dataset));
+        tracker.set_jailed(token, dataset);
+        assert!(tracker.is_jailed(dataset));
+        tracker.unjail(dataset);
+        assert!(!tracker.is_jailed(dataset));
+    }
+}
--- a/xcd/src/image/push.rs
+++ b/xcd/src/image/push.rs
@@ -156,6 +156,7 @@ pub async fn push_image(
        let mut selections = Vec::new();
        //            let (tx, upload_status) = tokio::sync::watch::channel(UploadStat::default());

+        #[allow(unreachable_code)]
        'layer_loop: for layer in layers.iter() {
            let maps = {
                let this = this.clone();
@@ -206,17 +207,13 @@ pub async fn push_image(
                "plain" => "application/vnd.oci.image.layer.v1.tar",
                _ => unreachable!(),
            };
-            /*
-                        let path = format!("{layers_dir}/{}", map.archive_digest);
-                        let path = std::path::Path::new(&path);
-            */
+
            let mut path = layers_dir.to_path_buf();
            path.push(map.archive_digest.as_str());
            let file = std::fs::OpenOptions::new().read(true).open(&path)?;
            let metadata = file.metadata().unwrap();
            let layer_size = metadata.len() as usize;

-            //                let dedup_check = Ok::<bool, std::io::Error>(false);
            let dedup_check = session.exists_digest(&map.archive_digest).await;

            _ = emitter.use_try(|state| {
@@ -268,14 +265,6 @@ pub async fn push_image(
                        size: layer_size,
                    }
                }
-
-                /*
-                let maybe_descriptor = session
-                    .upload_content(Some(tx), content_type.to_string(), file)
-                    .await;
-
-                maybe_descriptor?
-                */
            };
            uploads.push(descriptor);
            _ = emitter.use_try(|state| {
--- a/xcd/src/instantiate.rs
+++ b/xcd/src/instantiate.rs
@@ -23,10 +23,11 @@
 // SUCH DAMAGE.

 use crate::auth::Credential;
+use crate::dataset::JailedDatasetTracker;
 use crate::devfs_store::DevfsRulesetStore;
 use crate::ipc::InstantiateRequest;
 use crate::network_manager::NetworkManager;
-use crate::volume::{VolumeManager, Volume};
+use crate::volume::{Volume, VolumeManager};

 use anyhow::Context;
 use freebsd::event::EventFdNotify;
@@ -41,6 +42,7 @@ use xc::format::devfs_rules::DevfsRule;
 use xc::models::exec::{Jexec, StdioMode};
 use xc::models::jail_image::JailImage;
 use xc::models::network::{DnsSetting, IpAssign};
+use xc::models::EnforceStatfs;
 use xc::precondition_failure;

 pub struct AppliedInstantiateRequest {
@@ -52,7 +54,7 @@ pub struct AppliedInstantiateRequest {
    envs: HashMap<String, String>,
    allowing: Vec<String>,
    copies: Vec<xc::container::request::CopyFileReq>,
-    mount_req: Vec<Mount>,
+    enforce_statfs: EnforceStatfs,
 }

 impl AppliedInstantiateRequest {
@@ -143,7 +145,7 @@ impl AppliedInstantiateRequest {
            }
        }

-        let allowing = {
+        let mut allowing = {
            let mut allows = Vec::new();
            for allow in config.allow.iter() {
                if available_allows.contains(allow) {
@@ -155,6 +157,17 @@ impl AppliedInstantiateRequest {
            allows
        };

+        if !request.jail_datasets.is_empty() {
+            allowing.push("mount".to_string());
+            allowing.push("mount.zfs".to_string());
+        }
+
+        let enforce_statfs = if request.jail_datasets.is_empty() {
+            EnforceStatfs::Strict
+        } else {
+            EnforceStatfs::BelowRoot
+        };
+
        let copies: Vec<xc::container::request::CopyFileReq> = request
            .copies
            .move_to_vec()
@@ -165,58 +178,29 @@ impl AppliedInstantiateRequest {
            })
            .collect();

-        let mut mount_req = Vec::new();
-
-        for special_mount in config.special_mounts.iter() {
-            if special_mount.mount_type.as_str() == "procfs" {
-                mount_req.push(Mount::procfs(&special_mount.mount_point));
-            } else if special_mount.mount_type.as_str() == "fdescfs" {
-                mount_req.push(Mount::fdescfs(&special_mount.mount_point));
-            }
-        }
-
        let mut mount_specs = oci_config.jail_config().mounts;
-        let mut added_mount_specs = HashMap::new();

        for req in request.mount_req.iter() {
            let source_path = std::path::Path::new(&req.source);

-            let volume = if !source_path.is_absolute() {
+            if !source_path.is_absolute() {
                let name = source_path.to_string_lossy().to_string();
                match volume_manager.query_volume(&name) {
                    None => {
                        precondition_failure!(ENOENT, "no such volume {name}")
-                    },
+                    }
                    Some(volume) => {
                        if !volume.can_mount(cred.uid()) {
-                            precondition_failure!(EPERM, "this user is not allowed to mount the volume")
-                        } else {
-                            volume
+                            precondition_failure!(
+                                EPERM,
+                                "this user is not allowed to mount the volume"
+                            )
                        }
                    }
                }
-            } else {
-                Volume::adhoc(source_path)
-            };
-
-            let mount_spec = mount_specs.remove(&req.dest);
-
-            if mount_spec.is_some() {
-                added_mount_specs.insert(&req.dest, mount_spec.clone().unwrap());
            }

-            let mount = volume_manager.mount(cred, req, mount_spec.as_ref(), &volume)?;
-/*
-            let mount = match volume.driver {
-                VolumeDriverKind::Directory => {
-                    LocalDriver::default().mount(cred, req, mount_spec.as_ref(), &volume)?
-                },
-                VolumeDriverKind::ZfsDataset => {
-                    ZfsDriver::default().mount(cred, req, mount_spec.as_ref(), &volume)?
-                }
-            };
-*/
-            mount_req.push(mount);
+            mount_specs.remove(&req.dest);
        }

        for (key, spec) in mount_specs.iter() {
@@ -266,7 +250,7 @@ impl AppliedInstantiateRequest {
            main,
            envs,
            allowing,
-            mount_req,
+            enforce_statfs,
        })
    }
 }
@@ -304,6 +288,9 @@ pub struct InstantiateBlueprint {
    pub linux_no_create_proc_dir: bool,
    pub linux_no_mount_sys: bool,
    pub linux_no_mount_proc: bool,
+    pub override_props: HashMap<String, String>,
+    pub enforce_statfs: EnforceStatfs,
+    pub jailed_datasets: Vec<std::path::PathBuf>,
 }

 impl InstantiateBlueprint {
@@ -312,8 +299,10 @@ impl InstantiateBlueprint {
        oci_config: &JailImage,
        request: AppliedInstantiateRequest,
        devfs_store: &mut DevfsRulesetStore,
-        _cred: &Credential,
+        cred: &Credential,
        network_manager: &mut NetworkManager,
+        volume_manager: &VolumeManager,
+        dataset_tracker: &mut JailedDatasetTracker,
    ) -> anyhow::Result<InstantiateBlueprint> {
        let existing_ifaces = freebsd::net::ifconfig::interfaces()?;
        let config = oci_config.jail_config();
@@ -395,6 +384,64 @@ impl InstantiateBlueprint {
            };
        }

+        let mut mount_req = Vec::new();
+
+        for special_mount in config.special_mounts.iter() {
+            if special_mount.mount_type.as_str() == "procfs" {
+                mount_req.push(Mount::procfs(&special_mount.mount_point));
+            } else if special_mount.mount_type.as_str() == "fdescfs" {
+                mount_req.push(Mount::fdescfs(&special_mount.mount_point));
+            }
+        }
+
+        let mut mount_specs = oci_config.jail_config().mounts;
+        let mut added_mount_specs = HashMap::new();
+
+        for req in request.base.mount_req.iter() {
+            let source_path = std::path::Path::new(&req.source);
+
+            let volume = if !source_path.is_absolute() {
+                let name = source_path.to_string_lossy().to_string();
+                match volume_manager.query_volume(&name) {
+                    None => {
+                        precondition_failure!(ENOENT, "no such volume {name}")
+                    }
+                    Some(volume) => {
+                        if !volume.can_mount(cred.uid()) {
+                            precondition_failure!(
+                                EPERM,
+                                "this user is not allowed to mount the volume"
+                            )
+                        } else {
+                            volume
+                        }
+                    }
+                }
+            } else {
+                Volume::adhoc(source_path)
+            };
+
+            let mount_spec = mount_specs.remove(&req.dest);
+
+            if mount_spec.is_some() {
+                added_mount_specs.insert(&req.dest, mount_spec.clone().unwrap());
+            }
+
+            let mount = volume_manager.mount(id, cred, req, mount_spec.as_ref(), &volume)?;
+            mount_req.push(mount);
+        }
+
+        for dataset in request.base.jail_datasets.iter() {
+            if dataset_tracker.is_jailed(dataset) {
+                precondition_failure!(
+                    EPERM,
+                    "another container is using this dataset: {dataset:?}"
+                )
+            } else {
+                dataset_tracker.set_jailed(id, dataset)
+            }
+        }
+
        let mut devfs_rules = vec![
            "include 1".to_string(),
            "include 2".to_string(),
@@ -431,7 +478,7 @@ impl InstantiateBlueprint {
            main: request.main,
            ips: request.base.ips,
            ipreq: request.base.ipreq,
-            mount_req: request.mount_req,
+            mount_req,
            linux: config.linux,
            deinit_norun: request.base.deinit_norun,
            init_norun: request.base.init_norun,
@@ -453,6 +500,9 @@ impl InstantiateBlueprint {
            linux_no_create_proc_dir: request.base.linux_no_create_proc_dir,
            linux_no_mount_sys: request.base.linux_no_mount_sys,
            linux_no_mount_proc: request.base.linux_no_mount_proc,
+            override_props: request.base.override_props,
+            enforce_statfs: request.enforce_statfs,
+            jailed_datasets: request.base.jail_datasets,
        })
    }
 }
--- a/xcd/src/ipc.rs
+++ b/xcd/src/ipc.rs
@@ -39,7 +39,6 @@ use oci_util::digest::OciDigest;
 use oci_util::distribution::client::{BasicAuth, Registry};
 use oci_util::image_reference::{ImageReference, ImageTag};
 use serde::{Deserialize, Serialize};
-use xc::image_store::ImageStoreError;
 use std::collections::HashMap;
 use std::io::Seek;
 use std::net::IpAddr;
@@ -50,6 +49,7 @@ use std::sync::Arc;
 use tokio::sync::RwLock;
 use tracing::*;
 use xc::container::request::{MountReq, NetworkAllocRequest};
+use xc::image_store::ImageStoreError;
 use xc::models::exec::{Jexec, StdioMode};
 use xc::models::jail_image::JailConfig;
 use xc::models::network::{DnsSetting, IpAssign, PortRedirection};
@@ -199,7 +199,6 @@ pub struct EntryPointSpec {
 #[derive(Serialize, FromPacket)]
 pub struct InstantiateRequest {
    pub image_reference: ImageReference,
-    pub alt_root: Option<String>,
    pub envs: HashMap<String, String>,
    pub vnet: bool,
    pub ips: Vec<IpAssign>,
@@ -224,6 +223,8 @@ pub struct InstantiateRequest {
    pub linux_no_mount_proc: bool,
    pub user: Option<String>,
    pub group: Option<String>,
+    pub override_props: HashMap<String, String>,
+    pub jail_datasets: Vec<PathBuf>,
 }

 impl InstantiateRequest {
@@ -254,7 +255,6 @@ impl Default for InstantiateRequest {

        InstantiateRequest {
            image_reference,
-            alt_root: None,
            envs: HashMap::new(),
            vnet: false,
            ips: Vec::new(),
@@ -262,10 +262,6 @@ impl Default for InstantiateRequest {
            mount_req: Vec::new(),
            copies: List::new(),
            entry_point: None,
-            /*
-            entry_point: String::new(),
-            entry_point_args: Vec::new(),
-            */
            hostname: None,
            main_norun: false,
            init_norun: false,
@@ -283,6 +279,8 @@ impl Default for InstantiateRequest {
            linux_no_mount_proc: false,
            user: None,
            group: None,
+            override_props: HashMap::new(),
+            jail_datasets: Vec::new(),
        }
    }
 }
@@ -1138,7 +1136,7 @@ pub struct CreateVolumeRequest {
    pub template: Option<(ImageReference, String)>,
    pub device: Option<PathBuf>,
    pub zfs_props: HashMap<String, String>,
-    pub kind: VolumeDriverKind
+    pub kind: VolumeDriverKind,
 }

 #[ipc_method(method = "create_volume")]
@@ -1146,35 +1144,45 @@ async fn create_volume(
    context: Arc<RwLock<ServerContext>>,
    local_context: &mut ConnectionContext<Variables>,
    request: CreateVolumeRequest,
-) -> GenericResult<()>
-{
+) -> GenericResult<()> {
    let template = match request.template {
        None => None,
        Some((image_reference, volume)) => {
-            match context.read().await.image_manager.read().await.query_manifest(&image_reference).await {
+            match context
+                .read()
+                .await
+                .image_manager
+                .read()
+                .await
+                .query_manifest(&image_reference)
+                .await
+            {
                Ok(image) => {
                    let specs = image.manifest.jail_config().mounts;
                    specs.get(&volume).cloned()
-                },
+                }
                Err(ImageStoreError::ManifestNotFound(manifest)) => {
                    return enoent(&format!("no such manifest {manifest}"))
                }
                Err(ImageStoreError::TagNotFound(a, b)) => {
                    return enoent(&format!("no such image {a}:{b}"))
                }
-                Err(error) => {
-                    return ipc_err(EINVAL, &format!("image store error: {error:?}"))
-                }
+                Err(error) => return ipc_err(EINVAL, &format!("image store error: {error:?}")),
            }
        }
    };

-    if let Err(err) = context.write().await.create_volume(
-        &request.name,
-        template,
-        request.kind,
-        request.device,
-        request.zfs_props).await
+    if let Err(err) = context
+        .write()
+        .await
+        .create_volume(
+            &request.name,
+            template,
+            request.kind,
+            request.device,
+            request.zfs_props,
+        )
+        .await
    {
        ipc_err(err.errno(), &err.error_message())
    } else {
@@ -1186,9 +1194,8 @@ async fn create_volume(
 async fn list_volumes(
    context: Arc<RwLock<ServerContext>>,
    local_context: &mut ConnectionContext<Variables>,
-    request: ()
-) -> GenericResult<HashMap<String, Volume>>
-{
+    request: (),
+) -> GenericResult<HashMap<String, Volume>> {
    Ok(context.read().await.list_volumes().await)
 }

--- a/xcd/src/lib.rs
+++ b/xcd/src/lib.rs
@@ -25,6 +25,7 @@ mod auth;
 mod config;
 mod context;
 mod database;
+mod dataset;
 mod devfs_store;
 mod image;
 mod instantiate;
--- a/xcd/src/network/mod.rs
+++ b/xcd/src/network/mod.rs
@@ -38,18 +38,24 @@ pub struct Network {
 }

 impl Network {
-    pub fn parameterize<'a, 'b, A: AddressStore>(&'b self, name: &str, store: &'a A)
-        -> Netpool<'a, 'b, A>
-    {
+    pub fn parameterize<'a, 'b, A: AddressStore>(
+        &'b self,
+        name: &str,
+        store: &'a A,
+    ) -> Netpool<'a, 'b, A> {
        let last_addr = store.last_allocated_adddress(name).unwrap();
        let name = name.to_string();
        Netpool {
            network: self,
            store,
            last_addr,
-            start_addr: self.start_addr.unwrap_or_else(|| self.subnet.network_addr()),
-            end_addr: self.end_addr.unwrap_or_else(|| self.subnet.broadcast_addr()),
-            name
+            start_addr: self
+                .start_addr
+                .unwrap_or_else(|| self.subnet.network_addr()),
+            end_addr: self
+                .end_addr
+                .unwrap_or_else(|| self.subnet.broadcast_addr()),
+            name,
        }
    }
 }
@@ -72,6 +78,7 @@ pub struct Netpool<'a, 'b, A: AddressStore> {
    name: String,
 }

+#[allow(dead_code)]
 impl<'a, 'b, A: AddressStore> Netpool<'a, 'b, A> {
    pub fn all_allocated_addresses(&self) -> rusqlite::Result<Vec<IpAddr>> {
        self.store.all_allocated_addresses(&self.name)
@@ -79,7 +86,7 @@ impl<'a, 'b, A: AddressStore> Netpool<'a, 'b, A> {

    pub fn next_cidr(&mut self, token: &str) -> rusqlite::Result<Option<IpCidr>> {
        self.next_address(token)
-        .map(|x| x.and_then(|a| IpCidr::from_addr(a, self.network.subnet.mask())))
+            .map(|x| x.and_then(|a| IpCidr::from_addr(a, self.network.subnet.mask())))
    }
    pub fn next_address(&mut self, token: &str) -> rusqlite::Result<Option<IpAddr>> {
        macro_rules! next_addr {
@@ -133,11 +140,7 @@ impl<'a, 'b, A: AddressStore> Netpool<'a, 'b, A> {
        self.store.is_address_allocated(&self.name, addr)
    }

-    pub fn register_address(
-        &self,
-        addr: &IpAddr,
-        token: &str,
-    ) -> rusqlite::Result<()> {
+    pub fn register_address(&self, addr: &IpAddr, token: &str) -> rusqlite::Result<()> {
        self.store.add_address(&self.name, addr, token)
    }
 }
--- a/xcd/src/network_manager.rs
+++ b/xcd/src/network_manager.rs
@@ -32,9 +32,9 @@ use thiserror::Error;
 use xc::container::request::NetworkAllocRequest;
 use xc::models::network::IpAssign;

-use crate::network::{Network, AddressStore};
-use crate::database::Database;
 use crate::config::config_manager::InventoryManager;
+use crate::database::Database;
+use crate::network::{AddressStore, Network};

 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct NetworkInfo {
@@ -142,7 +142,9 @@ impl NetworkManager {
        &mut self,
        token: &str,
    ) -> anyhow::Result<HashMap<String, Vec<IpAddr>>> {
-        self.db.release_addresses(token).context("fail to release addresses")?;
+        self.db
+            .release_addresses(token)
+            .context("fail to release addresses")?;
        let networks = self.table_cache.remove(token).unwrap_or_default();
        Ok(networks)
    }
--- a/xcd/src/site.rs
+++ b/xcd/src/site.rs
@@ -440,6 +440,9 @@ impl Site {
                    image_reference: blueprint.image_reference,
                    default_router: blueprint.default_router,
                    log_directory: Some(std::path::PathBuf::from(&self.config.logs_dir)),
+                    override_props: blueprint.override_props,
+                    enforce_statfs: blueprint.enforce_statfs,
+                    jailed_datasets: blueprint.jailed_datasets,
                };

                let running_container = container
--- a/xcd/src/util.rs
+++ b/xcd/src/util.rs
@@ -75,6 +75,14 @@ impl<K: Clone + Hash + Eq, V: Clone + Hash + Eq> TwoWayMap<K, V> {
        self.main_map.get(key)
    }

+    pub fn contains_value<Q: Eq + Hash + ?Sized>(&self, value: &Q) -> bool
+    where
+        V: Borrow<Q>,
+    {
+        self.reverse_map.contains_key(value)
+    }
+
+    /// Remove all keys that referenced the value
    pub fn remove_all_referenced<Q>(&mut self, value: &Q) -> Option<HashSet<K>>
    where
        V: Borrow<Q>,
@@ -87,7 +95,6 @@ impl<K: Clone + Hash + Eq, V: Clone + Hash + Eq> TwoWayMap<K, V> {
        Some(keys)
    }

-    #[allow(unused)]
    pub fn remove<Q>(&mut self, key: &Q) -> Option<V>
    where
        K: Borrow<Q>,
@@ -97,6 +104,9 @@ impl<K: Clone + Hash + Eq, V: Clone + Hash + Eq> TwoWayMap<K, V> {
        if let Some(value) = &value {
            if let Some(keys) = self.reverse_map.get_mut(value) {
                keys.remove(key);
+                if keys.is_empty() {
+                    self.reverse_map.remove(value);
+                }
            }
        }
        value
--- a/xcd/src/volume/drivers/local.rs
+++ b/xcd/src/volume/drivers/local.rs
@@ -22,14 +22,20 @@
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.

-use freebsd::libc::{ENOENT, EPERM, ENOTDIR};
+use freebsd::libc::{ENOENT, ENOTDIR, EPERM};
 use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 use xc::models::MountSpec;
-use xc::{container::{request::{MountReq, Mount}, error::PreconditionFailure}, precondition_failure};
+use xc::{
+    container::{
+        error::PreconditionFailure,
+        request::{Mount, MountReq},
+    },
+    precondition_failure,
+};

-use crate::{auth::Credential, volume::Volume};
 use crate::volume::VolumeDriverKind;
+use crate::{auth::Credential, volume::Volume};

 use super::VolumeDriver;

@@ -43,11 +49,11 @@ impl VolumeDriver for LocalDriver {
        name: &str,
        _template: Option<MountSpec>,
        source: Option<std::path::PathBuf>,
-        _props: HashMap<String, String>) -> Result<Volume, PreconditionFailure>
-    {
+        _props: HashMap<String, String>,
+    ) -> Result<Volume, PreconditionFailure> {
        let path = match source {
            None => {
-                let Some(mut parent) = self.default_subdir.clone() else  {
+                let Some(mut parent) = self.default_subdir.clone() else {
                    precondition_failure!(ENOENT, "Default volume directory not found")
                };
                parent.push(name);
@@ -55,7 +61,7 @@ impl VolumeDriver for LocalDriver {
                    precondition_failure!(ENOENT, "Target directory already exists")
                }
                parent
-            },
+            }
            Some(path) => {
                if !path.exists() {
                    precondition_failure!(ENOENT, "No such directory")
@@ -66,12 +72,13 @@ impl VolumeDriver for LocalDriver {
            }
        };
        Ok(Volume {
+            name: Some(name.to_string()),
            rw_users: None,
            authorized_users: None,
            driver: VolumeDriverKind::Directory,
            mount_options: Vec::new(),
            driver_options: HashMap::new(),
-            device: path
+            device: path,
        })
    }

@@ -80,9 +87,8 @@ impl VolumeDriver for LocalDriver {
        cred: &Credential,
        mount_req: &MountReq,
        mount_spec: Option<&MountSpec>,
-        volume: &Volume
-    ) -> Result<Mount, PreconditionFailure>
-    {
+        volume: &Volume,
+    ) -> Result<Mount, PreconditionFailure> {
        let source_path = &volume.device;
        if !&source_path.exists() {
            precondition_failure!(ENOENT, "source mount point does not exist: {source_path:?}");
@@ -103,8 +109,8 @@ impl VolumeDriver for LocalDriver {

        let mut mount_options = HashSet::new();

-        if !volume.can_mount_rw(cred.uid()) ||
-            mount_spec.map(|spec| spec.read_only).unwrap_or_default()
+        if !volume.can_mount_rw(cred.uid())
+            || mount_spec.map(|spec| spec.read_only).unwrap_or_default()
        {
            mount_options.insert("ro".to_string());
        }
--- a/xcd/src/volume/drivers/mod.rs
+++ b/xcd/src/volume/drivers/mod.rs
@@ -24,10 +24,13 @@
 pub mod local;
 pub mod zfs;

-use crate::auth::Credential;
 use super::Volume;
+use crate::auth::Credential;
 use std::collections::HashMap;
-use xc::container::{error::PreconditionFailure, request::{Mount, MountReq}};
+use xc::container::{
+    error::PreconditionFailure,
+    request::{Mount, MountReq},
+};
 use xc::models::MountSpec;

 pub trait VolumeDriver {
@@ -36,14 +39,14 @@ pub trait VolumeDriver {
        cred: &Credential,
        mount_req: &MountReq,
        mount_spec: Option<&MountSpec>,
-        volume: &Volume) -> Result<Mount, PreconditionFailure>;
+        volume: &Volume,
+    ) -> Result<Mount, PreconditionFailure>;

    fn create(
        &self,
        name: &str,
        template: Option<MountSpec>,
        source: Option<std::path::PathBuf>,
-        props: HashMap<String, String>
+        props: HashMap<String, String>,
    ) -> Result<Volume, PreconditionFailure>;
-
 }
--- a/xcd/src/volume/drivers/zfs.rs
+++ b/xcd/src/volume/drivers/zfs.rs
@@ -22,12 +22,18 @@
 // OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 // SUCH DAMAGE.

-use freebsd::fs::zfs::{ZfsHandle, ZfsCreate};
-use freebsd::libc::{EIO, ENOENT, EPERM, EEXIST};
+use freebsd::fs::zfs::{ZfsCreate, ZfsHandle};
+use freebsd::libc::{EEXIST, EIO, ENOENT, EPERM};
 use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 use xc::models::MountSpec;
-use xc::{container::{request::{MountReq, Mount}, error::PreconditionFailure}, precondition_failure};
+use xc::{
+    container::{
+        error::PreconditionFailure,
+        request::{Mount, MountReq},
+    },
+    precondition_failure,
+};

 use crate::volume::VolumeDriverKind;
 use crate::{auth::Credential, volume::Volume};
@@ -46,8 +52,8 @@ impl VolumeDriver for ZfsDriver {
        name: &str,
        template: Option<MountSpec>,
        source: Option<std::path::PathBuf>,
-        props: HashMap<String, String>) -> Result<Volume, PreconditionFailure>
-    {
+        props: HashMap<String, String>,
+    ) -> Result<Volume, PreconditionFailure> {
        let mut zfs_props = props;

        if let Some(template) = template {
@@ -79,7 +85,7 @@ impl VolumeDriver for ZfsDriver {
                }

                dataset
-            },
+            }
            Some(dataset) => {
                if !self.handle.exists(&dataset) {
                    precondition_failure!(ENOENT, "Requested dataset does not exist")
@@ -89,12 +95,13 @@ impl VolumeDriver for ZfsDriver {
        };

        Ok(Volume {
+            name: Some(name.to_string()),
            rw_users: None,
            authorized_users: None,
            driver: VolumeDriverKind::ZfsDataset,
            mount_options: Vec::new(),
            driver_options: zfs_props,
-            device: dataset
+            device: dataset,
        })
    }

@@ -103,15 +110,17 @@ impl VolumeDriver for ZfsDriver {
        cred: &Credential,
        mount_req: &MountReq,
        mount_spec: Option<&MountSpec>,
-        volume: &Volume
-    )
-        -> Result<Mount, PreconditionFailure>
-    {
+        volume: &Volume,
+    ) -> Result<Mount, PreconditionFailure> {
        if !self.handle.exists(&volume.device) {
            precondition_failure!(ENOENT, "No such dataset: {:?}", volume.device);
        }
        let Ok(Some(mount_point)) = self.handle.mount_point(&volume.device) else {
-            precondition_failure!(ENOENT, "Dataset {:?} does not have a mount point", volume.device);
+            precondition_failure!(
+                ENOENT,
+                "Dataset {:?} does not have a mount point",
+                volume.device
+            );
        };
        let Ok(meta) = std::fs::metadata(&mount_point) else {
            precondition_failure!(EIO, "cannot get metadata of {mount_point:?}");
@@ -121,8 +130,8 @@ impl VolumeDriver for ZfsDriver {
        }

        let mut mount_options = HashSet::new();
-        if !volume.can_mount_rw(cred.uid()) ||
-            mount_spec.map(|spec| spec.read_only).unwrap_or_default()
+        if !volume.can_mount_rw(cred.uid())
+            || mount_spec.map(|spec| spec.read_only).unwrap_or_default()
        {
            mount_options.insert("ro".to_string());
        }
@@ -137,4 +146,3 @@ impl VolumeDriver for ZfsDriver {
        })
    }
 }
-
--- a/xcd/src/volume/mod.rs
+++ b/xcd/src/volume/mod.rs
@@ -25,18 +25,20 @@ pub mod drivers;

 use crate::auth::Credential;
 use crate::config::config_manager::InventoryManager;
-use crate::volume::drivers::VolumeDriver;
-use crate::volume::drivers::zfs::ZfsDriver;
 use crate::volume::drivers::local::LocalDriver;
+use crate::volume::drivers::zfs::ZfsDriver;
+use crate::volume::drivers::VolumeDriver;

+use freebsd::libc::EPERM;
 use serde::de::Deserializer;
 use serde::{Deserialize, Serialize};
-use xc::container::error::PreconditionFailure;
-use xc::container::request::{MountReq, Mount};
-use xc::models::MountSpec;
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
+use xc::container::error::PreconditionFailure;
+use xc::container::request::{Mount, MountReq};
+use xc::models::MountSpec;
+use xc::precondition_failure;

 #[derive(Default, PartialEq, Eq, Debug, Clone)]
 pub enum VolumeDriverKind {
@@ -61,7 +63,10 @@ impl std::str::FromStr for VolumeDriverKind {
        match s {
            "zfs" => Ok(Self::ZfsDataset),
            "directory" => Ok(Self::Directory),
-            _ => Err(std::io::Error::new(std::io::ErrorKind::Other, "invalid value"))
+            _ => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "invalid value",
+            )),
        }
    }
 }
@@ -116,17 +121,27 @@ pub struct Volume {

    #[serde(default)]
    pub driver_options: HashMap<String, String>,
+
+    #[serde(skip)]
+    pub name: Option<String>,
 }

 impl Volume {
+    fn adhoc_identifier(driver: &VolumeDriverKind, device: impl AsRef<Path>) -> String {
+        let device = device.as_ref().as_os_str().to_string_lossy();
+        format!("{}:{}", driver, device)
+    }

    /// Creates an instance of volume for ad-hoc mounting purpose, e.g. -v /source:/other/location
    pub(crate) fn adhoc(device: impl AsRef<Path>) -> Self {
+        let driver = VolumeDriverKind::Directory;
+        let device = device.as_ref().to_path_buf();
        Self {
+            name: Some(Self::adhoc_identifier(&driver, &device)),
            rw_users: None,
-            device: device.as_ref().to_path_buf(),
+            device,
            authorized_users: None,
-            driver: VolumeDriverKind::Directory,
+            driver,
            mount_options: Vec::new(),
            driver_options: HashMap::new(),
        }
@@ -166,38 +181,62 @@ impl Volume {
    }
 }

+#[derive(PartialEq, Eq, Debug)]
+pub enum VolumeShareMode {
+    Exclusive,
+    SingleWriter,
+}
+
 pub(crate) struct VolumeManager {
    inventory: Arc<Mutex<InventoryManager>>,
    default_volume_dataset: Option<PathBuf>,
    default_volume_dir: Option<PathBuf>,
+    constrained_shares: HashMap<String, VolumeShareMode>,
 }

 impl VolumeManager {
-
    pub(crate) fn new(
        inventory: Arc<Mutex<InventoryManager>>,
        default_volume_dataset: Option<PathBuf>,
        default_volume_dir: Option<PathBuf>,
-    ) -> VolumeManager
-    {
-        VolumeManager { inventory, default_volume_dataset, default_volume_dir }
+    ) -> VolumeManager {
+        VolumeManager {
+            inventory,
+            default_volume_dataset,
+            default_volume_dir,
+            constrained_shares: HashMap::new(),
+        }
    }

    // insert or override a volume
    pub(crate) fn add_volume(&mut self, name: &str, volume: &Volume) {
        self.inventory.lock().unwrap().modify(|inventory| {
-            inventory
-                .volumes
-                .insert(name.to_string(), volume.clone());
+            inventory.volumes.insert(name.to_string(), volume.clone());
        });
    }

    pub(crate) fn list_volumes(&self) -> HashMap<String, Volume> {
-        self.inventory.lock().unwrap().borrow().volumes.clone()
+        let mut hm = HashMap::new();
+        for (name, volume) in self.inventory.lock().unwrap().borrow().volumes.iter() {
+            let mut vol = volume.clone();
+            vol.name = Some(name.to_string());
+            hm.insert(name.to_string(), vol);
+        }
+        hm
    }

    pub(crate) fn query_volume(&self, name: &str) -> Option<Volume> {
-        self.inventory.lock().unwrap().borrow().volumes.get(name).cloned()
+        self.inventory
+            .lock()
+            .unwrap()
+            .borrow()
+            .volumes
+            .get(name)
+            .cloned()
+            .map(|mut vol| {
+                vol.name = Some(name.to_string());
+                vol
+            })
    }

    pub(crate) fn create_volume(
@@ -206,19 +245,19 @@ impl VolumeManager {
        name: &str,
        template: Option<MountSpec>,
        source: Option<PathBuf>,
-        props: HashMap<String, String>) -> Result<(), PreconditionFailure>
-    {
+        props: HashMap<String, String>,
+    ) -> Result<(), PreconditionFailure> {
        let volume = match kind {
            VolumeDriverKind::Directory => {
                let local_driver = LocalDriver {
-                    default_subdir: self.default_volume_dir.clone()
+                    default_subdir: self.default_volume_dir.clone(),
                };
                local_driver.create(name, template, source, props)?
-            },
+            }
            VolumeDriverKind::ZfsDataset => {
                let zfs_driver = ZfsDriver {
                    handle: freebsd::fs::zfs::ZfsHandle::default(),
-                    default_dataset: self.default_volume_dataset.clone()
+                    default_dataset: self.default_volume_dataset.clone(),
                };
                zfs_driver.create(name, template, source, props)?
            }
@@ -231,23 +270,38 @@ impl VolumeManager {

    pub(crate) fn mount(
        &self,
+        _token: &str,
        cred: &Credential,
        mount_req: &MountReq,
        mount_spec: Option<&MountSpec>,
-        volume: &Volume
-    ) -> Result<Mount, PreconditionFailure>
-    {
+        volume: &Volume,
+    ) -> Result<Mount, PreconditionFailure> {
+        for (name, share) in self.constrained_shares.iter() {
+            if volume.name.as_ref().unwrap() == name {
+                if share == &VolumeShareMode::Exclusive {
+                    precondition_failure!(
+                        EPERM,
+                        "The volume has been mounted exclusively by other container"
+                    )
+                } else if share == &VolumeShareMode::SingleWriter && !mount_req.read_only {
+                    precondition_failure!(
+                        EPERM,
+                        "The volume has been mounted for exclusively write by other container"
+                    )
+                }
+            }
+        }
        match volume.driver {
            VolumeDriverKind::Directory => {
                let local_driver = LocalDriver {
-                    default_subdir: self.default_volume_dir.clone()
+                    default_subdir: self.default_volume_dir.clone(),
                };
                local_driver.mount(cred, mount_req, mount_spec, volume)
-            },
+            }
            VolumeDriverKind::ZfsDataset => {
                let zfs_driver = ZfsDriver {
                    handle: freebsd::fs::zfs::ZfsHandle::default(),
-                    default_dataset: self.default_volume_dataset.clone()
+                    default_dataset: self.default_volume_dataset.clone(),
                };
                zfs_driver.mount(cred, mount_req, mount_spec, volume)
            }