1 Commits

Author SHA1 Message Date
a70ea99915 v0.5.0 (#61)
All checks were successful
ci/woodpecker/push/pipeline Pipeline was successful
Reviewed-on: #61
Co-authored-by: pptx704 <rafeed@omukk.dev>
Co-committed-by: pptx704 <rafeed@omukk.dev>
2026-07-25 19:43:45 +00:00
57 changed files with 3780 additions and 545 deletions

View File

@ -52,6 +52,12 @@ WRENN_ENCRYPTION_KEY=
# that deliver to internal endpoints.
WRENN_CHANNELS_ALLOW_PRIVATE=false
# External storage volumes
# Largest size a single volume may be created with. Accepts G/Gi/M/Mi suffixes
# (same form as WRENN_DEFAULT_ROOTFS_SIZE). Volumes are sparse files, so this
# bounds worst-case growth rather than upfront allocation.
WRENN_MAX_VOLUME_SIZE=20Gi
# OAuth
OAUTH_GITHUB_CLIENT_ID=
OAUTH_GITHUB_CLIENT_SECRET=

View File

@ -1 +1 @@
0.4.0
0.5.0

View File

@ -1 +1 @@
0.4.0
0.5.0

View File

@ -17,6 +17,7 @@ import (
"github.com/joho/godotenv"
"git.omukk.dev/wrenn/wrenn/internal/hostagent"
"git.omukk.dev/wrenn/wrenn/internal/units"
"git.omukk.dev/wrenn/wrenn/pkg/auth"
"git.omukk.dev/wrenn/wrenn/pkg/logging"
"git.omukk.dev/wrenn/wrenn/pkg/network"
@ -98,7 +99,7 @@ func main() {
// Parse default rootfs size from env (e.g. "5G", "2Gi", "1000M").
defaultRootfsSizeMB := sandbox.DefaultDiskSizeMB
if sizeStr := os.Getenv("WRENN_DEFAULT_ROOTFS_SIZE"); sizeStr != "" {
parsed, err := sandbox.ParseSizeToMB(sizeStr)
parsed, err := units.ParseSizeToMB(sizeStr)
if err != nil {
slog.Error("invalid WRENN_DEFAULT_ROOTFS_SIZE", "value", sizeStr, "error", err)
os.Exit(1)

View File

@ -0,0 +1,28 @@
-- +goose Up
-- External storage volumes: team-scoped block-storage disks that are attached
-- to a capsule at create time and mounted inside the guest. A volume is
-- host-local (Cloud Hypervisor can only attach a path on the same host), so it
-- is pinned to a host the first time it is attached and stays there. Volumes
-- are never auto-deleted: destroying a capsule frees its volumes back to
-- 'detached' (data preserved); removal is always an explicit delete.
CREATE TABLE volumes (
id UUID PRIMARY KEY,
team_id UUID NOT NULL REFERENCES teams(id),
host_id UUID REFERENCES hosts(id), -- NULL until first attach; pins the volume thereafter
name TEXT NOT NULL,
size_mb INTEGER NOT NULL,
status TEXT NOT NULL DEFAULT 'detached', -- detached | attaching | attached | deleting
sandbox_id UUID REFERENCES sandboxes(id), -- set while attaching/attached, cleared on capsule destroy
mount_path TEXT NOT NULL DEFAULT '', -- guest mount path in use; '' means the default /mnt/<vol-id>
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_attached_at TIMESTAMPTZ,
last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (team_id, name)
);
CREATE INDEX idx_volumes_team ON volumes(team_id);
CREATE INDEX idx_volumes_host ON volumes(host_id);
CREATE INDEX idx_volumes_sandbox ON volumes(sandbox_id) WHERE sandbox_id IS NOT NULL;
-- +goose Down
DROP TABLE IF EXISTS volumes;

View File

@ -0,0 +1,41 @@
-- +goose Up
-- Volume names are now always "vl-"-prefixed slugs: lowercase alphanumerics in
-- dash-separated groups, at most 40 characters. The prefix makes a name
-- self-describing and — since volume IDs use the distinct "vol-" prefix — lets
-- one API path segment be resolved as either an ID or a name unambiguously.
--
-- Backfill the names written before the rule existed. They were validated with
-- the looser SafeName allowlist, so coerce them into slug shape first
-- (lowercase, non-slug characters to dashes, collapse and trim dashes) and only
-- then prefix. A name that survives as empty falls back to the volume's ID,
-- which is also the default a nameless volume now gets.
UPDATE volumes
SET name = LEFT(
'vl-' || COALESCE(
NULLIF(TRIM(BOTH '-' FROM REGEXP_REPLACE(LOWER(name), '[^a-z0-9]+', '-', 'g')), ''),
REPLACE(id::text, '-', '')
),
40
)
WHERE name !~ '^vl-[a-z0-9]+(-[a-z0-9]+)*$';
-- The truncation and coercion above can collide two previously-distinct names
-- within a team, which UNIQUE (team_id, name) would have rejected. Nothing here
-- can fail silently, so re-assert the invariant: any row still not matching the
-- rule aborts the migration rather than leaving invalid names behind.
-- +goose StatementBegin
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM volumes WHERE name !~ '^vl-[a-z0-9]+(-[a-z0-9]+)*$') THEN
RAISE EXCEPTION 'volume name backfill left rows that are not valid vl- slugs';
END IF;
END
$$;
-- +goose StatementEnd
-- +goose Down
-- Strip the prefix back off. The original pre-coercion spelling is not
-- recoverable, so this restores the shape, not the exact former value.
UPDATE volumes
SET name = REGEXP_REPLACE(name, '^vl-', '')
WHERE name ~ '^vl-';

178
db/queries/volumes.sql Normal file
View File

@ -0,0 +1,178 @@
-- name: InsertVolume :one
INSERT INTO volumes (id, team_id, name, size_mb, status)
VALUES ($1, $2, $3, $4, 'detached')
RETURNING *;
-- name: GetVolume :one
SELECT * FROM volumes WHERE id = $1;
-- name: GetVolumeByTeam :one
SELECT * FROM volumes WHERE id = $1 AND team_id = $2;
-- name: GetVolumeByTeamAndName :one
-- Resolve a volume by its user-facing name. Names are unique per team, so this
-- is the name-based counterpart to GetVolumeByTeam and is what lets the API
-- accept "vl-cache" wherever it accepts "vol-<id>".
SELECT * FROM volumes WHERE team_id = $1 AND name = $2;
-- name: ListVolumesByTeam :many
SELECT * FROM volumes WHERE team_id = $1 ORDER BY created_at DESC;
-- name: ListVolumesBySandbox :many
SELECT * FROM volumes WHERE sandbox_id = $1 ORDER BY created_at DESC;
-- name: CountVolumesByHost :one
-- Guard used before a host can be removed: a host holding pinned volumes must
-- not be deleted out from under them.
SELECT COUNT(*) FROM volumes WHERE host_id = $1;
-- name: ReserveVolumeForAttach :one
-- Atomically claim a detached volume for a capsule about to boot. The CAS on
-- status = 'detached' enforces single-attach: a second capsule create racing
-- for the same volume finds no row and fails. sandbox_id is deliberately NOT
-- set here — the sandbox row does not exist yet, so setting the FK would fail.
-- It is set in MarkVolumeAttached once the capsule has booted.
UPDATE volumes
SET status = 'attaching',
last_updated = NOW()
WHERE id = $1 AND status = 'detached'
RETURNING *;
-- name: MarkVolumeAttached :one
-- Promote a reserved volume to attached once the capsule has booted with it.
-- Records the owning sandbox and pins the volume to the sandbox's host on
-- first-ever attach (COALESCE keeps an existing pin for a re-attached volume).
UPDATE volumes
SET status = 'attached',
host_id = COALESCE(host_id, $2),
sandbox_id = $3,
mount_path = $4,
last_attached_at = NOW(),
last_updated = NOW()
WHERE id = $1 AND status = 'attaching'
RETURNING *;
-- name: LinkVolumeReservation :exec
-- Stamp the owning sandbox onto a reservation as soon as the sandbox row
-- exists (ReserveVolumeForAttach runs before the insert, so it cannot set the
-- FK itself). This makes every in-flight reservation attributable to a capsule,
-- which is what lets the host monitor decide — against live host state —
-- whether a stuck 'attaching' row is safe to free.
UPDATE volumes
SET sandbox_id = $2,
last_updated = NOW()
WHERE id = $1 AND status = 'attaching';
-- name: ReleaseVolumeReservation :exec
-- Roll a reserved volume back to detached when the capsule create fails.
-- host_id is left untouched (a failed create never pins a fresh volume, since
-- MarkVolumeAttached is what sets the pin). Only call this once the host has
-- confirmed the capsule is not running — a volume freed while a VM still has
-- its backing file open can be re-attached elsewhere and corrupted.
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
last_updated = NOW()
WHERE id = $1 AND status = 'attaching';
-- name: DetachVolumesBySandbox :exec
-- Terminal sweep run when a capsule reaches a terminal state (destroyed,
-- stopped, errored, or reaped): free every volume it held back to detached
-- (data and host pin preserved) so it can be reused or deleted. Keyed on
-- sandbox_id, which is stamped on at reservation time. Never deletes the
-- volume — removal is always explicit.
--
-- 'attaching' is swept alongside 'attached' so a reservation whose promotion
-- never landed (lost RPC response, DB blip) is freed here — on confirmed host
-- state — rather than on a blind timer.
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
mount_path = '',
last_updated = NOW()
WHERE sandbox_id = $1 AND status IN ('attached', 'attaching');
-- name: DetachVolumesBySandboxIDs :exec
-- Bulk variant of DetachVolumesBySandbox for the host monitor, which stops
-- many sandboxes at once (e.g. after a host goes offline).
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
mount_path = '',
last_updated = NOW()
WHERE sandbox_id = ANY($1::uuid[]) AND status IN ('attached', 'attaching');
-- name: DetachVolumesByHost :exec
-- Free and un-pin every volume on a host being force-deleted. The host (and its
-- backing files) are going away, so the volumes are reset to a fresh detached,
-- un-pinned state — the rows survive (never auto-deleted) and can be re-attached
-- on a new host. Also required before DeleteHost so the host_id FK does not
-- block deletion.
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
host_id = NULL,
mount_path = '',
last_updated = NOW()
WHERE host_id = $1;
-- name: BeginVolumeDelete :one
-- Atomically claim a volume for deletion. The CAS blocks a concurrent attach
-- (which needs status='detached') so a volume can never be attached and deleted
-- at the same time. 'deleting' is also accepted so a delete that crashed
-- mid-flight can be retried rather than leaving the volume permanently stuck.
-- Returns the row (for host_id) when claimed; no row means it is not deletable
-- (attached/attaching, wrong team, or missing).
UPDATE volumes
SET status = 'deleting',
last_updated = NOW()
WHERE id = $1 AND team_id = $2 AND status IN ('detached', 'deleting')
RETURNING *;
-- name: AbortVolumeDelete :exec
-- Revert a delete claim back to detached when the host-side file removal fails,
-- so the volume stays usable rather than stuck in 'deleting'.
UPDATE volumes
SET status = 'detached',
last_updated = NOW()
WHERE id = $1 AND status = 'deleting';
-- name: ReleaseStaleVolumeReservations :execrows
-- Free volumes stuck in a transient state whose owning operation died before it
-- could ever reach a host — the control plane crashed between reserving a
-- volume and inserting the sandbox row, or mid-delete. Run periodically by the
-- volume reaper.
--
-- Deliberately limited to unattributed rows (sandbox_id IS NULL). A reservation
-- that already carries a sandbox_id may correspond to a capsule that really did
-- boot with the volume mounted — and a timer cannot tell the difference. Those
-- are freed only by the host-monitor sweep, which first confirms against the
-- host that the capsule is gone. Freeing a live volume here would let a second
-- capsule attach the same backing file and corrupt it.
--
-- The cutoff ($1 = now - grace) MUST exceed the capsule create timeout so a
-- legitimately in-flight attach is never freed out from under a booting capsule.
UPDATE volumes
SET status = 'detached',
mount_path = '',
last_updated = NOW()
WHERE status IN ('attaching', 'deleting')
AND sandbox_id IS NULL
AND last_updated < $1;
-- name: DeleteVolumeRow :exec
-- Drop the row only while the delete claim still holds. Without the status
-- guard a delete that lost its claim mid-flight (reaped, or aborted and then
-- re-attached by a racing capsule create) would erase the record of a volume
-- that is once again in use.
DELETE FROM volumes WHERE id = $1 AND team_id = $2 AND status = 'deleting';
-- name: DeleteVolumesByTeam :exec
-- Drop every volume record belonging to a team being deleted. This is the one
-- place volumes are removed without an explicit per-volume delete: the team that
-- owns them is going away, so there is no longer anyone who could reference,
-- re-attach, or delete them. Unlike DeleteVolumeRow there is no status guard —
-- the team's capsules have already been destroyed, so no volume can still be
-- legitimately in use, and a row left behind would be unreachable forever (and
-- would keep blocking deletion of the host it is pinned to).
DELETE FROM volumes WHERE team_id = $1;

2
envd-rs/Cargo.lock generated
View File

@ -529,7 +529,7 @@ dependencies = [
[[package]]
name = "envd"
version = "0.6.1"
version = "0.7.0"
dependencies = [
"async-stream",
"axum",

View File

@ -1,6 +1,6 @@
[package]
name = "envd"
version = "0.6.1"
version = "0.7.0"
edition = "2024"
rust-version = "1.95"

View File

@ -22,25 +22,17 @@ pub struct InitRequest {
#[serde(rename = "hyperloop_ip")]
pub hyperloop_ip: Option<String>,
pub timestamp: Option<String>,
#[serde(rename = "volume_mounts")]
pub volume_mounts: Option<Vec<VolumeMount>>,
pub sandbox_id: Option<String>,
pub template_id: Option<String>,
/// Public proxy domain (e.g. "wrenn.dev"). Used by `envd ports` to build
/// the {port}-{sandbox_id}.{domain} URLs.
pub proxy_domain: Option<String>,
/// New lifecycle identifier for this resume. When it changes between
/// /init calls, envd treats the call as a post-resume hook: port
/// forwarder is restarted and NFS mounts are refreshed.
/// /init calls, envd treats the call as a post-resume hook: the port
/// forwarder is restarted and the clock is stepped.
pub lifecycle_id: Option<String>,
}
#[derive(Deserialize)]
pub struct VolumeMount {
pub nfs_target: String,
pub path: String,
}
/// POST /init — called by host agent after boot.
pub async fn post_init(
State(state): State<Arc<AppState>>,
@ -183,20 +175,6 @@ pub async fn post_init(
setup_hyperloop(ip, &state.defaults.env_vars).await;
}
// NFS mounts. Awaited in parallel so callers that immediately access the
// mount path don't race the mount(2). Previously these were detached via
// tokio::spawn, which let /init return success before mounts existed.
if let Some(ref mounts) = init_req.volume_mounts {
let futs = mounts.iter().map(|m| {
let target = m.nfs_target.clone();
let path = m.path.clone();
async move {
setup_nfs(&target, &path).await;
}
});
futures::future::join_all(futs).await;
}
// Set sandbox/template metadata from request body. Deliberately NOT
// written into envd's own process environment: std::env::set_var is
// undefined behavior with the multi-threaded runtime live (concurrent
@ -293,45 +271,6 @@ async fn setup_hyperloop(address: &str, env_vars: &dashmap::DashMap<String, Stri
env_vars.insert("WRENN_EVENTS_ADDRESS".into(), format!("http://{address}"));
}
async fn setup_nfs(nfs_target: &str, path: &str) {
let mkdir = tokio::process::Command::new("mkdir")
.args(["-p", path])
.output()
.await;
if let Err(e) = mkdir {
tracing::error!(error = %e, path, "nfs: mkdir failed");
return;
}
let mount = tokio::process::Command::new("mount")
.args([
"-v",
"-t",
"nfs",
"-o",
"mountproto=tcp,mountport=2049,proto=tcp,port=2049,nfsvers=3,noacl",
nfs_target,
path,
])
.output()
.await;
match mount {
Ok(output) => {
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if output.status.success() {
tracing::info!(nfs_target, path, stdout = %stdout, "nfs: mount success");
} else {
tracing::error!(nfs_target, path, stderr = %stderr, "nfs: mount failed");
}
}
Err(e) => {
tracing::error!(error = %e, nfs_target, path, "nfs: mount command failed");
}
}
}
fn write_run_file(name: &str, value: &str) {
let dir = std::path::Path::new(crate::config::WRENN_RUN_DIR);
if let Err(e) = std::fs::create_dir_all(dir) {

View File

@ -7,6 +7,7 @@ pub mod init;
pub mod memory;
pub mod metrics;
pub mod snapshot;
pub mod volumes;
use std::sync::Arc;
use std::time::Duration;
@ -61,6 +62,8 @@ pub fn router(state: Arc<AppState>) -> Router {
post(memory::post_memory_preload_cancel),
)
.route("/files", get(files::get_files).put(files::put_files))
.route("/volumes/mount", post(volumes::post_mount))
.route("/volumes/unmount", post(volumes::post_unmount))
.layer(cors)
.with_state(state)
}

252
envd-rs/src/http/volumes.rs Normal file
View File

@ -0,0 +1,252 @@
use std::sync::Arc;
use std::time::Duration;
use axum::Json;
use axum::extract::State;
use axum::http::{StatusCode, header};
use axum::response::{IntoResponse, Response};
use serde::Deserialize;
use crate::state::AppState;
#[derive(Deserialize)]
pub struct MountRequest {
/// virtio-blk serial set by the host in the VM config; used to resolve the
/// block device via /sys/block/*/serial regardless of enumeration order.
pub serial: String,
/// Guest path to mount the volume at (e.g. "/mnt/vol-...").
pub mount_path: String,
}
#[derive(Deserialize)]
pub struct UnmountRequest {
pub mount_path: String,
}
fn json_error(status: StatusCode, msg: &str) -> Response {
let body = serde_json::json!({ "code": status.as_u16(), "message": msg });
(status, Json(body)).into_response()
}
fn no_content() -> Response {
(
StatusCode::NO_CONTENT,
[(header::CACHE_CONTROL, "no-store")],
)
.into_response()
}
/// POST /volumes/mount — called by the host agent after boot for each attached
/// volume. Resolves the block device by its virtio-blk serial, formats it with
/// ext4 only if it has no filesystem yet (existing data is never reformatted),
/// then mounts it at the requested path.
pub async fn post_mount(
State(_state): State<Arc<AppState>>,
Json(req): Json<MountRequest>,
) -> Response {
// Serials are host-generated hex tokens; reject anything else before it
// reaches a sysfs comparison / command argument.
if req.serial.is_empty() || !req.serial.chars().all(|c| c.is_ascii_alphanumeric()) {
return json_error(StatusCode::BAD_REQUEST, "invalid volume serial");
}
if req.mount_path.is_empty() {
return json_error(StatusCode::BAD_REQUEST, "mount_path is required");
}
let device = match resolve_device_by_serial(&req.serial).await {
Some(d) => d,
None => {
return json_error(
StatusCode::NOT_FOUND,
&format!("no block device with serial {}", req.serial),
);
}
};
// blkid exit status: 0 = a filesystem/signature is present (leave it alone),
// 2 = no recognized signature (format it), anything else = probe error.
// Tracked so the permissions of an existing volume are never touched — only
// a filesystem this call created gets opened up below.
let formatted = match has_filesystem(&device).await {
Ok(true) => {
tracing::info!(device, "volume already has a filesystem; skipping mkfs");
false
}
Ok(false) => {
if let Err(e) = mkfs_ext4(&device).await {
return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e);
}
true
}
Err(e) => {
return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e);
}
};
if let Err(e) = tokio::fs::create_dir_all(&req.mount_path).await {
return json_error(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("create mount dir: {e}"),
);
}
// No -t: let the kernel autodetect the filesystem, so a pre-existing
// non-ext4 volume still mounts.
match tokio::process::Command::new("mount")
.arg(&device)
.arg(&req.mount_path)
.output()
.await
{
Ok(out) if out.status.success() => {
if formatted {
open_mount_root(&req.mount_path);
}
tracing::info!(device, mount_path = %req.mount_path, "volume mounted");
no_content()
}
Ok(out) => {
let stderr = String::from_utf8_lossy(&out.stderr);
// Already mounted at the target is success for an idempotent retry.
if stderr.contains("already mounted") {
return no_content();
}
json_error(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("mount failed: {}", stderr.trim()),
)
}
Err(e) => json_error(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("mount command failed: {e}"),
),
}
}
/// open_mount_root relaxes the permissions of a just-formatted volume so every
/// user in the capsule can use it.
///
/// mkfs.ext4 leaves the filesystem root owned by root with mode 0755, which
/// locks out every non-root user — including the template's default user, which
/// is who the file API and exec run as. Without this a caller has to sudo to
/// write to their own volume. 1777 is /tmp's mode: world-writable, with the
/// sticky bit so one user cannot remove another's files.
///
/// Only ever called straight after a first-time format. Re-mounting an existing
/// volume leaves its permissions exactly as its owner last set them.
///
/// Applied to the mounted root rather than the directory created before the
/// mount — that one is hidden underneath and changing it would have no effect.
///
/// Best-effort: a failure here leaves a root-only volume that is still mounted
/// and holds its data, which is not worth failing the whole capsule create over.
fn open_mount_root(mount_path: &str) {
use std::os::unix::fs::PermissionsExt;
if let Err(e) = std::fs::set_permissions(mount_path, std::fs::Permissions::from_mode(0o1777)) {
tracing::warn!(
mount_path,
error = %e,
"failed to relax volume permissions; only root will be able to write to it"
);
}
}
/// POST /volumes/unmount — called best-effort before a graceful capsule destroy.
/// Flushes buffered writes (sync) and unmounts so the backing file is
/// consistent. Idempotent: unmounting an already-unmounted path is fine.
pub async fn post_unmount(
State(_state): State<Arc<AppState>>,
Json(req): Json<UnmountRequest>,
) -> Response {
if req.mount_path.is_empty() {
return json_error(StatusCode::BAD_REQUEST, "mount_path is required");
}
// sync flushes all filesystems, guaranteeing the volume's data reaches its
// backing file even if the umount below is skipped or fails.
let _ = tokio::task::spawn_blocking(nix::unistd::sync).await;
let _ = tokio::process::Command::new("umount")
.arg(&req.mount_path)
.output()
.await;
no_content()
}
/// resolve_device_by_serial finds the /dev path of the virtio-blk device whose
/// serial matches. Polls briefly since a boot-time disk may not be fully
/// enumerated the instant envd is asked to mount it.
async fn resolve_device_by_serial(serial: &str) -> Option<String> {
for _ in 0..50 {
if let Some(dev) = find_block_by_serial(serial) {
return Some(dev);
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
None
}
fn find_block_by_serial(serial: &str) -> Option<String> {
let entries = std::fs::read_dir("/sys/block").ok()?;
for entry in entries.flatten() {
let name = entry.file_name();
let name = name.to_string_lossy();
// Only virtio-blk devices carry a serial we set; skip loop/ram/etc.
if !name.starts_with("vd") {
continue;
}
let serial_path = format!("/sys/block/{name}/serial");
if let Ok(s) = std::fs::read_to_string(&serial_path) {
if s.trim() == serial {
return Some(format!("/dev/{name}"));
}
}
}
None
}
/// has_filesystem returns true when blkid detects any filesystem signature on
/// the device. Guards mkfs so existing data is never reformatted.
async fn has_filesystem(device: &str) -> Result<bool, String> {
let out = tokio::process::Command::new("blkid")
.arg(device)
.output()
.await
.map_err(|e| format!("blkid spawn failed: {e}"))?;
match out.status.code() {
// 0 with output = a signature was printed. Guard on the output too so a
// blkid variant that always exits 0 (e.g. busybox) is handled: empty
// output means no filesystem.
Some(0) => Ok(!String::from_utf8_lossy(&out.stdout).trim().is_empty()),
// util-linux blkid: 2 = nothing detected.
Some(2) => Ok(false),
// 4 (usage) / 8 (error) / signal — treat as a probe failure.
other => Err(format!(
"blkid probe failed (exit {:?}): {}",
other,
String::from_utf8_lossy(&out.stderr).trim()
)),
}
}
async fn mkfs_ext4(device: &str) -> Result<(), String> {
// -F: the target is a whole virtio-blk device, not a partition; without it
// mkfs.ext4 prompts and would hang. Safe here — only reached when blkid
// found no existing signature.
let out = tokio::process::Command::new("mkfs.ext4")
.args(["-q", "-F", device])
.output()
.await
.map_err(|e| format!("mkfs.ext4 spawn failed: {e}"))?;
if out.status.success() {
tracing::info!(device, "formatted volume with ext4");
Ok(())
} else {
Err(format!(
"mkfs.ext4 failed: {}",
String::from_utf8_lossy(&out.stderr).trim()
))
}
}

View File

@ -10,7 +10,7 @@ source "$(cd "$(dirname "$0")" && pwd)/build-common.sh"
# Alpine is musl-based: the static envd + static tini run fine. bash is added so
# wrenn-user has a familiar login shell; wrenn-init itself only needs /bin/sh.
PREP="set -e
apk add --no-cache socat chrony sudo wget curl ca-certificates git iproute2 tini bash gcc make nano vim
apk add --no-cache socat chrony sudo wget curl ca-certificates git iproute2 tini bash make nano vim e2fsprogs util-linux
adduser -D wrenn-user
${WRENN_SUDOERS_SETUP}"

View File

@ -12,7 +12,7 @@ source "$(cd "$(dirname "$0")" && pwd)/build-common.sh"
# tini is AUR-only on Arch (not in core/extra), so it is not installed here —
# rootfs-from-container.sh injects the static tini binary instead.
PREP="set -e
pacman -Sy --noconfirm --needed socat chrony sudo wget curl ca-certificates git iproute2 inetutils gcc make nano vim
pacman -Sy --noconfirm --needed socat chrony sudo wget curl ca-certificates git iproute2 inetutils make nano vim e2fsprogs
useradd -m -s /bin/bash wrenn-user
${WRENN_SUDOERS_SETUP}
pacman -Scc --noconfirm || true"

View File

@ -11,7 +11,7 @@ source "$(cd "$(dirname "$0")" && pwd)/build-common.sh"
PREP="set -e
# install_weak_deps=False keeps the image lean. The guest never runs systemd:
# PID 1 is wrenn-init -> tini -> envd.
dnf install -y --setopt=install_weak_deps=False socat chrony sudo wget curl ca-certificates git iproute hostname tini gcc make nano vim
dnf install -y --setopt=install_weak_deps=False socat chrony sudo wget curl ca-certificates git iproute hostname tini make nano vim e2fsprogs
useradd -m -s /bin/bash wrenn-user
${WRENN_SUDOERS_SETUP}
dnf clean all"

View File

@ -12,7 +12,7 @@ export DEBIAN_FRONTEND=noninteractive
apt-get update
# --no-install-recommends keeps the image lean (avoids pulling systemd-adjacent
# recommends). The guest never runs systemd: PID 1 is wrenn-init -> tini -> envd.
apt-get install -y --no-install-recommends socat chrony sudo wget curl ca-certificates git iproute2 hostname tini gcc make nano vim
apt-get install -y --no-install-recommends socat chrony sudo wget curl ca-certificates git iproute2 hostname tini make nano vim e2fsprogs
# Remove the stock 'ubuntu' user (uid 1000) shipped by the base image; it is
# replaced by wrenn-user. Also drop its cloud-init sudoers drop-in.
userdel -r ubuntu 2>/dev/null || true

View File

@ -30,6 +30,10 @@ type createSandboxRequest struct {
MemoryMB int32 `json:"memory_mb"`
TimeoutSec int32 `json:"timeout_sec"`
Metadata map[string]string `json:"metadata"`
// VolumeIDs are external storage volumes to attach at boot, each given as
// a volume ID ("vol-...") or a name ("vl-cache"). Each is mounted at
// /mnt/<volume-id> inside the guest.
VolumeIDs []string `json:"volume_ids"`
}
type sandboxResponse struct {
@ -102,6 +106,7 @@ func (h *sandboxHandler) Create(w http.ResponseWriter, r *http.Request) {
MemoryMB: req.MemoryMB,
TimeoutSec: req.TimeoutSec,
Metadata: req.Metadata,
VolumeRefs: req.VolumeIDs,
})
h.audit.LogSandboxCreate(r.Context(), ac, sb.ID, req.Template, err)
if err != nil {

View File

@ -0,0 +1,166 @@
package api
import (
"net/http"
"strings"
"time"
"github.com/go-chi/chi/v5"
"git.omukk.dev/wrenn/wrenn/internal/units"
"git.omukk.dev/wrenn/wrenn/pkg/apperr"
"git.omukk.dev/wrenn/wrenn/pkg/audit"
"git.omukk.dev/wrenn/wrenn/pkg/auth"
"git.omukk.dev/wrenn/wrenn/pkg/db"
"git.omukk.dev/wrenn/wrenn/pkg/id"
"git.omukk.dev/wrenn/wrenn/pkg/service"
)
type volumeHandler struct {
svc *service.VolumeService
audit *audit.AuditLogger
}
func newVolumeHandler(svc *service.VolumeService, al *audit.AuditLogger) *volumeHandler {
return &volumeHandler{svc: svc, audit: al}
}
type createVolumeRequest struct {
// Name is optional; omitted, the volume is named after its own ID.
Name string `json:"name"`
// Size is the human-readable form ("20Gi", "500M"). SizeMB is the plain
// megabyte form. Exactly one is needed; Size wins if both are given.
Size string `json:"size"`
SizeMB int32 `json:"size_mb"`
}
// sizeMB resolves the request's size into megabytes, accepting either field.
func (r createVolumeRequest) sizeMB() (int32, error) {
if s := strings.TrimSpace(r.Size); s != "" {
mb, err := units.ParseSizeToMB(s)
if err != nil {
return 0, apperr.InvalidRequest.WrapMsg(err, "Invalid volume size.")
}
return int32(mb), nil
}
if r.SizeMB <= 0 {
return 0, apperr.InvalidRequest.Msg(`A volume size is required (e.g. "size": "20Gi" or "size_mb": 20480).`)
}
return r.SizeMB, nil
}
type volumeResponse struct {
ID string `json:"id"`
TeamID string `json:"team_id"`
Name string `json:"name"`
SizeMB int32 `json:"size_mb"`
Status string `json:"status"`
HostID *string `json:"host_id,omitempty"`
SandboxID *string `json:"sandbox_id,omitempty"`
// Always emitted, "" while detached — host_id/sandbox_id above are pointers
// because they are genuinely nullable, but mount_path is not.
MountPath string `json:"mount_path"`
CreatedAt string `json:"created_at"`
LastAttachedAt *string `json:"last_attached_at,omitempty"`
}
func volumeToResponse(v db.Volume) volumeResponse {
resp := volumeResponse{
ID: id.FormatVolumeID(v.ID),
TeamID: id.FormatTeamID(v.TeamID),
Name: v.Name,
SizeMB: v.SizeMb,
Status: v.Status,
MountPath: v.MountPath,
}
if v.HostID.Valid {
s := id.FormatHostID(v.HostID)
resp.HostID = &s
}
if v.SandboxID.Valid {
s := id.FormatSandboxID(v.SandboxID)
resp.SandboxID = &s
}
if v.CreatedAt.Valid {
resp.CreatedAt = v.CreatedAt.Time.Format(time.RFC3339)
}
if v.LastAttachedAt.Valid {
s := v.LastAttachedAt.Time.Format(time.RFC3339)
resp.LastAttachedAt = &s
}
return resp
}
// Create handles POST /v1/volumes.
func (h *volumeHandler) Create(w http.ResponseWriter, r *http.Request) {
ac := auth.MustFromContext(r.Context())
var req createVolumeRequest
if err := decodeJSON(r, &req); err != nil {
writeErr(w, r, apperr.InvalidRequest.WrapMsg(err, "Invalid JSON body."))
return
}
sizeMB, err := req.sizeMB()
if err != nil {
writeErr(w, r, err)
return
}
vol, err := h.svc.Create(r.Context(), ac.TeamID, req.Name, sizeMB)
if err != nil {
writeErr(w, r, err)
return
}
h.audit.LogVolumeCreate(r.Context(), ac, vol.ID, vol.Name, int(vol.SizeMb), nil)
writeJSON(w, http.StatusCreated, volumeToResponse(vol))
}
// List handles GET /v1/volumes.
func (h *volumeHandler) List(w http.ResponseWriter, r *http.Request) {
ac := auth.MustFromContext(r.Context())
vols, err := h.svc.List(r.Context(), ac.TeamID)
if err != nil {
writeErr(w, r, err)
return
}
resp := make([]volumeResponse, len(vols))
for i, v := range vols {
resp[i] = volumeToResponse(v)
}
writeJSON(w, http.StatusOK, resp)
}
// Get handles GET /v1/volumes/{id}, where {id} is a volume ID or name.
func (h *volumeHandler) Get(w http.ResponseWriter, r *http.Request) {
ac := auth.MustFromContext(r.Context())
vol, err := h.svc.Resolve(r.Context(), ac.TeamID, chi.URLParam(r, "id"))
if err != nil {
writeErr(w, r, err)
return
}
writeJSON(w, http.StatusOK, volumeToResponse(vol))
}
// Delete handles DELETE /v1/volumes/{id}, where {id} is a volume ID or name.
func (h *volumeHandler) Delete(w http.ResponseWriter, r *http.Request) {
ac := auth.MustFromContext(r.Context())
vol, err := h.svc.Resolve(r.Context(), ac.TeamID, chi.URLParam(r, "id"))
if err != nil {
writeErr(w, r, err)
return
}
if err := h.svc.Delete(r.Context(), vol.ID, ac.TeamID); err != nil {
writeErr(w, r, err)
return
}
h.audit.LogVolumeDelete(r.Context(), ac, vol.ID, nil)
w.WriteHeader(http.StatusNoContent)
}

View File

@ -65,6 +65,19 @@ func NewHostMonitor(queries *db.Queries, pool *lifecycle.HostClientPool, al *aud
}
}
// detachVolumes frees any volumes attached to sandboxes the monitor has just
// moved to a terminal state, so a volume is never stranded on a capsule that
// died via host loss or reconciliation rather than an explicit destroy.
// Idempotent — only 'attached' rows change.
func (m *HostMonitor) detachVolumes(ctx context.Context, sandboxIDs []pgtype.UUID) {
if len(sandboxIDs) == 0 {
return
}
if err := m.db.DetachVolumesBySandboxIDs(ctx, sandboxIDs); err != nil {
slog.Warn("host monitor: failed to detach volumes for stopped sandboxes", "error", err)
}
}
// Start runs the monitor loop until the context is cancelled.
func (m *HostMonitor) Start(ctx context.Context) {
go func() {
@ -229,6 +242,7 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
}); err != nil {
slog.Warn("host monitor: failed to stop missing sandboxes", "host_id", id.FormatHostID(host.ID), "error", err)
} else {
m.detachVolumes(ctx, ids)
for _, sb := range toStop {
m.audit.LogSandboxDestroySystem(ctx, sb.TeamID, sb.ID, "orphaned", nil)
}
@ -279,6 +293,7 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
}); err != nil {
slog.Warn("host monitor: failed to mark stopped", "host_id", id.FormatHostID(host.ID), "error", err)
} else {
m.detachVolumes(ctx, ids)
for _, sb := range toStop {
m.audit.LogSandboxDestroySystem(ctx, sb.TeamID, sb.ID, "orphaned", nil)
}
@ -378,6 +393,26 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
slog.Info("host monitor: promoted transient sandbox to running", "sandbox_id", sbIDStr, "from", sb.Status)
}
}
// A capsule the host still reports alive while the DB says
// "stopping" means its destroy never landed — the control plane
// exhausted its retries, or died mid-destroy. Nothing else re-sends
// it, so without this the capsule sits in "stopping" forever and its
// volumes stay claimed. Re-issue against confirmed-live host state;
// the volumes are freed only once the host says it is gone.
if sb.Status == "stopping" &&
sb.LastUpdated.Valid && time.Since(sb.LastUpdated.Time) >= transientGracePeriod {
slog.Info("host monitor: re-issuing destroy for capsule stuck stopping", "sandbox_id", sbIDStr)
if _, err := agent.DestroySandbox(ctx, connect.NewRequest(&pb.DestroySandboxRequest{
SandboxId: sbIDStr,
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
slog.Warn("host monitor: destroy retry failed", "sandbox_id", sbIDStr, "error", err)
} else if _, err := m.db.UpdateSandboxStatusIf(ctx, db.UpdateSandboxStatusIfParams{
ID: sb.ID, Status: "stopping", Status_2: "stopped",
}); err == nil {
m.detachVolumes(ctx, []pgtype.UUID{sb.ID})
m.audit.LogSandboxDestroySystem(ctx, sb.TeamID, sb.ID, "destroy_retry", nil)
}
}
// A snapshot keeps the source sandbox alive throughout, so an alive
// sandbox does NOT mean the snapshot finished. Only recover it once
// it has been stuck past the snapshot grace period (i.e. the CP
@ -427,6 +462,11 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
ID: sb.ID, Status: fromStatus, Status_2: finalStatus,
}); err == nil {
slog.Info("host monitor: resolved transient sandbox", "sandbox_id", sbIDStr, "from", fromStatus, "to", finalStatus)
// error/stopped are terminal — free any attached volumes. "paused"
// (from pausing) is recoverable, so the volume stays attached.
if finalStatus == "error" || finalStatus == "stopped" {
m.detachVolumes(ctx, []pgtype.UUID{sb.ID})
}
inferredErr := errInferredTransientTimeout
switch fromStatus {
case "starting":

View File

@ -782,7 +782,9 @@ paths:
- sessionAuth: []
description: |
Owner only. Soft-deletes the team and destroys all running/paused/starting
capsules. All DB records are preserved. The team slug is permanently reserved.
capsules. Team-owned templates and storage volumes are deleted along with
their data; capsule records are preserved. The team slug is permanently
reserved.
responses:
"204":
description: Team deleted
@ -1582,6 +1584,113 @@ paths:
"503":
$ref: "#/components/responses/ServiceUnavailable"
/v1/volumes:
post:
summary: Create a storage volume
operationId: createVolume
tags: [volumes]
security:
- apiKeyAuth: []
- sessionAuth: []
description: |
Create an external storage volume. The volume starts detached and is
not placed on any host until it is first attached to a capsule (via
`volume_ids` on capsule create), at which point it is pinned to that
capsule's host for the rest of its life. Destroying a capsule only frees
its volumes, never deletes them; the one case where a volume is removed
without an explicit delete is deleting the team that owns it.
Every volume has a `vl-`-prefixed name that is unique within your team
and can be used anywhere a volume ID is accepted. Omit `name` and the
volume is named after its own ID.
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/CreateVolumeRequest"
responses:
"201":
description: Volume created
content:
application/json:
schema:
$ref: "#/components/schemas/Volume"
"400":
$ref: "#/components/responses/BadRequest"
"409":
description: A volume with this name already exists
content:
application/json:
schema:
$ref: "#/components/schemas/Error"
get:
summary: List storage volumes
operationId: listVolumes
tags: [volumes]
security:
- apiKeyAuth: []
- sessionAuth: []
responses:
"200":
description: List of volumes
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Volume"
/v1/volumes/{id}:
get:
summary: Get a storage volume
operationId: getVolume
tags: [volumes]
security:
- apiKeyAuth: []
- sessionAuth: []
description: |
Look up a volume by ID (`vol-...`) or by name (`vl-cache`, or plain
`cache` — the `vl-` prefix is optional on input).
parameters:
- $ref: "#/components/parameters/VolumeRef"
responses:
"200":
description: The volume
content:
application/json:
schema:
$ref: "#/components/schemas/Volume"
"400":
$ref: "#/components/responses/BadRequest"
"404":
$ref: "#/components/responses/NotFound"
delete:
summary: Delete a storage volume
operationId: deleteVolume
tags: [volumes]
security:
- apiKeyAuth: []
- sessionAuth: []
description: |
Delete a volume and its data. The volume must be detached — destroy any
capsule using it first. Accepts an ID or a name.
parameters:
- $ref: "#/components/parameters/VolumeRef"
responses:
"204":
description: Volume deleted
"400":
$ref: "#/components/responses/BadRequest"
"404":
$ref: "#/components/responses/NotFound"
"409":
description: Volume is attached to a capsule
content:
application/json:
schema:
$ref: "#/components/schemas/Error"
/v1/snapshots:
post:
summary: Create a snapshot template
@ -3933,6 +4042,19 @@ paths:
$ref: "#/components/responses/NotFound"
components:
parameters:
VolumeRef:
name: id
in: path
required: true
description: >
A volume ID (`vol-<base36>`) or name (`vl-cache`). The `vl-` prefix is
optional on input, so `cache` resolves the same volume as `vl-cache`.
The two namespaces cannot collide — IDs always start with `vol-`.
schema:
type: string
example: vl-cache
responses:
BadRequest:
description: Invalid request parameters
@ -4132,6 +4254,98 @@ components:
are at most 64 characters. Reserved system keys (kernel_version,
vmm_version, agent_version, envd_version) are rejected. Metadata is
set at create-time only.
volume_ids:
type: array
items: { type: string }
nullable: true
maxItems: 4
example: ["vl-cache", "vol-8f3kq2m1p7x0a9b4c6d8e2f5g"]
description: >
External storage volumes to attach at boot, each given as a volume
ID ("vol-...") or a name ("vl-cache"). Each is mounted at
/mnt/<volume-id> inside the guest, formatted on first use and
writable by any user in the capsule. At most 4 per capsule. Volumes
can only be attached at create time and must be detached and owned
by your team. If a volume is already pinned to a host, the capsule is
scheduled onto that host; attaching volumes pinned to different hosts
fails. Not supported for capsules created from a snapshot template.
CreateVolumeRequest:
type: object
description: >
Supply the size as either `size` or `size_mb`. `name` is optional.
properties:
name:
type: string
maxLength: 40
example: vl-cache
description: >
Name for the volume, unique within your team and usable anywhere a
volume ID is accepted. Lowercase letters, numbers, and dashes only
(no leading, trailing, or repeated dashes), at most 40 characters
including the `vl-` prefix. The prefix is added if you omit it, so
`cache` and `vl-cache` name the same volume. Omit the field entirely
and the volume is named after its own ID.
size:
type: string
example: 20Gi
description: >
Volume size with a G/Gi/M/Mi suffix; a bare number is read as MB.
Fixed at creation. Takes precedence over `size_mb` if both are set.
size_mb:
type: integer
minimum: 100
example: 5120
description: >
Volume size in MB, as an alternative to `size`. Fixed at creation.
The backing storage is sparse and grows on use, so the size is a
ceiling rather than an upfront allocation. The maximum is set per
deployment via `WRENN_MAX_VOLUME_SIZE` (default 20Gi); requesting
more returns 400.
Volume:
type: object
required: [id, team_id, name, size_mb, status, mount_path, created_at]
properties:
id:
type: string
description: Volume ID (e.g. "vol-...").
team_id:
type: string
name:
type: string
description: >
The volume's `vl-`-prefixed name, unique within the team. Accepted
in place of the ID on any volume route and in `volume_ids`.
example: vl-cache
size_mb:
type: integer
status:
type: string
enum: [detached, attaching, attached, deleting]
description: >
`detached` — free to attach or delete. `attaching` — reserved by a
capsule that is booting. `attached` — in use by a capsule.
`deleting` — a delete is in flight; transient, and reverts to
`detached` if the delete fails.
host_id:
type: string
nullable: true
description: Host the volume is pinned to; null until first attached.
sandbox_id:
type: string
nullable: true
description: Capsule the volume is currently attached to, if any.
mount_path:
type: string
description: Guest mount path while attached (e.g. "/mnt/vol-...").
created_at:
type: string
format: date-time
last_attached_at:
type: string
format: date-time
nullable: true
UsageResponse:
type: object

View File

@ -260,7 +260,20 @@ func (c *SandboxEventConsumer) handleAutoPaused(ctx context.Context, sandboxID p
}
}
// detachVolumes frees any volumes still attached to a sandbox that has reached
// a terminal state. Idempotent (only 'attached' rows change) and keyed on
// sandbox_id, so it runs safely from every terminal path — the explicit destroy
// flow, the TTL-reaper fallback, and the crash/failure path — not just
// SandboxService.Destroy.
func (c *SandboxEventConsumer) detachVolumes(ctx context.Context, sandboxID pgtype.UUID) {
if err := c.db.DetachVolumesBySandbox(ctx, sandboxID); err != nil {
slog.Warn("sandbox event consumer: failed to detach volumes", "sandbox_id", id.FormatSandboxID(sandboxID), "error", err)
}
}
func (c *SandboxEventConsumer) handleStopped(ctx context.Context, sandboxID pgtype.UUID) {
c.detachVolumes(ctx, sandboxID)
// stopping → stopped (CP-initiated destroy completed). No audit row here;
// the handler that issued the destroy already wrote one.
if _, err := c.db.UpdateSandboxStatusIf(ctx, db.UpdateSandboxStatusIfParams{
@ -289,6 +302,8 @@ func (c *SandboxEventConsumer) handleStopped(ctx context.Context, sandboxID pgty
// audit.Log writes the row only — it does NOT republish an event, which would
// loop back into this consumer. Do not switch to LogSandboxCreateSystem here.
func (c *SandboxEventConsumer) handleFailed(ctx context.Context, sandboxID pgtype.UUID, event events.Event) {
c.detachVolumes(ctx, sandboxID)
for _, fromStatus := range []string{"running", "starting", "pausing", "resuming", "snapshotting"} {
if _, err := c.db.UpdateSandboxStatusIf(ctx, db.UpdateSandboxStatusIfParams{
ID: sandboxID, Status: fromStatus, Status_2: "error",

View File

@ -82,7 +82,8 @@ func New(
}
// Shared service layer.
sandboxSvc := &service.SandboxService{DB: queries, Pool: pool, Scheduler: sched}
volumeSvc := &service.VolumeService{DB: queries, Pool: pool, MaxSizeMB: int32(sctx.Config.MaxVolumeSizeMB)}
sandboxSvc := &service.SandboxService{DB: queries, Pool: pool, Scheduler: sched, Volumes: volumeSvc}
sandboxSvc.PublishEvent = func(ctx context.Context, event service.SandboxStateEvent) {
if evt, ok := serviceEventToCanonical(event); ok {
// State-change events are ephemeral UI signals — mirror them to the
@ -116,6 +117,7 @@ func New(
authH := newAuthHandler(queries, pgPool, sessionSvc, mailer, rdb, oauthRedirectURL, authHooks)
oauthH := newOAuthHandler(queries, pgPool, jwtSecret, sessionSvc, oauthRegistry, oauthRedirectURL, authHooks)
apiKeys := newAPIKeyHandler(apiKeySvc, al)
volumes := newVolumeHandler(volumeSvc, al)
hostH := newHostHandler(hostSvc, queries, al, monitor)
teamH := newTeamHandler(teamSvc, al, mailer, sessionSvc)
usersH := newUsersHandler(queries, userSvc, al, sessionSvc)
@ -278,6 +280,16 @@ func New(
r.Delete("/{name}", snapshots.Delete)
})
// External storage volumes: API key (SDK) or session (browser).
r.Route("/v1/volumes", func(r chi.Router) {
r.Use(requireSessionOrAPIKey(queries, sessionSvc))
r.Use(csrf)
r.Post("/", volumes.Create)
r.Get("/", volumes.List)
r.Get("/{id}", volumes.Get)
r.Delete("/{id}", volumes.Delete)
})
// Host management.
r.Route("/v1/hosts", func(r chi.Router) {
// Unauthenticated: one-time registration token.
@ -480,6 +492,14 @@ func serviceEventToCanonical(e service.SandboxStateEvent) (events.Event, bool) {
case "sandbox.stopped":
eventType = events.CapsuleDestroy
outcome = events.OutcomeSuccess
case "sandbox.destroy_failed":
// The destroy RPC never reached the host, so the capsule stays in
// "stopping" holding its volumes until the host monitor re-issues it.
// This is the only lifecycle path that reaches no terminal state of its
// own, so the event is what tells anyone it is outstanding.
eventType = events.CapsuleDestroy
outcome = events.OutcomeError
metadata = map[string]string{"reason": "destroy_failed"}
case "sandbox.pause_failed":
// reason must be non-empty or channels.isRedundantSystemFollowup
// filters this system-actor event out of webhook delivery.

View File

@ -0,0 +1,72 @@
package api
import (
"context"
"log/slog"
"time"
"github.com/jackc/pgx/v5/pgtype"
"git.omukk.dev/wrenn/wrenn/pkg/db"
)
// VolumeReaper periodically frees volumes stuck in a transient state
// (attaching/deleting) whose owning operation died before it could ever reach a
// host — a control-plane crash between reserving a volume and inserting the
// sandbox row, or a delete that crashed mid-flight. It is the safety net for
// the one volume-lifecycle window that no synchronous path can clean up.
//
// It deliberately only touches rows with no sandbox_id. Once a reservation
// names its capsule, it may correspond to a VM that really did boot with the
// volume mounted, and no timer can tell the difference — freeing it would let a
// second capsule attach the same backing file and corrupt it. Those rows are
// resolved by the host monitor instead, which first confirms against the host
// that the capsule is gone.
//
// A volume is released only once it has been stuck longer than staleAfter, which
// MUST exceed the capsule create timeout so an in-flight attach is never freed
// out from under a booting capsule.
type VolumeReaper struct {
db *db.Queries
interval time.Duration
staleAfter time.Duration
}
// NewVolumeReaper creates a VolumeReaper that sweeps every interval and releases
// volumes stuck in a transient state for longer than staleAfter.
func NewVolumeReaper(queries *db.Queries, interval, staleAfter time.Duration) *VolumeReaper {
return &VolumeReaper{db: queries, interval: interval, staleAfter: staleAfter}
}
// Start runs the reaper loop until the context is cancelled.
func (r *VolumeReaper) Start(ctx context.Context) {
go func() {
ticker := time.NewTicker(r.interval)
defer ticker.Stop()
// Run immediately on startup so a crash-orphaned reservation is cleared
// without waiting a full interval.
r.run(ctx)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
r.run(ctx)
}
}
}()
}
func (r *VolumeReaper) run(ctx context.Context) {
cutoff := pgtype.Timestamptz{Time: time.Now().Add(-r.staleAfter), Valid: true}
n, err := r.db.ReleaseStaleVolumeReservations(ctx, cutoff)
if err != nil {
slog.Warn("volume reaper: failed to release stale reservations", "error", err)
return
}
if n > 0 {
slog.Info("volume reaper: released stale volume reservations", "count", n)
}
}

View File

@ -78,12 +78,17 @@ func (s *Server) CreateSandbox(
return nil, err
}
volumes, err := volumeSpecsFromProto(msg.Volumes)
if err != nil {
return nil, apperr.ToConnect(apperr.InvalidRequest.Wrap(err))
}
// disk_size_mb in the request is deprecated and never set by the control
// plane; passing 0 lets the manager pick the size (DefaultRootfsSizeMB,
// floored at the origin image size).
sb, diskSizeBytes, err := s.mgr.Create(ctx, msg.SandboxId, teamID, templateID,
int(msg.Vcpus), int(msg.MemoryMb), int(msg.TimeoutSec), 0,
msg.DefaultUser, msg.DefaultEnv)
msg.DefaultUser, msg.DefaultEnv, volumes)
if err != nil {
return nil, mapSandboxError(fmt.Errorf("create sandbox: %w", err))
}
@ -182,6 +187,51 @@ func (s *Server) FlattenRootfs(
}), nil
}
// DeleteVolume removes a detached volume's backing file from this host.
func (s *Server) DeleteVolume(
_ context.Context,
req *connect.Request[pb.DeleteVolumeRequest],
) (*connect.Response[pb.DeleteVolumeResponse], error) {
teamID, err := parseUUIDString(req.Msg.TeamId)
if err != nil {
return nil, apperr.ToConnect(apperr.InvalidRequest.Wrap(err))
}
volumeID, err := parseUUIDString(req.Msg.VolumeId)
if err != nil {
return nil, apperr.ToConnect(apperr.InvalidRequest.Wrap(err))
}
if err := s.mgr.DeleteVolumeFile(teamID, volumeID); err != nil {
return nil, mapSandboxError(err)
}
return connect.NewResponse(&pb.DeleteVolumeResponse{}), nil
}
// volumeSpecsFromProto converts the wire VolumeSpecs on a create request into
// the sandbox layer's attach specs, parsing the hex UUID fields.
func volumeSpecsFromProto(pbVols []*pb.VolumeSpec) ([]sandbox.VolumeAttachSpec, error) {
if len(pbVols) == 0 {
return nil, nil
}
specs := make([]sandbox.VolumeAttachSpec, 0, len(pbVols))
for _, v := range pbVols {
volumeID, err := parseUUIDString(v.VolumeId)
if err != nil {
return nil, fmt.Errorf("volume id: %w", err)
}
teamID, err := parseUUIDString(v.TeamId)
if err != nil {
return nil, fmt.Errorf("volume team id: %w", err)
}
specs = append(specs, sandbox.VolumeAttachSpec{
VolumeID: volumeID,
TeamID: teamID,
SizeMB: int(v.SizeMb),
MountPath: v.MountPath,
})
}
return specs, nil
}
// mapSandboxError translates sandbox.Manager errors to Connect errors
// carrying an apperr ErrorInfo detail, so the control plane surfaces the
// precise catalog error instead of guessing from the Connect code. Sentinels
@ -204,6 +254,10 @@ func mapSandboxError(err error) error {
return apperr.ToConnect(apperr.TemplateNotFound.WrapMsg(err, "The template's rootfs image is not available on this host."))
case errors.Is(err, sandbox.ErrEnvdNotReady):
return apperr.ToConnect(apperr.SandboxUnresponsive.WrapMsg(err, "The sandbox VM started but its agent did not become ready."))
case errors.Is(err, sandbox.ErrVolumesAttached):
return apperr.ToConnect(apperr.VolumesAttached.Wrap(err))
case errors.Is(err, sandbox.ErrVolumesOnSnapshotTemplate):
return apperr.ToConnect(apperr.InvalidRequest.WrapMsg(err, "Volumes can only be attached to capsules created from a base template, not a snapshot template."))
case errors.Is(err, network.ErrNoFreeSlots):
return apperr.ToConnect(apperr.CapacityUnavailable.WrapMsg(err, "This host has no free network slots. Try again shortly."))
default:

72
internal/units/size.go Normal file
View File

@ -0,0 +1,72 @@
// Package units parses the human-readable size strings used across wrenn's
// configuration and API surfaces (e.g. WRENN_DEFAULT_ROOTFS_SIZE=5Gi, a
// volume's "size": "20Gi"). It deliberately holds no dependencies so both the
// control plane and the host runtime can share one definition of what "5G"
// means.
package units
import (
"fmt"
"strconv"
"strings"
)
// maxSizeMB bounds a parsed size at what fits in an int32 megabyte count (~2
// PiB). Sizes are stored and carried as int32 throughout — DB columns, proto
// fields, request bodies — so a larger value could only reach them by silently
// truncating, turning an absurd request into a plausible-looking small volume.
// Reject it at the parse instead.
const maxSizeMB = 1<<31 - 1
// ParseSizeToMB parses a human-readable size string into megabytes.
// Supported suffixes: G, Gi (gibibytes), M, Mi (mebibytes). A bare number is
// read as megabytes.
// Examples: "5G" → 5120, "2Gi" → 2048, "1000M" → 1000, "512Mi" → 512.
func ParseSizeToMB(s string) (int, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, fmt.Errorf("empty size string")
}
// Find where the numeric part ends.
i := 0
for i < len(s) && (s[i] == '.' || (s[i] >= '0' && s[i] <= '9')) {
i++
}
if i == 0 {
return 0, fmt.Errorf("invalid size %q: no numeric value", s)
}
numStr := s[:i]
suffix := strings.TrimSpace(s[i:])
num, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0, fmt.Errorf("invalid size %q: %w", s, err)
}
var mb float64
switch suffix {
case "G", "Gi":
mb = num * 1024
case "M", "Mi", "":
mb = num
default:
return 0, fmt.Errorf("invalid size %q: unknown suffix %q (use G, Gi, M, or Mi)", s, suffix)
}
// Compare as float: converting first would already have truncated.
if mb > maxSizeMB {
return 0, fmt.Errorf("invalid size %q: too large (max %d MB)", s, maxSizeMB)
}
return int(mb), nil
}
// FormatMB renders a megabyte count back into the compact human form used in
// error messages, preferring whole gibibytes when the value divides evenly.
func FormatMB(mb int) string {
if mb > 0 && mb%1024 == 0 {
return strconv.Itoa(mb/1024) + "Gi"
}
return strconv.Itoa(mb) + "Mi"
}

View File

@ -0,0 +1,81 @@
package units
import "testing"
func TestParseSizeToMB(t *testing.T) {
tests := []struct {
name string
input string
want int
wantErr bool
}{
{"gibibytes", "5G", 5120, false},
{"gibibytes-explicit", "2Gi", 2048, false},
{"megabytes", "1000M", 1000, false},
{"mebibytes", "512Mi", 512, false},
{"bare-number-is-mb", "2048", 2048, false},
{"fractional", "1.5G", 1536, false},
{"trims-space", " 20Gi ", 20480, false},
{"default-volume-cap", "20Gi", 20 * 1024, false},
{"empty", "", 0, true},
{"whitespace-only", " ", 0, true},
{"no-number", "Gi", 0, true},
{"negative", "-5G", 0, true},
{"unknown-suffix", "5T", 0, true},
{"unknown-suffix-bytes", "500B", 0, true},
// Sizes are int32 megabytes everywhere downstream. Without an explicit
// bound these truncate: 4194305Gi wraps to exactly 1024 MB, which would
// sail past both the minimum and maximum checks as a 1 GiB volume.
{"overflows-int32", "4194305Gi", 0, true},
{"overflows-int32-exactly", "4194304Gi", 0, true},
{"huge", "999999999Gi", 0, true},
{"largest-accepted", "2147483647", 1<<31 - 1, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseSizeToMB(tt.input)
if (err != nil) != tt.wantErr {
t.Fatalf("ParseSizeToMB(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
}
if got != tt.want {
t.Errorf("ParseSizeToMB(%q) = %d, want %d", tt.input, got, tt.want)
}
})
}
}
func TestFormatMB(t *testing.T) {
tests := []struct {
input int
want string
}{
{20 * 1024, "20Gi"},
{1024, "1Gi"},
{5120, "5Gi"},
{100, "100Mi"},
{1500, "1500Mi"},
{0, "0Mi"},
}
for _, tt := range tests {
if got := FormatMB(tt.input); got != tt.want {
t.Errorf("FormatMB(%d) = %q, want %q", tt.input, got, tt.want)
}
}
}
// FormatMB's output must feed back into ParseSizeToMB unchanged, so an error
// message quoting a limit can be pasted straight into a request or env var.
func TestFormatMBRoundTrips(t *testing.T) {
for _, mb := range []int{100, 1024, 5120, 20 * 1024, 1500} {
got, err := ParseSizeToMB(FormatMB(mb))
if err != nil {
t.Fatalf("ParseSizeToMB(FormatMB(%d)) unexpected error: %v", mb, err)
}
if got != mb {
t.Errorf("round trip of %d MB produced %d", mb, got)
}
}
}

View File

@ -214,6 +214,31 @@ var (
Status: http.StatusNotFound,
Message: "User not found.",
})
VolumeNotFound = register(Def{
Code: "volume_not_found",
Status: http.StatusNotFound,
Message: "Volume not found.",
})
VolumeInUse = register(Def{
Code: "volume_in_use",
Status: http.StatusConflict,
Message: "Volume is attached to a capsule. Destroy the capsule before deleting the volume.",
})
VolumeHostMismatch = register(Def{
Code: "volume_host_mismatch",
Status: http.StatusConflict,
Message: "This volume is pinned to a different host than the requested capsule can run on.",
})
VolumesAttached = register(Def{
Code: "volumes_attached",
Status: http.StatusConflict,
Message: "Detach all volumes before creating a template from this capsule.",
})
VolumeNameTaken = register(Def{
Code: "volume_name_taken",
Status: http.StatusConflict,
Message: "A volume with this name already exists.",
})
)
// Lookup returns the catalog Def for a code, or false if unregistered.

View File

@ -452,6 +452,38 @@ func (l *AuditLogger) LogSnapshotDeleteSystem(ctx context.Context, teamID pgtype
})
}
// --- Volume events (scope: team) ---
// LogVolumeCreate records the creation of an external storage volume.
func (l *AuditLogger) LogVolumeCreate(ctx context.Context, ac auth.AuthContext, volumeID pgtype.UUID, name string, sizeMB int, err error) {
meta := map[string]any{"name": name, "size_mb": sizeMB}
l.Log(ctx, newEntry(ac, ac.TeamID, "team", "volume", id.FormatVolumeID(volumeID), "create", auditStatusFor(err, "success"), mergeMeta(meta, err)))
l.publish(ctx, events.Event{
Event: events.VolumeCreate,
Outcome: outcomeFromErr(err),
Timestamp: events.Now(),
TeamID: id.FormatTeamID(ac.TeamID),
Actor: actorToEvent(ac),
Resource: events.Resource{ID: id.FormatVolumeID(volumeID), Type: "volume"},
Metadata: map[string]string{"name": name},
Error: errString(err),
})
}
// LogVolumeDelete records the deletion of an external storage volume.
func (l *AuditLogger) LogVolumeDelete(ctx context.Context, ac auth.AuthContext, volumeID pgtype.UUID, err error) {
l.Log(ctx, newEntry(ac, ac.TeamID, "team", "volume", id.FormatVolumeID(volumeID), "delete", auditStatusFor(err, "warning"), mergeMeta(nil, err)))
l.publish(ctx, events.Event{
Event: events.VolumeDelete,
Outcome: outcomeFromErr(err),
Timestamp: events.Now(),
TeamID: id.FormatTeamID(ac.TeamID),
Actor: actorToEvent(ac),
Resource: events.Resource{ID: id.FormatVolumeID(volumeID), Type: "volume"},
Error: errString(err),
})
}
// --- Team events (scope: team) ---
func (l *AuditLogger) LogTeamRename(ctx context.Context, ac auth.AuthContext, teamID pgtype.UUID, oldName, newName string) {

View File

@ -6,6 +6,8 @@ import (
"strconv"
"github.com/joho/godotenv"
"git.omukk.dev/wrenn/wrenn/internal/units"
)
// Config holds the control plane configuration.
@ -36,6 +38,14 @@ type Config struct {
// deployments that legitimately deliver to internal endpoints.
ChannelsAllowPrivateTargets bool // WRENN_CHANNELS_ALLOW_PRIVATE
// MaxVolumeSizeMB caps how large a single external storage volume may be.
// WRENN_MAX_VOLUME_SIZE accepts the same human-readable form as
// WRENN_DEFAULT_ROOTFS_SIZE (e.g. "20Gi", "500G", "2048M"). Raise it on
// self-hosted deployments with the disk to back it — a volume is a sparse
// file, so the cap bounds worst-case growth rather than upfront allocation.
// 0 means unset; service.DefaultMaxVolumeSizeMB (20Gi) then applies.
MaxVolumeSizeMB int // WRENN_MAX_VOLUME_SIZE
// SMTP — transactional email. All fields optional; omitting SMTPHost disables email.
SMTPHost string // SMTP_HOST
SMTPPort int // SMTP_PORT (default 587)
@ -69,6 +79,10 @@ func Load() Config {
ChannelsAllowPrivateTargets: envOrDefaultBool("WRENN_CHANNELS_ALLOW_PRIVATE", false),
// 0 means "unset" — service.VolumeService owns the default so the
// limit is not declared in two places that can drift apart.
MaxVolumeSizeMB: envOrDefaultSizeMB("WRENN_MAX_VOLUME_SIZE", 0),
SMTPHost: os.Getenv("SMTP_HOST"),
SMTPPort: envOrDefaultInt("SMTP_PORT", 587),
SMTPUsername: os.Getenv("SMTP_USERNAME"),
@ -105,6 +119,21 @@ func envOrDefaultInt(key string, def int) int {
return n
}
// envOrDefaultSizeMB reads a human-readable size (e.g. "20Gi", "500G", "2048M")
// and returns it in megabytes, falling back to def when unset or unparseable —
// matching the fail-soft behaviour of the other envOrDefault helpers.
func envOrDefaultSizeMB(key string, def int) int {
v := os.Getenv(key)
if v == "" {
return def
}
mb, err := units.ParseSizeToMB(v)
if err != nil || mb <= 0 {
return def
}
return mb
}
func envOrDefaultBool(key string, def bool) bool {
v := os.Getenv(key)
if v == "" {

View File

@ -296,6 +296,12 @@ func Run(opts ...Option) {
rollup := api.NewDailyUsageRollup(queries, time.Hour)
rollup.Start(ctx)
// Reap volumes stuck in a transient attaching/deleting state after a crash.
// The 30m grace safely exceeds the 10m capsule create timeout, so an
// in-flight attach is never freed out from under a booting capsule.
volumeReaper := api.NewVolumeReaper(queries, 10*time.Minute, 30*time.Minute)
volumeReaper.Start(ctx)
// Start extension background workers.
for _, ext := range o.extensions {
for _, worker := range ext.BackgroundWorkers(sctx) {

View File

@ -239,3 +239,17 @@ type UsersTeam struct {
Role string `json:"role"`
CreatedAt pgtype.Timestamptz `json:"created_at"`
}
type Volume struct {
ID pgtype.UUID `json:"id"`
TeamID pgtype.UUID `json:"team_id"`
HostID pgtype.UUID `json:"host_id"`
Name string `json:"name"`
SizeMb int32 `json:"size_mb"`
Status string `json:"status"`
SandboxID pgtype.UUID `json:"sandbox_id"`
MountPath string `json:"mount_path"`
CreatedAt pgtype.Timestamptz `json:"created_at"`
LastAttachedAt pgtype.Timestamptz `json:"last_attached_at"`
LastUpdated pgtype.Timestamptz `json:"last_updated"`
}

511
pkg/db/volumes.sql.go Normal file
View File

@ -0,0 +1,511 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.30.0
// source: volumes.sql
package db
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const abortVolumeDelete = `-- name: AbortVolumeDelete :exec
UPDATE volumes
SET status = 'detached',
last_updated = NOW()
WHERE id = $1 AND status = 'deleting'
`
// Revert a delete claim back to detached when the host-side file removal fails,
// so the volume stays usable rather than stuck in 'deleting'.
func (q *Queries) AbortVolumeDelete(ctx context.Context, id pgtype.UUID) error {
_, err := q.db.Exec(ctx, abortVolumeDelete, id)
return err
}
const beginVolumeDelete = `-- name: BeginVolumeDelete :one
UPDATE volumes
SET status = 'deleting',
last_updated = NOW()
WHERE id = $1 AND team_id = $2 AND status IN ('detached', 'deleting')
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
`
type BeginVolumeDeleteParams struct {
ID pgtype.UUID `json:"id"`
TeamID pgtype.UUID `json:"team_id"`
}
// Atomically claim a volume for deletion. The CAS blocks a concurrent attach
// (which needs status='detached') so a volume can never be attached and deleted
// at the same time. 'deleting' is also accepted so a delete that crashed
// mid-flight can be retried rather than leaving the volume permanently stuck.
// Returns the row (for host_id) when claimed; no row means it is not deletable
// (attached/attaching, wrong team, or missing).
func (q *Queries) BeginVolumeDelete(ctx context.Context, arg BeginVolumeDeleteParams) (Volume, error) {
row := q.db.QueryRow(ctx, beginVolumeDelete, arg.ID, arg.TeamID)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}
const countVolumesByHost = `-- name: CountVolumesByHost :one
SELECT COUNT(*) FROM volumes WHERE host_id = $1
`
// Guard used before a host can be removed: a host holding pinned volumes must
// not be deleted out from under them.
func (q *Queries) CountVolumesByHost(ctx context.Context, hostID pgtype.UUID) (int64, error) {
row := q.db.QueryRow(ctx, countVolumesByHost, hostID)
var count int64
err := row.Scan(&count)
return count, err
}
const deleteVolumeRow = `-- name: DeleteVolumeRow :exec
DELETE FROM volumes WHERE id = $1 AND team_id = $2 AND status = 'deleting'
`
type DeleteVolumeRowParams struct {
ID pgtype.UUID `json:"id"`
TeamID pgtype.UUID `json:"team_id"`
}
// Drop the row only while the delete claim still holds. Without the status
// guard a delete that lost its claim mid-flight (reaped, or aborted and then
// re-attached by a racing capsule create) would erase the record of a volume
// that is once again in use.
func (q *Queries) DeleteVolumeRow(ctx context.Context, arg DeleteVolumeRowParams) error {
_, err := q.db.Exec(ctx, deleteVolumeRow, arg.ID, arg.TeamID)
return err
}
const deleteVolumesByTeam = `-- name: DeleteVolumesByTeam :exec
DELETE FROM volumes WHERE team_id = $1
`
// Drop every volume record belonging to a team being deleted. This is the one
// place volumes are removed without an explicit per-volume delete: the team that
// owns them is going away, so there is no longer anyone who could reference,
// re-attach, or delete them. Unlike DeleteVolumeRow there is no status guard —
// the team's capsules have already been destroyed, so no volume can still be
// legitimately in use, and a row left behind would be unreachable forever (and
// would keep blocking deletion of the host it is pinned to).
func (q *Queries) DeleteVolumesByTeam(ctx context.Context, teamID pgtype.UUID) error {
_, err := q.db.Exec(ctx, deleteVolumesByTeam, teamID)
return err
}
const detachVolumesByHost = `-- name: DetachVolumesByHost :exec
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
host_id = NULL,
mount_path = '',
last_updated = NOW()
WHERE host_id = $1
`
// Free and un-pin every volume on a host being force-deleted. The host (and its
// backing files) are going away, so the volumes are reset to a fresh detached,
// un-pinned state — the rows survive (never auto-deleted) and can be re-attached
// on a new host. Also required before DeleteHost so the host_id FK does not
// block deletion.
func (q *Queries) DetachVolumesByHost(ctx context.Context, hostID pgtype.UUID) error {
_, err := q.db.Exec(ctx, detachVolumesByHost, hostID)
return err
}
const detachVolumesBySandbox = `-- name: DetachVolumesBySandbox :exec
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
mount_path = '',
last_updated = NOW()
WHERE sandbox_id = $1 AND status IN ('attached', 'attaching')
`
// Terminal sweep run when a capsule reaches a terminal state (destroyed,
// stopped, errored, or reaped): free every volume it held back to detached
// (data and host pin preserved) so it can be reused or deleted. Keyed on
// sandbox_id, which is stamped on at reservation time. Never deletes the
// volume — removal is always explicit.
//
// 'attaching' is swept alongside 'attached' so a reservation whose promotion
// never landed (lost RPC response, DB blip) is freed here — on confirmed host
// state — rather than on a blind timer.
func (q *Queries) DetachVolumesBySandbox(ctx context.Context, sandboxID pgtype.UUID) error {
_, err := q.db.Exec(ctx, detachVolumesBySandbox, sandboxID)
return err
}
const detachVolumesBySandboxIDs = `-- name: DetachVolumesBySandboxIDs :exec
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
mount_path = '',
last_updated = NOW()
WHERE sandbox_id = ANY($1::uuid[]) AND status IN ('attached', 'attaching')
`
// Bulk variant of DetachVolumesBySandbox for the host monitor, which stops
// many sandboxes at once (e.g. after a host goes offline).
func (q *Queries) DetachVolumesBySandboxIDs(ctx context.Context, dollar_1 []pgtype.UUID) error {
_, err := q.db.Exec(ctx, detachVolumesBySandboxIDs, dollar_1)
return err
}
const getVolume = `-- name: GetVolume :one
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE id = $1
`
func (q *Queries) GetVolume(ctx context.Context, id pgtype.UUID) (Volume, error) {
row := q.db.QueryRow(ctx, getVolume, id)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}
const getVolumeByTeam = `-- name: GetVolumeByTeam :one
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE id = $1 AND team_id = $2
`
type GetVolumeByTeamParams struct {
ID pgtype.UUID `json:"id"`
TeamID pgtype.UUID `json:"team_id"`
}
func (q *Queries) GetVolumeByTeam(ctx context.Context, arg GetVolumeByTeamParams) (Volume, error) {
row := q.db.QueryRow(ctx, getVolumeByTeam, arg.ID, arg.TeamID)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}
const getVolumeByTeamAndName = `-- name: GetVolumeByTeamAndName :one
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE team_id = $1 AND name = $2
`
type GetVolumeByTeamAndNameParams struct {
TeamID pgtype.UUID `json:"team_id"`
Name string `json:"name"`
}
// Resolve a volume by its user-facing name. Names are unique per team, so this
// is the name-based counterpart to GetVolumeByTeam and is what lets the API
// accept "vl-cache" wherever it accepts "vol-<id>".
func (q *Queries) GetVolumeByTeamAndName(ctx context.Context, arg GetVolumeByTeamAndNameParams) (Volume, error) {
row := q.db.QueryRow(ctx, getVolumeByTeamAndName, arg.TeamID, arg.Name)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}
const insertVolume = `-- name: InsertVolume :one
INSERT INTO volumes (id, team_id, name, size_mb, status)
VALUES ($1, $2, $3, $4, 'detached')
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
`
type InsertVolumeParams struct {
ID pgtype.UUID `json:"id"`
TeamID pgtype.UUID `json:"team_id"`
Name string `json:"name"`
SizeMb int32 `json:"size_mb"`
}
func (q *Queries) InsertVolume(ctx context.Context, arg InsertVolumeParams) (Volume, error) {
row := q.db.QueryRow(ctx, insertVolume,
arg.ID,
arg.TeamID,
arg.Name,
arg.SizeMb,
)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}
const linkVolumeReservation = `-- name: LinkVolumeReservation :exec
UPDATE volumes
SET sandbox_id = $2,
last_updated = NOW()
WHERE id = $1 AND status = 'attaching'
`
type LinkVolumeReservationParams struct {
ID pgtype.UUID `json:"id"`
SandboxID pgtype.UUID `json:"sandbox_id"`
}
// Stamp the owning sandbox onto a reservation as soon as the sandbox row
// exists (ReserveVolumeForAttach runs before the insert, so it cannot set the
// FK itself). This makes every in-flight reservation attributable to a capsule,
// which is what lets the host monitor decide — against live host state —
// whether a stuck 'attaching' row is safe to free.
func (q *Queries) LinkVolumeReservation(ctx context.Context, arg LinkVolumeReservationParams) error {
_, err := q.db.Exec(ctx, linkVolumeReservation, arg.ID, arg.SandboxID)
return err
}
const listVolumesBySandbox = `-- name: ListVolumesBySandbox :many
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE sandbox_id = $1 ORDER BY created_at DESC
`
func (q *Queries) ListVolumesBySandbox(ctx context.Context, sandboxID pgtype.UUID) ([]Volume, error) {
rows, err := q.db.Query(ctx, listVolumesBySandbox, sandboxID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Volume
for rows.Next() {
var i Volume
if err := rows.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listVolumesByTeam = `-- name: ListVolumesByTeam :many
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE team_id = $1 ORDER BY created_at DESC
`
func (q *Queries) ListVolumesByTeam(ctx context.Context, teamID pgtype.UUID) ([]Volume, error) {
rows, err := q.db.Query(ctx, listVolumesByTeam, teamID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Volume
for rows.Next() {
var i Volume
if err := rows.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const markVolumeAttached = `-- name: MarkVolumeAttached :one
UPDATE volumes
SET status = 'attached',
host_id = COALESCE(host_id, $2),
sandbox_id = $3,
mount_path = $4,
last_attached_at = NOW(),
last_updated = NOW()
WHERE id = $1 AND status = 'attaching'
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
`
type MarkVolumeAttachedParams struct {
ID pgtype.UUID `json:"id"`
HostID pgtype.UUID `json:"host_id"`
SandboxID pgtype.UUID `json:"sandbox_id"`
MountPath string `json:"mount_path"`
}
// Promote a reserved volume to attached once the capsule has booted with it.
// Records the owning sandbox and pins the volume to the sandbox's host on
// first-ever attach (COALESCE keeps an existing pin for a re-attached volume).
func (q *Queries) MarkVolumeAttached(ctx context.Context, arg MarkVolumeAttachedParams) (Volume, error) {
row := q.db.QueryRow(ctx, markVolumeAttached,
arg.ID,
arg.HostID,
arg.SandboxID,
arg.MountPath,
)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}
const releaseStaleVolumeReservations = `-- name: ReleaseStaleVolumeReservations :execrows
UPDATE volumes
SET status = 'detached',
mount_path = '',
last_updated = NOW()
WHERE status IN ('attaching', 'deleting')
AND sandbox_id IS NULL
AND last_updated < $1
`
// Free volumes stuck in a transient state whose owning operation died before it
// could ever reach a host — the control plane crashed between reserving a
// volume and inserting the sandbox row, or mid-delete. Run periodically by the
// volume reaper.
//
// Deliberately limited to unattributed rows (sandbox_id IS NULL). A reservation
// that already carries a sandbox_id may correspond to a capsule that really did
// boot with the volume mounted — and a timer cannot tell the difference. Those
// are freed only by the host-monitor sweep, which first confirms against the
// host that the capsule is gone. Freeing a live volume here would let a second
// capsule attach the same backing file and corrupt it.
//
// The cutoff ($1 = now - grace) MUST exceed the capsule create timeout so a
// legitimately in-flight attach is never freed out from under a booting capsule.
func (q *Queries) ReleaseStaleVolumeReservations(ctx context.Context, lastUpdated pgtype.Timestamptz) (int64, error) {
result, err := q.db.Exec(ctx, releaseStaleVolumeReservations, lastUpdated)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const releaseVolumeReservation = `-- name: ReleaseVolumeReservation :exec
UPDATE volumes
SET status = 'detached',
sandbox_id = NULL,
last_updated = NOW()
WHERE id = $1 AND status = 'attaching'
`
// Roll a reserved volume back to detached when the capsule create fails.
// host_id is left untouched (a failed create never pins a fresh volume, since
// MarkVolumeAttached is what sets the pin). Only call this once the host has
// confirmed the capsule is not running — a volume freed while a VM still has
// its backing file open can be re-attached elsewhere and corrupted.
func (q *Queries) ReleaseVolumeReservation(ctx context.Context, id pgtype.UUID) error {
_, err := q.db.Exec(ctx, releaseVolumeReservation, id)
return err
}
const reserveVolumeForAttach = `-- name: ReserveVolumeForAttach :one
UPDATE volumes
SET status = 'attaching',
last_updated = NOW()
WHERE id = $1 AND status = 'detached'
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
`
// Atomically claim a detached volume for a capsule about to boot. The CAS on
// status = 'detached' enforces single-attach: a second capsule create racing
// for the same volume finds no row and fails. sandbox_id is deliberately NOT
// set here — the sandbox row does not exist yet, so setting the FK would fail.
// It is set in MarkVolumeAttached once the capsule has booted.
func (q *Queries) ReserveVolumeForAttach(ctx context.Context, id pgtype.UUID) (Volume, error) {
row := q.db.QueryRow(ctx, reserveVolumeForAttach, id)
var i Volume
err := row.Scan(
&i.ID,
&i.TeamID,
&i.HostID,
&i.Name,
&i.SizeMb,
&i.Status,
&i.SandboxID,
&i.MountPath,
&i.CreatedAt,
&i.LastAttachedAt,
&i.LastUpdated,
)
return i, err
}

View File

@ -360,6 +360,62 @@ func (c *Client) PrepareSnapshot(ctx context.Context) error {
return nil
}
// MountVolume asks envd to mount a data volume inside the guest. The volume's
// block device is resolved by its virtio-blk serial (set in the VM config), so
// this is robust to device-enumeration order. envd formats the device with
// ext4 only if it has no filesystem yet (existing data is never reformatted),
// then mounts it at mountPath.
func (c *Client) MountVolume(ctx context.Context, serial, mountPath string) error {
payload, err := json.Marshal(map[string]string{"serial": serial, "mount_path": mountPath})
if err != nil {
return fmt.Errorf("marshal mount body: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/volumes/mount", bytes.NewReader(payload))
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("mount volume: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, MaxEnvdControlBytes))
return fmt.Errorf("mount volume: status %d: %s", resp.StatusCode, string(respBody))
}
return nil
}
// UnmountVolume asks envd to flush (sync) and unmount a data volume. Used
// best-effort before a graceful capsule destroy so buffered writes reach the
// backing file. Safe to call even if the path is not mounted.
func (c *Client) UnmountVolume(ctx context.Context, mountPath string) error {
payload, err := json.Marshal(map[string]string{"mount_path": mountPath})
if err != nil {
return fmt.Errorf("marshal unmount body: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/volumes/unmount", bytes.NewReader(payload))
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("unmount volume: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, MaxEnvdControlBytes))
return fmt.Errorf("unmount volume: status %d: %s", resp.StatusCode, string(respBody))
}
return nil
}
// MemoryPreloadStatus mirrors envd's /memory/preload response.
//
// State values: "idle", "running", "done", "failed", "cancelled".

View File

@ -90,6 +90,12 @@ const (
SnapshotCreate = "template.snapshot.create"
SnapshotDelete = "template.snapshot.delete"
// Durable, subscribable. Volume lifecycle. Attach/detach are not separate
// events — a volume attaches only as part of capsule.create and frees on
// capsule.destroy.
VolumeCreate = "volume.create"
VolumeDelete = "volume.delete"
// Durable, no outcome (binary by name).
HostUp = "host.up"
HostDown = "host.down"
@ -108,6 +114,8 @@ var SubscribableEventTypes = []string{
CapsuleDestroy,
SnapshotCreate,
SnapshotDelete,
VolumeCreate,
VolumeDelete,
HostUp,
HostDown,
}

View File

@ -39,6 +39,7 @@ func NewAdminPermissionID() pgtype.UUID { return newUUID() }
func NewChannelID() pgtype.UUID { return newUUID() }
func NewTemplateID() pgtype.UUID { return newUUID() }
func NewVolumeID() pgtype.UUID { return newUUID() }
// NewSnapshotName generates a snapshot name: "template-" + 8 hex chars.
func NewSnapshotName() string {
@ -78,6 +79,7 @@ const (
PrefixBuild = "bld-"
PrefixAdminPermission = "perm-"
PrefixChannel = "ch-"
PrefixVolume = "vol-"
)
// UUIDToBase36 encodes 16 UUID bytes as a 25-char base36 string (0-9a-z).
@ -127,6 +129,7 @@ func FormatRefreshTokenID(id pgtype.UUID) string { return formatUUID(PrefixRefre
func FormatAuditLogID(id pgtype.UUID) string { return formatUUID(PrefixAuditLog, id) }
func FormatBuildID(id pgtype.UUID) string { return formatUUID(PrefixBuild, id) }
func FormatChannelID(id pgtype.UUID) string { return formatUUID(PrefixChannel, id) }
func FormatVolumeID(id pgtype.UUID) string { return formatUUID(PrefixVolume, id) }
// --- Parsing (prefixed string from API/RPC input → pgtype.UUID) ---
@ -150,6 +153,23 @@ func ParseHostTokenID(s string) (pgtype.UUID, error) { return parseUUID(PrefixHo
func ParseAuditLogID(s string) (pgtype.UUID, error) { return parseUUID(PrefixAuditLog, s) }
func ParseBuildID(s string) (pgtype.UUID, error) { return parseUUID(PrefixBuild, s) }
func ParseChannelID(s string) (pgtype.UUID, error) { return parseUUID(PrefixChannel, s) }
func ParseVolumeID(s string) (pgtype.UUID, error) { return parseUUID(PrefixVolume, s) }
// DefaultVolumeMountPath returns the guest path a volume is mounted at when no
// explicit path is requested. Single source of truth shared by the control
// plane (which records it) and the host agent (which mounts there).
func DefaultVolumeMountPath(v pgtype.UUID) string {
return "/mnt/" + FormatVolumeID(v)
}
// VolumeSerial returns a 16-hex-char (64-bit) token derived from a volume ID.
// It is set as the virtio-blk disk serial in the VM config so the guest can
// resolve the block device via /sys/block/*/serial regardless of enumeration
// order, without relying on positional /dev/vdX guessing. Kept well under the
// virtio-blk serial cap (VIRTIO_BLK_ID_BYTES = 20).
func VolumeSerial(v pgtype.UUID) string {
return hex.EncodeToString(v.Bytes[:8])
}
// --- Well-known IDs ---

View File

@ -100,6 +100,51 @@ func TestMaxUUID(t *testing.T) {
}
}
func TestVolumeIDRoundTrip(t *testing.T) {
id := NewVolumeID()
formatted := FormatVolumeID(id)
if formatted[:len(PrefixVolume)] != PrefixVolume {
t.Fatalf("expected %s prefix, got %s", PrefixVolume, formatted)
}
parsed, err := ParseVolumeID(formatted)
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed != id {
t.Fatalf("round-trip failed: %v → %s → %v", id, formatted, parsed)
}
}
func TestVolumeSerial(t *testing.T) {
// The virtio-blk serial (VIRTIO_BLK_ID_BYTES) caps at 20 bytes; the guest
// resolves the device by this value, so it must always fit.
for i := 0; i < 1000; i++ {
s := VolumeSerial(NewVolumeID())
if len(s) > 20 {
t.Fatalf("serial %q exceeds 20 bytes (%d)", s, len(s))
}
if s == "" {
t.Fatal("serial must not be empty")
}
}
// Deterministic: the same volume yields the same serial (required so a
// resumed capsule rebuilds the identical disk symlink).
id := NewVolumeID()
first := VolumeSerial(id)
second := VolumeSerial(id)
if first != second {
t.Fatalf("VolumeSerial must be deterministic: %q != %q", first, second)
}
}
func TestDefaultVolumeMountPath(t *testing.T) {
id := NewVolumeID()
want := "/mnt/" + FormatVolumeID(id)
if got := DefaultVolumeMountPath(id); got != want {
t.Fatalf("DefaultVolumeMountPath = %q, want %q", got, want)
}
}
func BenchmarkFormatSandboxID(b *testing.B) {
id := pgtype.UUID{Bytes: uuid.New(), Valid: true}
b.ResetTimer()

View File

@ -169,6 +169,24 @@ func compareSemver(a, b string) int {
return 0
}
// VolumeDir returns the on-disk directory for an external storage volume,
// mirroring the templates tree but under its own volumes/ root so it can never
// be touched by sandbox teardown (which only ever removes SandboxDir):
//
// {wrennDir}/volumes/teams/{base36(teamID)}/{base36(volumeID)}
func VolumeDir(wrennDir string, teamID, volumeID pgtype.UUID) string {
return filepath.Join(wrennDir, "volumes", "teams",
id.UUIDToBase36(teamID.Bytes),
id.UUIDToBase36(volumeID.Bytes))
}
// VolumeDataPath returns the path to a volume's backing image — a plain sparse
// file (not a dm-snapshot CoW), formatted with ext4 inside the guest on first
// use and handed to Cloud Hypervisor as a Raw disk.
func VolumeDataPath(wrennDir string, teamID, volumeID pgtype.UUID) string {
return filepath.Join(VolumeDir(wrennDir, teamID, volumeID), "data.img")
}
// SlotsDir returns the directory holding network slot claim files. Each
// allocated slot index is an empty file named after the index, created with
// O_EXCL so concurrent processes can never claim the same slot.

View File

@ -6,11 +6,10 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/jackc/pgx/v5/pgtype"
"git.omukk.dev/wrenn/wrenn/internal/units"
"git.omukk.dev/wrenn/wrenn/pkg/id"
"git.omukk.dev/wrenn/wrenn/pkg/layout"
)
@ -69,37 +68,13 @@ func EnsureImageSizes(wrennDir string, targetMB int) error {
// ParseSizeToMB parses a human-readable size string into megabytes.
// Supported suffixes: G, Gi (gibibytes), M, Mi (mebibytes).
// Examples: "5G" → 5120, "2Gi" → 2048, "1000M" → 1000, "512Mi" → 512.
//
// The implementation lives in internal/units so the control plane can share it
// without importing the host runtime. That package is unimportable from outside
// this module, so this stays the entry point for external consumers of the
// sandbox runtime (e.g. the wr CLI).
func ParseSizeToMB(s string) (int, error) {
s = strings.TrimSpace(s)
if s == "" {
return 0, fmt.Errorf("empty size string")
}
// Find where the numeric part ends.
i := 0
for i < len(s) && (s[i] == '.' || (s[i] >= '0' && s[i] <= '9')) {
i++
}
if i == 0 {
return 0, fmt.Errorf("invalid size %q: no numeric value", s)
}
numStr := s[:i]
suffix := strings.TrimSpace(s[i:])
num, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0, fmt.Errorf("invalid size %q: %w", s, err)
}
switch suffix {
case "G", "Gi":
return int(num * 1024), nil
case "M", "Mi", "":
return int(num), nil
default:
return 0, fmt.Errorf("invalid size %q: unknown suffix %q (use G, Gi, M, or Mi)", s, suffix)
}
return units.ParseSizeToMB(s)
}
// ShrinkSystemImages shrinks the built-in system base rootfs images back to

View File

@ -44,6 +44,13 @@ var (
// ErrEnvdNotReady is returned when the VM booted (or resumed) but envd did
// not become healthy within the readiness budget.
ErrEnvdNotReady = errors.New("envd not ready")
// ErrVolumesAttached is returned when an operation (e.g. creating a template
// from a capsule) is refused because the sandbox still has volumes attached.
ErrVolumesAttached = errors.New("sandbox has volumes attached")
// ErrVolumesOnSnapshotTemplate is returned when a create requests volumes on
// a sandbox launched from a snapshot template — CH --restore reconstructs
// only the snapshot's device set, so a new boot-time disk cannot be added.
ErrVolumesOnSnapshotTemplate = errors.New("volumes are not supported when creating from a snapshot template")
)
// MinTimeoutSec is the minimum inactivity TTL accepted by Create/Resume.
@ -209,6 +216,12 @@ type sandboxState struct {
dmDevice *devicemapper.SnapshotDevice
baseImagePath string // path to the base template rootfs (for loop registry release)
// volumes holds the external storage volumes attached to this sandbox at
// boot. Read under m.mu, mutated only while holding lifecycleMu (like
// dmDevice). Persisted into runningState + snapshotMeta so re-attach and
// resume can rebuild each disk symlink.
volumes []*attachedVolume
// sandboxDirOverride, when non-empty, pins this sandbox's VMConfig.SandboxDir
// to a path other than the default vm.SandboxTmpDir(sb.ID). Set when the
// sandbox was launched from a snapshot template — CH's saved config.json
@ -326,6 +339,7 @@ func (m *Manager) Create(
vcpus, memoryMB, timeoutSec, diskSizeMB int,
defaultUser string,
defaultEnv map[string]string,
volumes []VolumeAttachSpec,
) (*models.Sandbox, int64, error) {
if m.draining.Load() {
return nil, 0, ErrDraining
@ -385,6 +399,12 @@ func (m *Manager) Create(
// the restore path with a confusing error downstream.
templateDir := layout.TemplateDir(m.cfg.WrennDir, teamID, templateID)
if !layout.IsSystemTemplate(teamID, templateID) && layout.IsSnapshotTemplate(templateDir) {
// A snapshot-template launch restores CH's saved device topology via
// --restore; a boot-time data disk cannot be added on top of it, and we
// no longer hot-plug. Refuse rather than silently dropping the volumes.
if len(volumes) > 0 {
return nil, 0, ErrVolumesOnSnapshotTemplate
}
return m.createFromSnapshotTemplate(ctx, sandboxID, teamID, templateID,
vcpus, memoryMB, timeoutSec, diskSizeMB, defaultUser, defaultEnv)
}
@ -449,6 +469,16 @@ func (m *Manager) Create(
}
res.slot = slot
// Provision data volumes (sparse backing files) before boot so they can be
// baked into vm.create as extra Raw disks. The files persist across the VM
// lifecycle; rollback below tears down the VM but deliberately leaves volume
// data intact (volumes are owned by their own lifecycle, never auto-deleted).
volumeDisks, attachedVols, err := m.prepareVolumeDisks(volumes)
if err != nil {
res.rollback()
return nil, 0, fmt.Errorf("provision volumes: %w", err)
}
// Boot VM — CH gets the dm device path.
vmCfg := vm.VMConfig{
SandboxID: sandboxID,
@ -465,6 +495,7 @@ func (m *Manager) Create(
NetMask: slot.GuestNetMask,
VMMBin: m.cfg.VMMBin,
LogDir: filepath.Join(m.cfg.WrennDir, "logs"),
Volumes: volumeDisks,
}
if _, err := m.vm.Create(ctx, vmCfg); err != nil {
@ -496,6 +527,14 @@ func (m *Manager) Create(
}
initCancel()
// Format (if empty) and mount each attached volume. A mount failure fails
// the whole create — a capsule that was asked for a volume must not come up
// without it.
if err := m.mountVolumes(ctx, client, attachedVols); err != nil {
res.rollback()
return nil, 0, fmt.Errorf("mount volumes: %w", err)
}
now := time.Now()
sb := &sandboxState{
Sandbox: models.Sandbox{
@ -517,6 +556,7 @@ func (m *Manager) Create(
connTracker: &ConnTracker{},
dmDevice: dmDev,
baseImagePath: baseRootfs,
volumes: attachedVols,
}
sb.client.Store(client)
@ -619,6 +659,10 @@ func (m *Manager) cleanup(ctx context.Context, sb *sandboxState) {
}
}
m.stopSampler(sb)
// Flush volumes before killing the VM so buffered guest writes land in the
// backing files. Best-effort and only meaningful for a running sandbox
// (paused ones were already synced at pause time and have no live envd).
m.unmountVolumesBestEffort(sb)
if err := m.vm.Destroy(ctx, sb.ID); err != nil {
slog.Warn("vm destroy error", "id", sb.ID, "error", err)
}

View File

@ -103,6 +103,11 @@ type snapshotMeta struct {
// forward verbatim through the chain.
SandboxDir string `json:"sandbox_dir"`
CreatedAt time.Time `json:"created_at"`
// Volumes records the data volumes attached at pause time so Resume can
// rebuild each disk symlink and CH's restored config.json paths resolve.
// Only ever set for pause snapshots — snapshot templates are volume-free
// (CreateSnapshot refuses to run while volumes are attached).
Volumes []*attachedVolume `json:"volumes,omitempty"`
}
// effectiveSandboxDir returns the tmpfs SandboxDir the running VM uses — the
@ -269,6 +274,7 @@ func (m *Manager) Pause(ctx context.Context, sandboxID string) error {
CowPath: sb.dmDevice.CowPath,
SandboxDir: effectiveSandboxDir(sb),
CreatedAt: time.Now(),
Volumes: sb.volumes,
}
if err := writeSnapshotMeta(stageDir, meta); err != nil {
// Without meta, Resume cannot reconstruct the sandbox. Treat as fatal.
@ -665,6 +671,12 @@ func (m *Manager) resumeFromMeta(ctx context.Context, sb *sandboxState, meta *sn
// keeps its original ID/SandboxDir so the disk path baked into
// config.json (`/tmp/ch-vm-{originalID}/rootfs.ext4`) resolves to the
// re-attached dm device via the tmpfs symlink set up by the launcher.
// The attached-volume records from the snapshot meta drive the restore
// launch: re-create each disk symlink (CH's saved config.json references
// them) so the guest mount — already present in the restored memory image —
// resolves to the same device.
restoredVols := meta.Volumes
vmCfg := m.buildRestoreVMConfig(restoreInputs{
sandboxID: sb.ID,
templateID: id.UUIDString(pgtype.UUID{Bytes: sb.TemplateID, Valid: true}),
@ -674,6 +686,7 @@ func (m *Manager) resumeFromMeta(ctx context.Context, sb *sandboxState, meta *sn
memoryMB: meta.MemoryMB,
slot: slot,
sandboxDir: meta.SandboxDir,
volumes: volumeDisksFor(restoredVols),
})
client, err := m.launchRestoredVM(ctx, vmCfg, slot.HostIP.String())
if err != nil {
@ -697,6 +710,7 @@ func (m *Manager) resumeFromMeta(ctx context.Context, sb *sandboxState, meta *sn
// it empty so a Destroy-before-Resume cannot underflow the registry.
sb.baseImagePath = meta.BaseTemplate
sb.lazyRestore = true
sb.volumes = restoredVols
sb.connTracker.Reset()
sb.HostIP = slot.HostIP
sb.RootfsPath = dmDev.DevicePath
@ -860,6 +874,15 @@ func (m *Manager) CreateSnapshot(ctx context.Context, sandboxID string, teamID,
sb.lifecycleMu.Lock()
defer sb.lifecycleMu.Unlock()
// A template must be volume-free: CH's snapshot config would otherwise
// reference the volume disks, producing a template that cannot launch on a
// fresh sandbox. Refuse while any volume is attached. (For a paused sandbox
// re-attached across an agent restart sb.volumes may be empty; that case is
// caught again from the on-disk meta in snapshotPausedToTemplate.)
if len(sb.volumes) > 0 {
return 0, fmt.Errorf("%w", ErrVolumesAttached)
}
// Refuse silent overwrites: every snapshot must land in a fresh
// templateID. Defends against caller bugs and concurrent CreateSnapshot
// races for the same destination. User-facing snapshot-name uniqueness
@ -1001,6 +1024,11 @@ func (m *Manager) snapshotPausedToTemplate(ctx context.Context, sb *sandboxState
snapDir := layout.PauseSnapshotDir(m.cfg.WrennDir, sb.ID)
meta, err := readSnapshotMeta(snapDir)
if err == nil && len(meta.Volumes) > 0 {
// A sandbox paused with volumes attached carries them in its snapshot;
// promoting it to a template would bake in dangling disk references.
return 0, fmt.Errorf("%w", ErrVolumesAttached)
}
if err != nil {
return 0, fmt.Errorf("load pause snapshot meta: %w", err)
}
@ -1199,6 +1227,11 @@ func (m *Manager) FlattenRootfs(ctx context.Context, sandboxID string, teamID, t
return 0, fmt.Errorf("%w: %s (status: %s)", ErrNotRunning, sandboxID, sb.Status)
}
// A flattened image template must be volume-free (see CreateSnapshot).
if len(sb.volumes) > 0 {
return 0, fmt.Errorf("%w", ErrVolumesAttached)
}
dstDir := layout.TemplateDir(m.cfg.WrennDir, teamID, templateID)
stageDir := filepath.Join(layout.SandboxesDir(m.cfg.WrennDir),
fmt.Sprintf(".stage-%s-%d", sandboxID, time.Now().UnixNano()))

View File

@ -31,7 +31,8 @@ type restoreInputs struct {
vcpus int
memoryMB int
slot *network.Slot
sandboxDir string // override for VMConfig.SandboxDir; "" = default
sandboxDir string // override for VMConfig.SandboxDir; "" = default
volumes []vm.VolumeDisk // data volumes to re-symlink so restored disk paths resolve
}
// buildRestoreVMConfig assembles the VMConfig used to launch a CH process in
@ -57,6 +58,7 @@ func (m *Manager) buildRestoreVMConfig(in restoreInputs) vm.VMConfig {
RestoreFromDir: in.snapDir,
RestoreLazyMemory: true,
SandboxDir: in.sandboxDir,
Volumes: in.volumes,
}
}

View File

@ -202,6 +202,7 @@ func (m *Manager) reattachRunning(st *runningState) error {
baseImagePath: st.BaseImagePath,
sandboxDirOverride: st.SandboxDirOverride,
lazyRestore: st.LazyRestore,
volumes: st.Volumes,
}
sb.client.Store(envdclient.New(slot.HostIP.String()))

View File

@ -67,6 +67,10 @@ type runningState struct {
LazyRestore bool `json:"lazy_restore,omitempty"`
CreatedAt time.Time `json:"created_at"`
Metadata map[string]string `json:"metadata,omitempty"`
// Volumes records the data volumes attached to this sandbox so a process
// re-attaching to the still-live VM (RestoreRunningSandboxes) can rebuild
// its in-memory volume state without contacting the control plane.
Volumes []*attachedVolume `json:"volumes,omitempty"`
}
// writeRunningState persists sb's re-attach state next to its CoW file.
@ -99,6 +103,7 @@ func (m *Manager) writeRunningState(sb *sandboxState) {
LazyRestore: sb.lazyRestore,
CreatedAt: sb.CreatedAt,
Metadata: sb.Metadata,
Volumes: sb.volumes,
}
data, err := json.MarshalIndent(st, "", " ")

164
pkg/sandbox/volumes.go Normal file
View File

@ -0,0 +1,164 @@
// Package sandbox: external storage volumes attached to a sandbox at boot.
//
// A volume is a plain sparse file on the host (never a dm-snapshot CoW — a data
// volume has no base image). It is handed to Cloud Hypervisor as an extra Raw
// disk in the initial vm.create and symlinked into the VMM's private tmpfs like
// the rootfs, so its disk path survives pause/resume via CH's snapshot config.
// The guest resolves the device by its virtio-blk serial and envd formats (if
// empty) and mounts it. Volumes are only ever attached at create time; they are
// freed (not deleted) when the capsule is destroyed.
package sandbox
import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"github.com/jackc/pgx/v5/pgtype"
"git.omukk.dev/wrenn/wrenn/pkg/envdclient"
"git.omukk.dev/wrenn/wrenn/pkg/id"
"git.omukk.dev/wrenn/wrenn/pkg/layout"
"git.omukk.dev/wrenn/wrenn/pkg/vm"
)
// VolumeAttachSpec describes an external storage volume to attach to a sandbox
// at boot. Passed to Create by the host-agent RPC layer; the host provisions
// the backing file on first use.
type VolumeAttachSpec struct {
VolumeID pgtype.UUID
TeamID pgtype.UUID
SizeMB int
MountPath string // "" => default /mnt/<vol-id>
}
// attachedVolume is the host's lean record of a volume attached to a sandbox.
// It doubles as the on-disk form (JSON tags) stored in runningState
// (cross-process re-attach) and snapshotMeta (pause/resume) — self-contained so
// a re-attaching or resuming process needs no re-derivation from IDs.
type attachedVolume struct {
VolumeID string `json:"volume_id"` // formatted "vol-..." (logging + default mount path)
HostPath string `json:"host_path"` // backing sparse file on the host
Serial string `json:"serial"` // virtio-blk serial; guest resolves the device by this
MountPath string `json:"mount_path"` // effective guest mount path
}
// volumeDisksFor returns the vm.VolumeDisk list for a set of attached volumes,
// used to rebuild the VM config on resume so CH's restored disk paths resolve.
func volumeDisksFor(vols []*attachedVolume) []vm.VolumeDisk {
if len(vols) == 0 {
return nil
}
disks := make([]vm.VolumeDisk, 0, len(vols))
for _, v := range vols {
disks = append(disks, vm.VolumeDisk{HostPath: v.HostPath, Serial: v.Serial})
}
return disks
}
// ensureVolumeFile makes sure the backing sparse file exists at path, creating
// it (and its parent dir) and truncating to sizeBytes on first use. Idempotent:
// an existing file is left untouched so previously-written data survives across
// detach and re-attach (size is fixed at first provision).
func ensureVolumeFile(path string, sizeBytes int64) error {
if _, err := os.Stat(path); err == nil {
return nil // already provisioned; preserve existing data
} else if !os.IsNotExist(err) {
return fmt.Errorf("stat volume file: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return fmt.Errorf("create volume dir: %w", err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("create volume file: %w", err)
}
if err := f.Truncate(sizeBytes); err != nil {
f.Close()
os.Remove(path)
return fmt.Errorf("truncate volume file: %w", err)
}
if err := f.Close(); err != nil {
return fmt.Errorf("close volume file: %w", err)
}
return nil
}
// prepareVolumeDisks provisions the backing file for each spec and returns the
// vm.VolumeDisk list to bake into the VM config plus the attachedVolume records
// to track in sandbox state. Called before the VM boots.
func (m *Manager) prepareVolumeDisks(specs []VolumeAttachSpec) ([]vm.VolumeDisk, []*attachedVolume, error) {
if len(specs) == 0 {
return nil, nil, nil
}
disks := make([]vm.VolumeDisk, 0, len(specs))
attached := make([]*attachedVolume, 0, len(specs))
for _, spec := range specs {
volIDStr := id.FormatVolumeID(spec.VolumeID)
hostPath := layout.VolumeDataPath(m.cfg.WrennDir, spec.TeamID, spec.VolumeID)
if err := ensureVolumeFile(hostPath, int64(spec.SizeMB)*1024*1024); err != nil {
return nil, nil, fmt.Errorf("provision volume %s: %w", volIDStr, err)
}
serial := id.VolumeSerial(spec.VolumeID)
mountPath := spec.MountPath
if mountPath == "" {
mountPath = id.DefaultVolumeMountPath(spec.VolumeID)
}
disks = append(disks, vm.VolumeDisk{HostPath: hostPath, Serial: serial})
attached = append(attached, &attachedVolume{
VolumeID: volIDStr,
HostPath: hostPath,
Serial: serial,
MountPath: mountPath,
})
}
return disks, attached, nil
}
// mountVolumes asks envd to format (if empty) and mount every attached volume.
// Called once after a fresh boot. On resume the guest mount survives inside the
// restored memory image, so this is not called again.
func (m *Manager) mountVolumes(ctx context.Context, client *envdclient.Client, vols []*attachedVolume) error {
for _, v := range vols {
mctx, cancel := context.WithTimeout(ctx, m.cfg.EnvdTimeout)
err := client.MountVolume(mctx, v.Serial, v.MountPath)
cancel()
if err != nil {
return fmt.Errorf("mount volume %s: %w", v.VolumeID, err)
}
}
return nil
}
// unmountVolumesBestEffort asks envd to sync+unmount each volume before a
// graceful destroy so buffered guest writes reach the backing file. Best-effort:
// on a crash envd is already gone and every call fails harmlessly.
func (m *Manager) unmountVolumesBestEffort(sb *sandboxState) {
if len(sb.volumes) == 0 {
return
}
client := sb.client.Load()
if client == nil {
return
}
for _, v := range sb.volumes {
ctx, cancel := context.WithTimeout(context.Background(), m.cfg.EnvdTimeout)
if err := client.UnmountVolume(ctx, v.MountPath); err != nil {
slog.Warn("volume unmount on destroy failed", "id", sb.ID, "volume", v.VolumeID, "error", err)
}
cancel()
}
}
// DeleteVolumeFile removes an external storage volume's backing file and its
// (now-empty) directory tree from this host. Called by the control plane when a
// detached volume is deleted. Idempotent: a missing file is not an error.
func (m *Manager) DeleteVolumeFile(teamID, volumeID pgtype.UUID) error {
dir := layout.VolumeDir(m.cfg.WrennDir, teamID, volumeID)
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("remove volume dir %s: %w", id.FormatVolumeID(volumeID), err)
}
return nil
}

View File

@ -0,0 +1,42 @@
package sandbox
import (
"os"
"path/filepath"
"testing"
)
func TestEnsureVolumeFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "team", "vol", "data.img")
const size = 5 * 1024 * 1024 // 5 MiB
// First call creates the parent dirs and a sparse file of the right size.
if err := ensureVolumeFile(path, size); err != nil {
t.Fatalf("first ensureVolumeFile: %v", err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat after create: %v", err)
}
if info.Size() != size {
t.Fatalf("size = %d, want %d", info.Size(), size)
}
// Write data, then call again: an existing file must be left untouched so
// previously-written data survives detach/re-attach.
marker := []byte("volume-data-must-survive")
if err := os.WriteFile(path, marker, 0o644); err != nil {
t.Fatalf("write marker: %v", err)
}
if err := ensureVolumeFile(path, size); err != nil {
t.Fatalf("second ensureVolumeFile: %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read back: %v", err)
}
if string(got) != string(marker) {
t.Fatalf("existing data was clobbered: got %q, want %q", got, marker)
}
}

View File

@ -14,6 +14,7 @@ import (
"github.com/jackc/pgx/v5/pgtype"
"github.com/redis/go-redis/v9"
"git.omukk.dev/wrenn/wrenn/pkg/apperr"
"git.omukk.dev/wrenn/wrenn/pkg/auth"
"git.omukk.dev/wrenn/wrenn/pkg/db"
"git.omukk.dev/wrenn/wrenn/pkg/id"
@ -510,6 +511,19 @@ func (s *HostService) Delete(ctx context.Context, hostID, userID, teamID pgtype.
return &HostHasSandboxesError{SandboxIDs: ids}
}
// Volumes are pinned to a host; deleting it orphans their data. Refuse
// unless forced (data on a host being intentionally destroyed is expected
// to go with it).
if !force {
volCount, err := s.DB.CountVolumesByHost(ctx, hostID)
if err != nil {
return fmt.Errorf("count volumes: %w", err)
}
if volCount > 0 {
return apperr.Conflict.Msgf("This host has %d volume(s) pinned to it. Deleting it will orphan their data. Pass ?force=true to proceed.", volCount)
}
}
hostIDStr := id.FormatHostID(hostID)
// Gracefully destroy running sandboxes and terminate the agent (best-effort).
@ -552,6 +566,15 @@ func (s *HostService) Delete(ctx context.Context, hostID, userID, teamID pgtype.
slog.Warn("delete host: failed to revoke refresh tokens", "host_id", hostIDStr, "error", err)
}
// Free and un-pin any volumes on this host. The host (and its backing files)
// are going away, so the volumes reset to a fresh detached, un-pinned state —
// the rows survive (volumes are never auto-deleted) and can be re-attached on
// a new host. This must also run before DeleteHost so the volumes.host_id FK
// does not block the delete.
if err := s.DB.DetachVolumesByHost(ctx, hostID); err != nil {
return fmt.Errorf("release volumes on host: %w", err)
}
// Evict the client from the pool so no further RPCs are sent.
if s.Pool != nil {
s.Pool.Evict(id.FormatHostID(hostID))

View File

@ -37,9 +37,12 @@ type SandboxStateEvent struct {
// SandboxService provides sandbox lifecycle operations shared between the
// REST API and the dashboard.
type SandboxService struct {
DB *db.Queries
Pool *lifecycle.HostClientPool
Scheduler scheduler.HostScheduler
DB *db.Queries
Pool *lifecycle.HostClientPool
Scheduler scheduler.HostScheduler
// Volumes resolves the volume references passed at create time. Required
// only when callers attach volumes.
Volumes *VolumeService
PublishEvent SandboxEventPublisher
}
@ -53,8 +56,18 @@ type SandboxCreateParams struct {
// Metadata holds user-supplied key/value labels attached at create-time.
// Reserved system keys (kernel_version, etc.) are rejected by validation.
Metadata map[string]string
// VolumeRefs are external storage volumes to attach at boot, each given as
// a volume ID ("vol-...") or a name ("vl-cache"). All must be detached and
// owned by TeamID; if any is pinned to a host, the capsule is scheduled onto
// that host. Volumes are only ever attached at create time.
VolumeRefs []string
}
// MaxVolumesPerSandbox caps how many volumes one capsule may attach. Each
// becomes a virtio-blk device on the VM, so an unbounded list would fail late
// and expensively — at boot, after every volume had already been reserved.
const MaxVolumesPerSandbox = 4
// MinTimeoutSec mirrors pkg/sandbox.MinTimeoutSec. Sub-minute TTLs race
// the post-create startup window (DB insert → /init → memory loader); the
// agent silently clamps anyway, but the CP must clamp too so the DB record
@ -106,6 +119,188 @@ func (s *SandboxService) publishEvent(ctx context.Context, event SandboxStateEve
}
}
// reservedVolume is a volume claimed for an in-flight capsule create.
type reservedVolume struct {
ID pgtype.UUID
TeamID pgtype.UUID
SizeMB int32
MountPath string
}
// reserveVolumesForCreate validates the requested volumes, computes the host
// they pin the capsule to (if any), and atomically claims each one. On any
// failure it releases whatever it already claimed so the caller can abort
// cleanly. The claim records only the 'attaching' status — the owning
// sandbox_id is set later in markVolumesAttached, because the sandbox row does
// not exist yet (its FK would reject an early write). Returns the reserved
// volumes and, when at least one volume is already pinned, that host.
func (s *SandboxService) reserveVolumesForCreate(ctx context.Context, teamID pgtype.UUID, volumeRefs []string) ([]reservedVolume, pgtype.UUID, error) {
if len(volumeRefs) == 0 {
return nil, pgtype.UUID{}, nil
}
if len(volumeRefs) > MaxVolumesPerSandbox {
return nil, pgtype.UUID{}, apperr.InvalidRequest.Msgf(
"A capsule may attach at most %d volumes.", MaxVolumesPerSandbox)
}
if s.Volumes == nil {
return nil, pgtype.UUID{}, apperr.InvalidRequest.Msg("Volumes are not available on this deployment.")
}
var pinnedHost pgtype.UUID
seen := make(map[[16]byte]bool, len(volumeRefs))
reserved := make([]reservedVolume, 0, len(volumeRefs))
release := func() { s.releaseVolumeReservations(ctx, reserved) }
for _, ref := range volumeRefs {
vol, err := s.Volumes.Resolve(ctx, teamID, ref)
if err != nil {
release()
return nil, pgtype.UUID{}, err
}
if seen[vol.ID.Bytes] {
continue // a volume attaches at most once per capsule
}
seen[vol.ID.Bytes] = true
// All pinned volumes must share one host — a capsule runs on a single
// host and cannot straddle hosts.
if vol.HostID.Valid {
if pinnedHost.Valid && pinnedHost.Bytes != vol.HostID.Bytes {
release()
return nil, pgtype.UUID{}, apperr.VolumeHostMismatch.New()
}
pinnedHost = vol.HostID
}
if _, err := s.DB.ReserveVolumeForAttach(ctx, vol.ID); err != nil {
// CAS missed: the volume is not detached (already attached, or being
// claimed by a racing create).
release()
return nil, pgtype.UUID{}, apperr.VolumeInUse.Msgf("Volume %q is not available to attach.", vol.Name)
}
reserved = append(reserved, reservedVolume{
ID: vol.ID,
TeamID: teamID,
SizeMB: vol.SizeMb,
MountPath: id.DefaultVolumeMountPath(vol.ID),
})
}
return reserved, pinnedHost, nil
}
func (s *SandboxService) releaseVolumeReservations(ctx context.Context, reserved []reservedVolume) {
for _, rv := range reserved {
if err := s.DB.ReleaseVolumeReservation(ctx, rv.ID); err != nil {
slog.Warn("failed to release volume reservation", "volume", id.FormatVolumeID(rv.ID), "error", err)
}
}
}
// linkVolumeReservations stamps the owning sandbox onto each reservation once
// the sandbox row exists. Reservations are claimed before the insert (the FK
// would reject an earlier write), which would otherwise leave them anonymous —
// and an anonymous reservation is indistinguishable from one whose capsule
// really did boot, so nothing could safely reclaim it.
func (s *SandboxService) linkVolumeReservations(ctx context.Context, reserved []reservedVolume, sandboxID pgtype.UUID) {
for _, rv := range reserved {
if err := s.DB.LinkVolumeReservation(ctx, db.LinkVolumeReservationParams{
ID: rv.ID, SandboxID: sandboxID,
}); err != nil {
slog.Warn("failed to link volume reservation to sandbox", "volume", id.FormatVolumeID(rv.ID), "error", err)
}
}
}
// markVolumesAttachedRetries bounds how hard we try to record a successful
// attach. The capsule is already running with the volume mounted at this point,
// so losing this write leaves the row stuck at 'attaching' until the capsule
// ends — recoverable, but worth retrying through a transient DB blip.
const markVolumesAttachedRetries = 3
// destroySandboxRetries bounds how many times a destroy is re-sent before the
// capsule is handed to the host monitor. Worth retrying because the failure
// path deliberately reaches no terminal state — it leaves the capsule in
// "stopping" with its volumes still claimed.
const destroySandboxRetries = 3
func (s *SandboxService) markVolumesAttached(ctx context.Context, reserved []reservedVolume, hostID, sandboxID pgtype.UUID) {
for _, rv := range reserved {
var err error
for attempt := range markVolumesAttachedRetries {
if attempt > 0 {
select {
case <-ctx.Done():
return
case <-time.After(time.Duration(attempt) * 500 * time.Millisecond):
}
}
if _, err = s.DB.MarkVolumeAttached(ctx, db.MarkVolumeAttachedParams{
ID: rv.ID, HostID: hostID, SandboxID: sandboxID, MountPath: rv.MountPath,
}); err == nil {
break
}
}
if err != nil {
// The row keeps its sandbox_id, so the host-monitor sweep still
// frees it when the capsule reaches a terminal state. Loud because
// the volume is unusable by anything else until then.
slog.Error("failed to mark volume attached; volume stays reserved until its capsule ends",
"volume", id.FormatVolumeID(rv.ID), "sandbox_id", id.FormatSandboxID(sandboxID), "error", err)
}
}
}
// sandboxConfirmedGone asks the host to destroy a capsule and reports whether
// the host is now known not to be running it. Used on failure paths before
// freeing resources the capsule may still hold: a NotFound means it never
// existed, a success means it has been torn down, and anything else (including
// an unreachable host) means we cannot tell and must not assume.
func (s *SandboxService) sandboxConfirmedGone(ctx context.Context, agent hostagentClient, sandboxIDStr string) bool {
_, err := agent.DestroySandbox(ctx, connect.NewRequest(&pb.DestroySandboxRequest{
SandboxId: sandboxIDStr,
}))
return err == nil || connect.CodeOf(err) == connect.CodeNotFound
}
func volumeSpecsToProto(reserved []reservedVolume) []*pb.VolumeSpec {
if len(reserved) == 0 {
return nil
}
specs := make([]*pb.VolumeSpec, 0, len(reserved))
for _, rv := range reserved {
specs = append(specs, &pb.VolumeSpec{
VolumeId: id.UUIDString(rv.ID),
TeamId: id.UUIDString(rv.TeamID),
SizeMb: rv.SizeMB,
MountPath: rv.MountPath,
})
}
return specs
}
// resolvePinnedHost returns the host a set of already-pinned volumes forces the
// capsule onto, verifying it is online and eligible for the team. It bypasses
// the scheduler: a pinned volume has exactly one valid host, so there is no
// placement choice — only a check that the host can still run the capsule.
func (s *SandboxService) resolvePinnedHost(ctx context.Context, hostID pgtype.UUID, isByoc bool, teamID pgtype.UUID) (db.Host, error) {
host, err := s.DB.GetHost(ctx, hostID)
if err != nil {
return db.Host{}, apperr.VolumeHostMismatch.WrapMsg(err, "The host holding this volume no longer exists.")
}
if host.Status != "online" {
return db.Host{}, apperr.VolumeHostMismatch.Msg("The host holding this volume is not online. Try again once it recovers.")
}
if isByoc {
if host.Type != "byoc" || !host.TeamID.Valid || host.TeamID.Bytes != teamID.Bytes {
return db.Host{}, apperr.VolumeHostMismatch.New()
}
} else if host.Type != "regular" {
return db.Host{}, apperr.VolumeHostMismatch.New()
}
return host, nil
}
// hostagentClient is a local alias to avoid the full package path in signatures.
type hostagentClient = interface {
CreateSandbox(ctx context.Context, req *connect.Request[pb.CreateSandboxRequest]) (*connect.Response[pb.CreateSandboxResponse], error)
@ -116,6 +311,7 @@ type hostagentClient = interface {
GetSandboxMetrics(ctx context.Context, req *connect.Request[pb.GetSandboxMetricsRequest]) (*connect.Response[pb.GetSandboxMetricsResponse], error)
FlushSandboxMetrics(ctx context.Context, req *connect.Request[pb.FlushSandboxMetricsRequest]) (*connect.Response[pb.FlushSandboxMetricsResponse], error)
CreateSnapshot(ctx context.Context, req *connect.Request[pb.CreateSnapshotRequest]) (*connect.Response[pb.CreateSnapshotResponse], error)
DeleteVolume(ctx context.Context, req *connect.Request[pb.DeleteVolumeRequest]) (*connect.Response[pb.DeleteVolumeResponse], error)
}
// resolveTemplateRef resolves a parsed template reference to its owning
@ -207,9 +403,34 @@ func (s *SandboxService) Create(ctx context.Context, p SandboxCreateParams) (db.
return db.Sandbox{}, apperr.TeamNotFound.Wrap(err)
}
host, err := s.Scheduler.SelectHost(ctx, p.TeamID, team.IsByoc, p.MemoryMB, 0)
sandboxID := id.NewSandboxID()
// Claim any requested volumes and learn the host they pin the capsule to.
// The reservations are released by the deferred cleanup below unless the
// create hands them off to the background goroutine (committed = true).
reserved, pinnedHost, err := s.reserveVolumesForCreate(ctx, p.TeamID, p.VolumeRefs)
if err != nil {
return db.Sandbox{}, fmt.Errorf("select host: %w", err)
return db.Sandbox{}, err
}
committed := false
defer func() {
if !committed {
s.releaseVolumeReservations(ctx, reserved)
}
}()
// A pinned volume forces the host; otherwise the scheduler picks one.
var host db.Host
if pinnedHost.Valid {
host, err = s.resolvePinnedHost(ctx, pinnedHost, team.IsByoc, p.TeamID)
} else {
host, err = s.Scheduler.SelectHost(ctx, p.TeamID, team.IsByoc, p.MemoryMB, 0)
if err != nil {
err = fmt.Errorf("select host: %w", err)
}
}
if err != nil {
return db.Sandbox{}, err
}
agent, err := s.Pool.GetForHost(host)
@ -217,7 +438,6 @@ func (s *SandboxService) Create(ctx context.Context, p SandboxCreateParams) (db.
return db.Sandbox{}, fmt.Errorf("get agent client: %w", err)
}
sandboxID := id.NewSandboxID()
sandboxIDStr := id.FormatSandboxID(sandboxID)
hostIDStr := id.FormatHostID(host.ID)
@ -246,9 +466,18 @@ func (s *SandboxService) Create(ctx context.Context, p SandboxCreateParams) (db.
return db.Sandbox{}, fmt.Errorf("insert sandbox: %w", err)
}
// The sandbox row now exists, so the reservations can name their owner.
// Do this before handing off to the background goroutine: from here on,
// every claimed volume is attributable to a capsule the host monitor can
// reconcile against live host state.
s.linkVolumeReservations(ctx, reserved, sandboxID)
teamIDStr := id.FormatTeamID(p.TeamID)
s.publishStateChanged(ctx, sandboxIDStr, teamIDStr, hostIDStr, "", "starting")
go s.createInBackground(sandboxID, sandboxIDStr, hostIDStr, teamIDStr, agent, p, templateTeamID, templateID, templateDefaultUser, templateDefaultEnv)
// The background goroutine now owns the reservations (it marks them attached
// on success or releases them on failure).
committed = true
go s.createInBackground(sandboxID, sandboxIDStr, hostIDStr, teamIDStr, agent, p, templateTeamID, templateID, templateDefaultUser, templateDefaultEnv, reserved, host.ID)
return sb, nil
}
@ -258,6 +487,7 @@ func (s *SandboxService) createInBackground(
agent hostagentClient, p SandboxCreateParams,
templateTeamID, templateID pgtype.UUID,
defaultUser string, defaultEnv map[string]string,
reserved []reservedVolume, hostID pgtype.UUID,
) {
bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
@ -273,23 +503,46 @@ func (s *SandboxService) createInBackground(
DiskSizeMb: 0,
DefaultUser: defaultUser,
DefaultEnv: defaultEnv,
Volumes: volumeSpecsToProto(reserved),
}))
if err != nil {
slog.Warn("background create failed", "sandbox_id", sandboxIDStr, "error", err)
errCtx, errCancel := context.WithTimeout(context.Background(), 10*time.Second)
errCtx, errCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer errCancel()
if _, dbErr := s.DB.UpdateSandboxStatusIf(errCtx, db.UpdateSandboxStatusIfParams{
ID: sandboxID, Status: "starting", Status_2: "error",
}); dbErr != nil {
slog.Warn("failed to update sandbox to error after create failure", "id", sandboxIDStr, "error", dbErr)
// A failed RPC does not prove the capsule never booted — a lost or
// timed-out response leaves a live VM with the volumes mounted. Tear it
// down on the host first, and only free the volumes once the host
// confirms it is gone. Releasing a volume that a running VM still has
// open would let the next capsule attach the same backing file.
if s.sandboxConfirmedGone(errCtx, agent, sandboxIDStr) {
s.releaseVolumeReservations(errCtx, reserved)
if _, dbErr := s.DB.UpdateSandboxStatusIf(errCtx, db.UpdateSandboxStatusIfParams{
ID: sandboxID, Status: "starting", Status_2: "error",
}); dbErr != nil {
slog.Warn("failed to update sandbox to error after create failure", "id", sandboxIDStr, "error", dbErr)
}
s.publishEvent(errCtx, SandboxStateEvent{
Event: "sandbox.failed", SandboxID: sandboxIDStr, TeamID: teamIDStr, HostID: hostIDStr,
Error: err.Error(), Timestamp: time.Now().Unix(),
})
return
}
s.publishEvent(errCtx, SandboxStateEvent{
Event: "sandbox.failed", SandboxID: sandboxIDStr, TeamID: teamIDStr, HostID: hostIDStr,
Error: err.Error(), Timestamp: time.Now().Unix(),
})
// The host could not confirm the teardown, so the capsule may still be
// running with its volumes. Leave the sandbox in "starting" and the
// volumes reserved: the host monitor owns both from here, and resolves
// them against live host state once the host is reachable again.
slog.Error("create failed and host could not confirm teardown; leaving capsule for host monitor",
"sandbox_id", sandboxIDStr, "host_id", hostIDStr, "volumes", len(reserved))
return
}
// The capsule booted with its volumes attached; promote each reservation to
// attached, record the owning sandbox, and pin the volume to this host
// (COALESCE keeps an existing pin).
s.markVolumesAttached(bgCtx, reserved, hostID, sandboxID)
if resp.Msg.DiskSizeMb > 0 {
if err := s.DB.UpdateSandboxDiskSize(bgCtx, db.UpdateSandboxDiskSizeParams{
ID: sandboxID,
@ -714,10 +967,49 @@ func (s *SandboxService) destroyInBackground(sandboxID pgtype.UUID, sandboxIDStr
s.flushAndPersistMetrics(bgCtx, agent, sandboxID, false)
}
if _, err := agent.DestroySandbox(bgCtx, connect.NewRequest(&pb.DestroySandboxRequest{
SandboxId: sandboxIDStr,
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
slog.Warn("background destroy failed", "sandbox_id", sandboxIDStr, "error", err)
// Retry before giving up: most non-NotFound failures here are transport
// hiccups, and the alternative — leaving the capsule mid-destroy for the
// host monitor — costs the user a reconciliation cycle.
var destroyErr error
for attempt := range destroySandboxRetries {
if attempt > 0 {
select {
case <-bgCtx.Done():
return
case <-time.After(time.Duration(attempt) * time.Second):
}
}
_, destroyErr = agent.DestroySandbox(bgCtx, connect.NewRequest(&pb.DestroySandboxRequest{
SandboxId: sandboxIDStr,
}))
// NotFound means the host does not have it — the outcome we wanted.
if destroyErr == nil || connect.CodeOf(destroyErr) == connect.CodeNotFound {
destroyErr = nil
break
}
}
if destroyErr != nil {
// The host never confirmed the teardown, so the VM may still be running
// with its volumes mounted. Leave the capsule in "stopping" and its
// volumes held — freeing a volume here could hand its backing file to a
// second capsule while the first still has it open. The host monitor
// re-issues the destroy against live host state.
//
// Publish the failure so the operation is not silently open-ended: this
// is the one path that reaches no terminal state of its own.
slog.Error("background destroy failed; leaving capsule for host monitor",
"sandbox_id", sandboxIDStr, "error", destroyErr)
s.publishEvent(bgCtx, SandboxStateEvent{
Event: "sandbox.destroy_failed", SandboxID: sandboxIDStr, TeamID: teamIDStr, HostID: hostIDStr,
Error: destroyErr.Error(), Timestamp: time.Now().Unix(),
})
return
}
// Free any attached volumes back to detached (data and host pin preserved).
// Volumes are never auto-deleted — only an explicit delete removes them.
if err := s.DB.DetachVolumesBySandbox(bgCtx, sandboxID); err != nil {
slog.Warn("failed to detach volumes on destroy", "sandbox_id", sandboxIDStr, "error", err)
}
if prevStatus == "paused" {

View File

@ -2,6 +2,7 @@ package service
import (
"context"
"errors"
"fmt"
"log/slog"
"regexp"
@ -346,6 +347,15 @@ func (s *TeamService) deleteTeamCore(ctx context.Context, teamID pgtype.UUID) er
// as that would leave orphaned "running" records for a deleted team.
return fmt.Errorf("update sandbox statuses: %w", err)
}
// Free any volumes those capsules held. The destroys above already ran
// against the host, so this is the same confirmed-gone signal the host
// monitor uses — no VM is left holding a backing file open. Doing it here
// (rather than relying on the volume cleanup below) keeps volume state
// consistent even if that cleanup fails.
if err := s.DB.DetachVolumesBySandboxIDs(ctx, stopIDs); err != nil {
slog.Warn("team delete: failed to detach volumes", "team_id", id.FormatTeamID(teamID), "error", err)
}
}
// Delete sandbox metrics for this team.
@ -366,8 +376,10 @@ func (s *TeamService) deleteTeamCore(ctx context.Context, teamID pgtype.UUID) er
slog.Warn("team delete: failed to delete channels", "team_id", id.FormatTeamID(teamID), "error", err)
}
// Clean up team-owned templates from all hosts in the background.
// Clean up team-owned templates and storage volumes from all hosts in the
// background.
go s.cleanupTeamTemplates(context.Background(), teamID)
go s.cleanupTeamVolumes(context.Background(), teamID)
if err := s.DB.SoftDeleteTeam(ctx, teamID); err != nil {
return fmt.Errorf("soft delete team: %w", err)
@ -421,6 +433,65 @@ func (s *TeamService) cleanupTeamTemplates(ctx context.Context, teamID pgtype.UU
}
}
// cleanupTeamVolumes deletes every storage volume owned by a team being deleted:
// the backing file on each volume's pinned host, then the DB records. Called
// asynchronously during team deletion.
//
// Normal volume deletion is always explicit (VolumeService.Delete) — a destroyed
// capsule only frees its volumes, it never removes them. Team deletion is the
// exception: the owner of the volumes is going away, so leaving the data behind
// would orphan it on the host with no way left to reach it.
//
// Unlike VolumeService.Delete there is no per-volume status claim: the team's
// capsules were destroyed before this ran, so nothing can be attaching or
// re-attaching. Records are dropped even when a host is unreachable, matching
// template cleanup — a row kept for a deleted team is unreachable forever and
// would keep blocking deletion of the host it is pinned to.
func (s *TeamService) cleanupTeamVolumes(ctx context.Context, teamID pgtype.UUID) {
volumes, err := s.DB.ListVolumesByTeam(ctx, teamID)
if err != nil {
slog.Warn("team delete: failed to list volumes for cleanup", "team_id", id.FormatTeamID(teamID), "error", err)
return
}
if len(volumes) == 0 {
return
}
for _, vol := range volumes {
// A volume that was never attached has no file on any host, and one whose
// pinned host is gone lost its data with that host's disk. Either way
// there is nothing to remove — just drop the record below.
if !vol.HostID.Valid {
continue
}
host, err := s.DB.GetHost(ctx, vol.HostID)
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
slog.Warn("team delete: failed to load volume host",
"volume_id", id.FormatVolumeID(vol.ID), "error", err)
}
continue
}
agent, err := s.HostPool.GetForHost(host)
if err != nil {
slog.Warn("team delete: failed to reach volume host",
"host_id", id.FormatHostID(host.ID), "volume_id", id.FormatVolumeID(vol.ID), "error", err)
continue
}
if _, err := agent.DeleteVolume(ctx, connect.NewRequest(&pb.DeleteVolumeRequest{
TeamId: id.UUIDString(teamID),
VolumeId: id.UUIDString(vol.ID),
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
slog.Warn("team delete: failed to delete volume on host",
"host_id", id.FormatHostID(host.ID), "volume_id", id.FormatVolumeID(vol.ID), "error", err)
}
}
if err := s.DB.DeleteVolumesByTeam(ctx, teamID); err != nil {
slog.Warn("team delete: failed to delete volume records", "team_id", id.FormatTeamID(teamID), "error", err)
}
}
// GetMembers returns all members of the team with their emails and roles.
func (s *TeamService) GetMembers(ctx context.Context, teamID pgtype.UUID) ([]MemberInfo, error) {
rows, err := s.DB.GetTeamMembers(ctx, teamID)

197
pkg/service/volume.go Normal file
View File

@ -0,0 +1,197 @@
package service
import (
"context"
"errors"
"fmt"
"strings"
"connectrpc.com/connect"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"git.omukk.dev/wrenn/wrenn/internal/units"
"git.omukk.dev/wrenn/wrenn/pkg/apperr"
"git.omukk.dev/wrenn/wrenn/pkg/db"
"git.omukk.dev/wrenn/wrenn/pkg/id"
"git.omukk.dev/wrenn/wrenn/pkg/lifecycle"
"git.omukk.dev/wrenn/wrenn/pkg/validate"
pb "git.omukk.dev/wrenn/wrenn/proto/hostagent/gen"
)
// MinVolumeSizeMB is the smallest volume that can be created. Below roughly
// this size an ext4 filesystem's own metadata dominates the usable space.
const MinVolumeSizeMB int32 = 100
// DefaultMaxVolumeSizeMB is the per-volume ceiling used when a VolumeService is
// constructed without one. Deployments override it with WRENN_MAX_VOLUME_SIZE.
const DefaultMaxVolumeSizeMB int32 = 20 * 1024 // 20 GiB
// VolumeService provides external-storage-volume lifecycle operations. Volume
// metadata lives entirely in Postgres; the only host interaction is deleting a
// pinned volume's backing file. Unlike SandboxService there is no async state
// machine — create/list/get are pure DB ops and delete is synchronous.
//
// Attach/detach are not offered here: a volume is attached only at capsule
// create (see SandboxService.Create) and freed when the capsule is destroyed.
type VolumeService struct {
DB *db.Queries
Pool *lifecycle.HostClientPool
// MaxSizeMB caps the size of a single volume. Zero means
// DefaultMaxVolumeSizeMB.
MaxSizeMB int32
}
func (s *VolumeService) maxSizeMB() int32 {
if s.MaxSizeMB > 0 {
return s.MaxSizeMB
}
return DefaultMaxVolumeSizeMB
}
// Create records a new detached volume. No host is chosen yet — a volume is
// pinned to a host only when first attached to a capsule.
//
// name is optional: an empty name yields "vl-<volume id>", so a caller that
// does not care about naming still gets a stable, addressable handle. A
// supplied name is normalized to its "vl-"-prefixed slug form.
func (s *VolumeService) Create(ctx context.Context, teamID pgtype.UUID, name string, sizeMB int32) (db.Volume, error) {
if !teamID.Valid {
return db.Volume{}, apperr.InvalidRequest.Msg("A team_id is required.")
}
if sizeMB < MinVolumeSizeMB {
return db.Volume{}, apperr.InvalidRequest.Msgf("Volume size must be at least %s.", units.FormatMB(int(MinVolumeSizeMB)))
}
if max := s.maxSizeMB(); sizeMB > max {
return db.Volume{}, apperr.InvalidRequest.Msgf("Volume size must not exceed %s.", units.FormatMB(int(max)))
}
volumeID := id.NewVolumeID()
// A nameless volume is named after itself. id.FormatVolumeID carries the
// "vol-" ID prefix, so use the bare base36 body to keep the name in the
// "vl-" namespace.
if strings.TrimSpace(name) == "" {
name = validate.VolumeNamePrefix + id.UUIDToBase36(volumeID.Bytes)
}
name, err := validate.VolumeName(name)
if err != nil {
return db.Volume{}, apperr.ValidationFailed.Msgf("Invalid volume name: %v.", err)
}
vol, err := s.DB.InsertVolume(ctx, db.InsertVolumeParams{
ID: volumeID,
TeamID: teamID,
Name: name,
SizeMb: sizeMB,
})
if err != nil {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
return db.Volume{}, apperr.VolumeNameTaken.Msgf("A volume named %q already exists in this team.", name)
}
return db.Volume{}, fmt.Errorf("insert volume: %w", err)
}
return vol, nil
}
// Resolve turns a user-supplied reference into the team's volume. A reference
// is either a volume ID ("vol-<base36>") or a name ("vl-cache", or plain
// "cache" — the prefix is optional on input). The two namespaces cannot
// collide: an ID always carries the "vol-" prefix, a name always "vl-".
func (s *VolumeService) Resolve(ctx context.Context, teamID pgtype.UUID, ref string) (db.Volume, error) {
ref = strings.TrimSpace(ref)
if ref == "" {
return db.Volume{}, apperr.InvalidRequest.Msg("A volume ID or name is required.")
}
if strings.HasPrefix(ref, id.PrefixVolume) {
volumeID, err := id.ParseVolumeID(ref)
if err != nil {
return db.Volume{}, apperr.InvalidRequest.WrapMsg(err, "Invalid volume ID.")
}
return s.Get(ctx, volumeID, teamID)
}
name, err := validate.VolumeName(ref)
if err != nil {
return db.Volume{}, apperr.InvalidRequest.Msgf("Invalid volume reference: %v.", err)
}
vol, err := s.DB.GetVolumeByTeamAndName(ctx, db.GetVolumeByTeamAndNameParams{TeamID: teamID, Name: name})
if err != nil {
return db.Volume{}, apperr.VolumeNotFound.WrapMsg(err, fmt.Sprintf("Volume %q not found.", name))
}
return vol, nil
}
// List returns all volumes owned by the team, newest first.
func (s *VolumeService) List(ctx context.Context, teamID pgtype.UUID) ([]db.Volume, error) {
return s.DB.ListVolumesByTeam(ctx, teamID)
}
// Get returns a single volume owned by the team.
func (s *VolumeService) Get(ctx context.Context, volumeID, teamID pgtype.UUID) (db.Volume, error) {
vol, err := s.DB.GetVolumeByTeam(ctx, db.GetVolumeByTeamParams{ID: volumeID, TeamID: teamID})
if err != nil {
return db.Volume{}, apperr.VolumeNotFound.Wrap(err)
}
return vol, nil
}
// Delete removes a detached volume: it deletes the backing file on the volume's
// pinned host (if any), then the DB row. An attached volume cannot be deleted —
// the capsule using it must be destroyed first.
func (s *VolumeService) Delete(ctx context.Context, volumeID, teamID pgtype.UUID) error {
// Atomically claim the volume for deletion (detached → deleting). The CAS
// blocks a concurrent attach — which needs status='detached' — closing the
// TOCTOU window between checking the status and removing the file/row.
vol, err := s.DB.BeginVolumeDelete(ctx, db.BeginVolumeDeleteParams{ID: volumeID, TeamID: teamID})
if err != nil {
if !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("claim volume for delete: %w", err)
}
// Not claimable: either the volume is missing / not this team's, or it
// is currently attached/attaching. Distinguish for a precise error.
if _, gerr := s.DB.GetVolumeByTeam(ctx, db.GetVolumeByTeamParams{ID: volumeID, TeamID: teamID}); gerr != nil {
return apperr.VolumeNotFound.Wrap(gerr)
}
return apperr.VolumeInUse.New()
}
// Remove the backing file from the host that holds it. A volume that was
// never attached (host_id NULL) has no file on any host — just drop the row.
if vol.HostID.Valid {
host, err := s.DB.GetHost(ctx, vol.HostID)
switch {
case errors.Is(err, pgx.ErrNoRows):
// The pinned host was deleted; its disk (and the volume's file) are
// gone with it. Nothing to remove — proceed to drop the row so the
// volume isn't left permanently un-deletable.
case err != nil:
_ = s.DB.AbortVolumeDelete(ctx, volumeID) // revert to detached
return apperr.HostNotFound.Wrap(err)
default:
agent, err := s.Pool.GetForHost(host)
if err != nil {
_ = s.DB.AbortVolumeDelete(ctx, volumeID)
return fmt.Errorf("get agent client: %w", err)
}
// NotFound is fine — the file may already be gone. Any other error
// must fail the delete (reverting the claim) so we never drop the
// row while the file lingers on a reachable host.
if _, err := agent.DeleteVolume(ctx, connect.NewRequest(&pb.DeleteVolumeRequest{
TeamId: id.UUIDString(teamID),
VolumeId: id.UUIDString(volumeID),
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
_ = s.DB.AbortVolumeDelete(ctx, volumeID)
return fmt.Errorf("delete volume file on host: %w", err)
}
}
}
if err := s.DB.DeleteVolumeRow(ctx, db.DeleteVolumeRowParams{ID: volumeID, TeamID: teamID}); err != nil {
return fmt.Errorf("delete volume row: %w", err)
}
return nil
}

44
pkg/validate/volume.go Normal file
View File

@ -0,0 +1,44 @@
package validate
import (
"fmt"
"strings"
)
// VolumeNamePrefix is the mandatory leading marker on every volume name. It
// makes a name self-describing in logs, mount paths, and SDK code, and — since
// volume IDs carry the distinct "vol-" prefix — lets a single API path segment
// be resolved as either an ID or a name without ambiguity.
const VolumeNamePrefix = "vl-"
// VolumeName normalizes and validates a user-supplied volume name. The name
// body follows the same rules as a team slug (lowercase alphanumerics in
// dash-separated groups) so names stay safe in paths, URLs, and shell contexts.
//
// The prefix is optional on input: both "cache" and "vl-cache" normalize to
// "vl-cache", so an SDK caller who forgets it still gets the volume they meant.
// The returned name is always the prefixed, lower-cased form and is what
// callers should store and compare against.
func VolumeName(name string) (string, error) {
normalized := strings.ToLower(strings.TrimSpace(name))
if normalized == "" {
return "", fmt.Errorf("volume name must not be empty")
}
normalized = strings.TrimPrefix(normalized, VolumeNamePrefix)
if normalized == "" {
return "", fmt.Errorf("volume name must have something after %q", VolumeNamePrefix)
}
normalized = VolumeNamePrefix + normalized
// The whole prefixed name is capped at 40 characters, which comfortably
// fits the default "vl-" + 25-char base36 ID form (28 characters).
if len(normalized) > 40 {
return "", fmt.Errorf("volume name must be at most 40 characters including the %q prefix", VolumeNamePrefix)
}
// slugRe is shared with team slugs: lowercase alphanumerics in
// dash-separated groups, no leading, trailing, or repeated dashes.
if !slugRe.MatchString(normalized) {
return "", fmt.Errorf("volume name may only contain lowercase letters, numbers, and dashes (no leading, trailing, or repeated dashes)")
}
return normalized, nil
}

View File

@ -0,0 +1,78 @@
package validate
import (
"strings"
"testing"
)
func TestVolumeName(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{"bare-name-gets-prefix", "cache", "vl-cache", false},
{"already-prefixed", "vl-cache", "vl-cache", false},
{"multi-group", "vl-build-cache-2", "vl-build-cache-2", false},
{"digits", "vl-1", "vl-1", false},
{"uppercase-lowered", "VL-Cache", "vl-cache", false},
{"trims-space", " cache ", "vl-cache", false},
{"default-id-form", "vl-" + strings.Repeat("a", 25), "vl-" + strings.Repeat("a", 25), false},
{"max-length", "vl-" + strings.Repeat("a", 37), "vl-" + strings.Repeat("a", 37), false},
// A second "vl-" is part of the name, not a repeated prefix — only one
// is ever stripped, so this stays distinct from "vl-cache".
{"double-prefix-kept", "vl-vl-cache", "vl-vl-cache", false},
{"empty", "", "", true},
{"prefix-only", "vl-", "", true},
{"whitespace-only", " ", "", true},
{"too-long", "vl-" + strings.Repeat("a", 38), "", true},
{"underscore", "my_cache", "", true},
{"dot", "my.cache", "", true},
{"slash", "team/cache", "", true},
{"space-inside", "my cache", "", true},
{"leading-dash", "-cache", "", true},
{"trailing-dash", "cache-", "", true},
{"double-dash", "my--cache", "", true},
{"non-ascii", "café", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := VolumeName(tt.input)
if (err != nil) != tt.wantErr {
t.Fatalf("VolumeName(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
}
if got != tt.want {
t.Errorf("VolumeName(%q) = %q, want %q", tt.input, got, tt.want)
}
})
}
}
// Normalizing an already-normalized name must be a no-op, otherwise a name
// would drift every time it round-trips through the API.
func TestVolumeNameIsIdempotent(t *testing.T) {
for _, input := range []string{"cache", "vl-cache", "VL-Build-Cache", "vl-vl-cache"} {
once, err := VolumeName(input)
if err != nil {
t.Fatalf("VolumeName(%q) unexpected error: %v", input, err)
}
twice, err := VolumeName(once)
if err != nil {
t.Fatalf("VolumeName(%q) unexpected error: %v", once, err)
}
if once != twice {
t.Errorf("VolumeName not idempotent for %q: %q then %q", input, once, twice)
}
}
}
// A volume ID prefix must never be mistaken for a name prefix, which is what
// lets one path segment carry either without ambiguity.
func TestVolumeNamePrefixDoesNotCollideWithIDPrefix(t *testing.T) {
if strings.HasPrefix("vol-", VolumeNamePrefix) || strings.HasPrefix(VolumeNamePrefix, "vol-") {
t.Fatalf("volume name prefix %q collides with the vol- ID prefix", VolumeNamePrefix)
}
}

View File

@ -107,6 +107,9 @@ type chDisk struct {
Path string `json:"path"`
Readonly bool `json:"readonly,omitempty"`
ImageType string `json:"image_type,omitempty"`
// Serial is exposed to the guest as the virtio-blk serial so envd can
// resolve the device via /sys/block/*/serial. Only set for data volumes.
Serial string `json:"serial,omitempty"`
}
type chNet struct {
@ -141,6 +144,23 @@ type chCreatePayload struct {
func (c *chClient) createVM(ctx context.Context, cfg *VMConfig) error {
memBytes := uint64(cfg.MemoryMB) * 1024 * 1024
// Rootfs is always disk 0 (guest /dev/vda, referenced by root=/dev/vda in
// the kernel cmdline). Data volumes follow as vdb, vdc, ... — but the guest
// resolves each by serial, not by enumeration order.
disks := []chDisk{
{
Path: cfg.SandboxDir + "/rootfs.ext4",
ImageType: "Raw",
},
}
for _, v := range cfg.Volumes {
disks = append(disks, chDisk{
Path: cfg.SandboxDir + "/" + v.symlinkName(),
ImageType: "Raw",
Serial: v.Serial,
})
}
payload := chCreatePayload{
Payload: chPayload{
Kernel: cfg.KernelPath,
@ -155,12 +175,7 @@ func (c *chClient) createVM(ctx context.Context, cfg *VMConfig) error {
Shared: true,
Thp: boolPtr(false),
},
Disks: []chDisk{
{
Path: cfg.SandboxDir + "/rootfs.ext4",
ImageType: "Raw",
},
},
Disks: disks,
Net: []chNet{
{
Tap: cfg.TapDevice,

View File

@ -15,6 +15,27 @@ func SandboxSocketPath(sandboxID string) string {
return fmt.Sprintf("/tmp/ch-%s.sock", sandboxID)
}
// VolumeDisk is an external storage volume attached to a VM at boot.
type VolumeDisk struct {
// HostPath is the backing sparse file on the host filesystem. It is
// visible inside the VMM's private mount namespace (unshare -m isolates
// the mount table, not the filesystem tree), so CH can open it via the
// stable symlink created in SandboxDir.
HostPath string
// Serial is the virtio-blk serial (derived from the volume ID) the guest
// uses to resolve the block device via /sys/block/*/serial. It also names
// the in-namespace symlink so the disk path is stable across restore.
Serial string
}
// symlinkName returns the volume's stable filename inside SandboxDir. CH bakes
// this path into its snapshot config.json, so it must be reconstructable on
// every restore — hence keyed by the (deterministic) serial, not enumeration
// order.
func (v VolumeDisk) symlinkName() string {
return "volume-" + v.Serial + ".img"
}
// VMConfig holds the configuration for creating a Cloud Hypervisor microVM.
type VMConfig struct {
// SandboxID is the unique identifier for this sandbox (e.g., "cl-a1b2c3d4").
@ -30,6 +51,12 @@ type VMConfig struct {
// Typically a dm-snapshot device (e.g., /dev/mapper/wrenn-cl-a1b2c3d4).
RootfsPath string
// Volumes are external storage disks attached at boot, after the rootfs.
// Each is symlinked into SandboxDir (like the rootfs) so the disk path CH
// records in its snapshot config.json stays stable across restore, and is
// exposed to the guest as an additional virtio-blk device (vdb, vdc, ...).
Volumes []VolumeDisk
// VCPUs is the number of virtual CPUs to allocate (default: 1).
VCPUs int

View File

@ -137,6 +137,15 @@ func buildLaunchScript(cfg *VMConfig, extraArgs string) string {
if extraArgs != "" {
chCmd += " " + extraArgs
}
// One symlink per data volume, pointing the stable in-namespace path (baked
// into CH's config.json) at the real backing file on the host. Recreated
// identically on every restore so the snapshot's disk paths resolve.
var volumeLinks strings.Builder
for _, v := range cfg.Volumes {
fmt.Fprintf(&volumeLinks, "ln -s %s %s/%s\n", v.HostPath, cfg.SandboxDir, v.symlinkName())
}
return fmt.Sprintf(`
set -euo pipefail
@ -147,13 +156,14 @@ mount -t tmpfs tmpfs %[1]s
ln -s %[2]s %[1]s/vmlinux
ln -s %[3]s %[1]s/rootfs.ext4
%[5]s
exec %[4]s
`,
cfg.SandboxDir, // 1
cfg.KernelPath, // 2
cfg.RootfsPath, // 3
chCmd, // 4
cfg.SandboxDir, // 1
cfg.KernelPath, // 2
cfg.RootfsPath, // 3
chCmd, // 4
volumeLinks.String(), // 5
)
}

File diff suppressed because it is too large Load Diff

View File

@ -122,6 +122,9 @@ const (
// HostAgentServiceGetTemplateSizeProcedure is the fully-qualified name of the HostAgentService's
// GetTemplateSize RPC.
HostAgentServiceGetTemplateSizeProcedure = "/hostagent.v1.HostAgentService/GetTemplateSize"
// HostAgentServiceDeleteVolumeProcedure is the fully-qualified name of the HostAgentService's
// DeleteVolume RPC.
HostAgentServiceDeleteVolumeProcedure = "/hostagent.v1.HostAgentService/DeleteVolume"
)
// HostAgentServiceClient is a client for the hostagent.v1.HostAgentService service.
@ -201,6 +204,10 @@ type HostAgentServiceClient interface {
// size 0 in the database (e.g. system base templates seeded before the
// rootfs was built).
GetTemplateSize(context.Context, *connect.Request[gen.GetTemplateSizeRequest]) (*connect.Response[gen.GetTemplateSizeResponse], error)
// DeleteVolume removes an external storage volume's backing file from this
// host. Called by the control plane when a detached volume is deleted; the
// volume must not be attached to any sandbox on this host.
DeleteVolume(context.Context, *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error)
}
// NewHostAgentServiceClient constructs a client for the hostagent.v1.HostAgentService service. By
@ -394,6 +401,12 @@ func NewHostAgentServiceClient(httpClient connect.HTTPClient, baseURL string, op
connect.WithSchema(hostAgentServiceMethods.ByName("GetTemplateSize")),
connect.WithClientOptions(opts...),
),
deleteVolume: connect.NewClient[gen.DeleteVolumeRequest, gen.DeleteVolumeResponse](
httpClient,
baseURL+HostAgentServiceDeleteVolumeProcedure,
connect.WithSchema(hostAgentServiceMethods.ByName("DeleteVolume")),
connect.WithClientOptions(opts...),
),
}
}
@ -429,6 +442,7 @@ type hostAgentServiceClient struct {
killProcess *connect.Client[gen.KillProcessRequest, gen.KillProcessResponse]
connectProcess *connect.Client[gen.ConnectProcessRequest, gen.ConnectProcessResponse]
getTemplateSize *connect.Client[gen.GetTemplateSizeRequest, gen.GetTemplateSizeResponse]
deleteVolume *connect.Client[gen.DeleteVolumeRequest, gen.DeleteVolumeResponse]
}
// CreateSandbox calls hostagent.v1.HostAgentService.CreateSandbox.
@ -581,6 +595,11 @@ func (c *hostAgentServiceClient) GetTemplateSize(ctx context.Context, req *conne
return c.getTemplateSize.CallUnary(ctx, req)
}
// DeleteVolume calls hostagent.v1.HostAgentService.DeleteVolume.
func (c *hostAgentServiceClient) DeleteVolume(ctx context.Context, req *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error) {
return c.deleteVolume.CallUnary(ctx, req)
}
// HostAgentServiceHandler is an implementation of the hostagent.v1.HostAgentService service.
type HostAgentServiceHandler interface {
// CreateSandbox boots a new microVM with the given configuration.
@ -658,6 +677,10 @@ type HostAgentServiceHandler interface {
// size 0 in the database (e.g. system base templates seeded before the
// rootfs was built).
GetTemplateSize(context.Context, *connect.Request[gen.GetTemplateSizeRequest]) (*connect.Response[gen.GetTemplateSizeResponse], error)
// DeleteVolume removes an external storage volume's backing file from this
// host. Called by the control plane when a detached volume is deleted; the
// volume must not be attached to any sandbox on this host.
DeleteVolume(context.Context, *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error)
}
// NewHostAgentServiceHandler builds an HTTP handler from the service implementation. It returns the
@ -847,6 +870,12 @@ func NewHostAgentServiceHandler(svc HostAgentServiceHandler, opts ...connect.Han
connect.WithSchema(hostAgentServiceMethods.ByName("GetTemplateSize")),
connect.WithHandlerOptions(opts...),
)
hostAgentServiceDeleteVolumeHandler := connect.NewUnaryHandler(
HostAgentServiceDeleteVolumeProcedure,
svc.DeleteVolume,
connect.WithSchema(hostAgentServiceMethods.ByName("DeleteVolume")),
connect.WithHandlerOptions(opts...),
)
return "/hostagent.v1.HostAgentService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case HostAgentServiceCreateSandboxProcedure:
@ -909,6 +938,8 @@ func NewHostAgentServiceHandler(svc HostAgentServiceHandler, opts ...connect.Han
hostAgentServiceConnectProcessHandler.ServeHTTP(w, r)
case HostAgentServiceGetTemplateSizeProcedure:
hostAgentServiceGetTemplateSizeHandler.ServeHTTP(w, r)
case HostAgentServiceDeleteVolumeProcedure:
hostAgentServiceDeleteVolumeHandler.ServeHTTP(w, r)
default:
http.NotFound(w, r)
}
@ -1037,3 +1068,7 @@ func (UnimplementedHostAgentServiceHandler) ConnectProcess(context.Context, *con
func (UnimplementedHostAgentServiceHandler) GetTemplateSize(context.Context, *connect.Request[gen.GetTemplateSizeRequest]) (*connect.Response[gen.GetTemplateSizeResponse], error) {
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("hostagent.v1.HostAgentService.GetTemplateSize is not implemented"))
}
func (UnimplementedHostAgentServiceHandler) DeleteVolume(context.Context, *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error) {
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("hostagent.v1.HostAgentService.DeleteVolume is not implemented"))
}

View File

@ -109,6 +109,26 @@ service HostAgentService {
// size 0 in the database (e.g. system base templates seeded before the
// rootfs was built).
rpc GetTemplateSize(GetTemplateSizeRequest) returns (GetTemplateSizeResponse);
// DeleteVolume removes an external storage volume's backing file from this
// host. Called by the control plane when a detached volume is deleted; the
// volume must not be attached to any sandbox on this host.
rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse);
}
// VolumeSpec describes an external storage volume to attach to a sandbox at
// boot time. The host provisions the backing sparse file on first use, hands
// it to Cloud Hypervisor as a Raw disk, and asks envd to format (if empty) and
// mount it in the guest.
message VolumeSpec {
// Volume UUID (hex string).
string volume_id = 1;
// Team UUID that owns the volume (hex string) — locates the backing file.
string team_id = 2;
// Size of the volume in MB (used to size the sparse file on first attach).
int32 size_mb = 3;
// Guest mount path. Empty means the default /mnt/<volume-id>.
string mount_path = 4;
}
message CreateSandboxRequest {
@ -142,6 +162,9 @@ message CreateSandboxRequest {
// Default environment variables (set in envd via PostInit).
map<string, string> default_env = 10;
// External storage volumes to attach at boot time and mount in the guest.
repeated VolumeSpec volumes = 11;
}
message CreateSandboxResponse {
@ -464,6 +487,15 @@ message GetTemplateSizeResponse {
int64 size_bytes = 1;
}
message DeleteVolumeRequest {
// Team UUID that owns the volume (hex string).
string team_id = 1;
// Volume UUID (hex string).
string volume_id = 2;
}
message DeleteVolumeResponse {}
// ── PTY ─────────────────────────────────────────────────────────────
message PtyAttachRequest {