Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a70ea99915 |
@ -52,6 +52,12 @@ WRENN_ENCRYPTION_KEY=
|
||||
# that deliver to internal endpoints.
|
||||
WRENN_CHANNELS_ALLOW_PRIVATE=false
|
||||
|
||||
# External storage volumes
|
||||
# Largest size a single volume may be created with. Accepts G/Gi/M/Mi suffixes
|
||||
# (same form as WRENN_DEFAULT_ROOTFS_SIZE). Volumes are sparse files, so this
|
||||
# bounds worst-case growth rather than upfront allocation.
|
||||
WRENN_MAX_VOLUME_SIZE=20Gi
|
||||
|
||||
# OAuth
|
||||
OAUTH_GITHUB_CLIENT_ID=
|
||||
OAUTH_GITHUB_CLIENT_SECRET=
|
||||
|
||||
@ -1 +1 @@
|
||||
0.4.0
|
||||
0.5.0
|
||||
|
||||
@ -1 +1 @@
|
||||
0.4.0
|
||||
0.5.0
|
||||
|
||||
@ -17,6 +17,7 @@ import (
|
||||
"github.com/joho/godotenv"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/hostagent"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/units"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/auth"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/logging"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/network"
|
||||
@ -98,7 +99,7 @@ func main() {
|
||||
// Parse default rootfs size from env (e.g. "5G", "2Gi", "1000M").
|
||||
defaultRootfsSizeMB := sandbox.DefaultDiskSizeMB
|
||||
if sizeStr := os.Getenv("WRENN_DEFAULT_ROOTFS_SIZE"); sizeStr != "" {
|
||||
parsed, err := sandbox.ParseSizeToMB(sizeStr)
|
||||
parsed, err := units.ParseSizeToMB(sizeStr)
|
||||
if err != nil {
|
||||
slog.Error("invalid WRENN_DEFAULT_ROOTFS_SIZE", "value", sizeStr, "error", err)
|
||||
os.Exit(1)
|
||||
|
||||
28
db/migrations/20260725043541_add_volumes.sql
Normal file
28
db/migrations/20260725043541_add_volumes.sql
Normal file
@ -0,0 +1,28 @@
|
||||
-- +goose Up
|
||||
-- External storage volumes: team-scoped block-storage disks that are attached
|
||||
-- to a capsule at create time and mounted inside the guest. A volume is
|
||||
-- host-local (Cloud Hypervisor can only attach a path on the same host), so it
|
||||
-- is pinned to a host the first time it is attached and stays there. Volumes
|
||||
-- are never auto-deleted: destroying a capsule frees its volumes back to
|
||||
-- 'detached' (data preserved); removal is always an explicit delete.
|
||||
CREATE TABLE volumes (
|
||||
id UUID PRIMARY KEY,
|
||||
team_id UUID NOT NULL REFERENCES teams(id),
|
||||
host_id UUID REFERENCES hosts(id), -- NULL until first attach; pins the volume thereafter
|
||||
name TEXT NOT NULL,
|
||||
size_mb INTEGER NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'detached', -- detached | attaching | attached | deleting
|
||||
sandbox_id UUID REFERENCES sandboxes(id), -- set while attaching/attached, cleared on capsule destroy
|
||||
mount_path TEXT NOT NULL DEFAULT '', -- guest mount path in use; '' means the default /mnt/<vol-id>
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_attached_at TIMESTAMPTZ,
|
||||
last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (team_id, name)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_volumes_team ON volumes(team_id);
|
||||
CREATE INDEX idx_volumes_host ON volumes(host_id);
|
||||
CREATE INDEX idx_volumes_sandbox ON volumes(sandbox_id) WHERE sandbox_id IS NOT NULL;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS volumes;
|
||||
41
db/migrations/20260725105719_volume_name_prefix.sql
Normal file
41
db/migrations/20260725105719_volume_name_prefix.sql
Normal file
@ -0,0 +1,41 @@
|
||||
-- +goose Up
|
||||
-- Volume names are now always "vl-"-prefixed slugs: lowercase alphanumerics in
|
||||
-- dash-separated groups, at most 40 characters. The prefix makes a name
|
||||
-- self-describing and — since volume IDs use the distinct "vol-" prefix — lets
|
||||
-- one API path segment be resolved as either an ID or a name unambiguously.
|
||||
--
|
||||
-- Backfill the names written before the rule existed. They were validated with
|
||||
-- the looser SafeName allowlist, so coerce them into slug shape first
|
||||
-- (lowercase, non-slug characters to dashes, collapse and trim dashes) and only
|
||||
-- then prefix. A name that survives as empty falls back to the volume's ID,
|
||||
-- which is also the default a nameless volume now gets.
|
||||
UPDATE volumes
|
||||
SET name = LEFT(
|
||||
'vl-' || COALESCE(
|
||||
NULLIF(TRIM(BOTH '-' FROM REGEXP_REPLACE(LOWER(name), '[^a-z0-9]+', '-', 'g')), ''),
|
||||
REPLACE(id::text, '-', '')
|
||||
),
|
||||
40
|
||||
)
|
||||
WHERE name !~ '^vl-[a-z0-9]+(-[a-z0-9]+)*$';
|
||||
|
||||
-- The truncation and coercion above can collide two previously-distinct names
|
||||
-- within a team, which UNIQUE (team_id, name) would have rejected. Nothing here
|
||||
-- can fail silently, so re-assert the invariant: any row still not matching the
|
||||
-- rule aborts the migration rather than leaving invalid names behind.
|
||||
-- +goose StatementBegin
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM volumes WHERE name !~ '^vl-[a-z0-9]+(-[a-z0-9]+)*$') THEN
|
||||
RAISE EXCEPTION 'volume name backfill left rows that are not valid vl- slugs';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- Strip the prefix back off. The original pre-coercion spelling is not
|
||||
-- recoverable, so this restores the shape, not the exact former value.
|
||||
UPDATE volumes
|
||||
SET name = REGEXP_REPLACE(name, '^vl-', '')
|
||||
WHERE name ~ '^vl-';
|
||||
178
db/queries/volumes.sql
Normal file
178
db/queries/volumes.sql
Normal file
@ -0,0 +1,178 @@
|
||||
-- name: InsertVolume :one
|
||||
INSERT INTO volumes (id, team_id, name, size_mb, status)
|
||||
VALUES ($1, $2, $3, $4, 'detached')
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetVolume :one
|
||||
SELECT * FROM volumes WHERE id = $1;
|
||||
|
||||
-- name: GetVolumeByTeam :one
|
||||
SELECT * FROM volumes WHERE id = $1 AND team_id = $2;
|
||||
|
||||
-- name: GetVolumeByTeamAndName :one
|
||||
-- Resolve a volume by its user-facing name. Names are unique per team, so this
|
||||
-- is the name-based counterpart to GetVolumeByTeam and is what lets the API
|
||||
-- accept "vl-cache" wherever it accepts "vol-<id>".
|
||||
SELECT * FROM volumes WHERE team_id = $1 AND name = $2;
|
||||
|
||||
-- name: ListVolumesByTeam :many
|
||||
SELECT * FROM volumes WHERE team_id = $1 ORDER BY created_at DESC;
|
||||
|
||||
-- name: ListVolumesBySandbox :many
|
||||
SELECT * FROM volumes WHERE sandbox_id = $1 ORDER BY created_at DESC;
|
||||
|
||||
-- name: CountVolumesByHost :one
|
||||
-- Guard used before a host can be removed: a host holding pinned volumes must
|
||||
-- not be deleted out from under them.
|
||||
SELECT COUNT(*) FROM volumes WHERE host_id = $1;
|
||||
|
||||
-- name: ReserveVolumeForAttach :one
|
||||
-- Atomically claim a detached volume for a capsule about to boot. The CAS on
|
||||
-- status = 'detached' enforces single-attach: a second capsule create racing
|
||||
-- for the same volume finds no row and fails. sandbox_id is deliberately NOT
|
||||
-- set here — the sandbox row does not exist yet, so setting the FK would fail.
|
||||
-- It is set in MarkVolumeAttached once the capsule has booted.
|
||||
UPDATE volumes
|
||||
SET status = 'attaching',
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'detached'
|
||||
RETURNING *;
|
||||
|
||||
-- name: MarkVolumeAttached :one
|
||||
-- Promote a reserved volume to attached once the capsule has booted with it.
|
||||
-- Records the owning sandbox and pins the volume to the sandbox's host on
|
||||
-- first-ever attach (COALESCE keeps an existing pin for a re-attached volume).
|
||||
UPDATE volumes
|
||||
SET status = 'attached',
|
||||
host_id = COALESCE(host_id, $2),
|
||||
sandbox_id = $3,
|
||||
mount_path = $4,
|
||||
last_attached_at = NOW(),
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'attaching'
|
||||
RETURNING *;
|
||||
|
||||
-- name: LinkVolumeReservation :exec
|
||||
-- Stamp the owning sandbox onto a reservation as soon as the sandbox row
|
||||
-- exists (ReserveVolumeForAttach runs before the insert, so it cannot set the
|
||||
-- FK itself). This makes every in-flight reservation attributable to a capsule,
|
||||
-- which is what lets the host monitor decide — against live host state —
|
||||
-- whether a stuck 'attaching' row is safe to free.
|
||||
UPDATE volumes
|
||||
SET sandbox_id = $2,
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'attaching';
|
||||
|
||||
-- name: ReleaseVolumeReservation :exec
|
||||
-- Roll a reserved volume back to detached when the capsule create fails.
|
||||
-- host_id is left untouched (a failed create never pins a fresh volume, since
|
||||
-- MarkVolumeAttached is what sets the pin). Only call this once the host has
|
||||
-- confirmed the capsule is not running — a volume freed while a VM still has
|
||||
-- its backing file open can be re-attached elsewhere and corrupted.
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'attaching';
|
||||
|
||||
-- name: DetachVolumesBySandbox :exec
|
||||
-- Terminal sweep run when a capsule reaches a terminal state (destroyed,
|
||||
-- stopped, errored, or reaped): free every volume it held back to detached
|
||||
-- (data and host pin preserved) so it can be reused or deleted. Keyed on
|
||||
-- sandbox_id, which is stamped on at reservation time. Never deletes the
|
||||
-- volume — removal is always explicit.
|
||||
--
|
||||
-- 'attaching' is swept alongside 'attached' so a reservation whose promotion
|
||||
-- never landed (lost RPC response, DB blip) is freed here — on confirmed host
|
||||
-- state — rather than on a blind timer.
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE sandbox_id = $1 AND status IN ('attached', 'attaching');
|
||||
|
||||
-- name: DetachVolumesBySandboxIDs :exec
|
||||
-- Bulk variant of DetachVolumesBySandbox for the host monitor, which stops
|
||||
-- many sandboxes at once (e.g. after a host goes offline).
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE sandbox_id = ANY($1::uuid[]) AND status IN ('attached', 'attaching');
|
||||
|
||||
-- name: DetachVolumesByHost :exec
|
||||
-- Free and un-pin every volume on a host being force-deleted. The host (and its
|
||||
-- backing files) are going away, so the volumes are reset to a fresh detached,
|
||||
-- un-pinned state — the rows survive (never auto-deleted) and can be re-attached
|
||||
-- on a new host. Also required before DeleteHost so the host_id FK does not
|
||||
-- block deletion.
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
host_id = NULL,
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE host_id = $1;
|
||||
|
||||
-- name: BeginVolumeDelete :one
|
||||
-- Atomically claim a volume for deletion. The CAS blocks a concurrent attach
|
||||
-- (which needs status='detached') so a volume can never be attached and deleted
|
||||
-- at the same time. 'deleting' is also accepted so a delete that crashed
|
||||
-- mid-flight can be retried rather than leaving the volume permanently stuck.
|
||||
-- Returns the row (for host_id) when claimed; no row means it is not deletable
|
||||
-- (attached/attaching, wrong team, or missing).
|
||||
UPDATE volumes
|
||||
SET status = 'deleting',
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND team_id = $2 AND status IN ('detached', 'deleting')
|
||||
RETURNING *;
|
||||
|
||||
-- name: AbortVolumeDelete :exec
|
||||
-- Revert a delete claim back to detached when the host-side file removal fails,
|
||||
-- so the volume stays usable rather than stuck in 'deleting'.
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'deleting';
|
||||
|
||||
-- name: ReleaseStaleVolumeReservations :execrows
|
||||
-- Free volumes stuck in a transient state whose owning operation died before it
|
||||
-- could ever reach a host — the control plane crashed between reserving a
|
||||
-- volume and inserting the sandbox row, or mid-delete. Run periodically by the
|
||||
-- volume reaper.
|
||||
--
|
||||
-- Deliberately limited to unattributed rows (sandbox_id IS NULL). A reservation
|
||||
-- that already carries a sandbox_id may correspond to a capsule that really did
|
||||
-- boot with the volume mounted — and a timer cannot tell the difference. Those
|
||||
-- are freed only by the host-monitor sweep, which first confirms against the
|
||||
-- host that the capsule is gone. Freeing a live volume here would let a second
|
||||
-- capsule attach the same backing file and corrupt it.
|
||||
--
|
||||
-- The cutoff ($1 = now - grace) MUST exceed the capsule create timeout so a
|
||||
-- legitimately in-flight attach is never freed out from under a booting capsule.
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE status IN ('attaching', 'deleting')
|
||||
AND sandbox_id IS NULL
|
||||
AND last_updated < $1;
|
||||
|
||||
-- name: DeleteVolumeRow :exec
|
||||
-- Drop the row only while the delete claim still holds. Without the status
|
||||
-- guard a delete that lost its claim mid-flight (reaped, or aborted and then
|
||||
-- re-attached by a racing capsule create) would erase the record of a volume
|
||||
-- that is once again in use.
|
||||
DELETE FROM volumes WHERE id = $1 AND team_id = $2 AND status = 'deleting';
|
||||
|
||||
-- name: DeleteVolumesByTeam :exec
|
||||
-- Drop every volume record belonging to a team being deleted. This is the one
|
||||
-- place volumes are removed without an explicit per-volume delete: the team that
|
||||
-- owns them is going away, so there is no longer anyone who could reference,
|
||||
-- re-attach, or delete them. Unlike DeleteVolumeRow there is no status guard —
|
||||
-- the team's capsules have already been destroyed, so no volume can still be
|
||||
-- legitimately in use, and a row left behind would be unreachable forever (and
|
||||
-- would keep blocking deletion of the host it is pinned to).
|
||||
DELETE FROM volumes WHERE team_id = $1;
|
||||
2
envd-rs/Cargo.lock
generated
2
envd-rs/Cargo.lock
generated
@ -529,7 +529,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "envd"
|
||||
version = "0.6.1"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"axum",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "envd"
|
||||
version = "0.6.1"
|
||||
version = "0.7.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.95"
|
||||
|
||||
|
||||
@ -22,25 +22,17 @@ pub struct InitRequest {
|
||||
#[serde(rename = "hyperloop_ip")]
|
||||
pub hyperloop_ip: Option<String>,
|
||||
pub timestamp: Option<String>,
|
||||
#[serde(rename = "volume_mounts")]
|
||||
pub volume_mounts: Option<Vec<VolumeMount>>,
|
||||
pub sandbox_id: Option<String>,
|
||||
pub template_id: Option<String>,
|
||||
/// Public proxy domain (e.g. "wrenn.dev"). Used by `envd ports` to build
|
||||
/// the {port}-{sandbox_id}.{domain} URLs.
|
||||
pub proxy_domain: Option<String>,
|
||||
/// New lifecycle identifier for this resume. When it changes between
|
||||
/// /init calls, envd treats the call as a post-resume hook: port
|
||||
/// forwarder is restarted and NFS mounts are refreshed.
|
||||
/// /init calls, envd treats the call as a post-resume hook: the port
|
||||
/// forwarder is restarted and the clock is stepped.
|
||||
pub lifecycle_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct VolumeMount {
|
||||
pub nfs_target: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// POST /init — called by host agent after boot.
|
||||
pub async fn post_init(
|
||||
State(state): State<Arc<AppState>>,
|
||||
@ -183,20 +175,6 @@ pub async fn post_init(
|
||||
setup_hyperloop(ip, &state.defaults.env_vars).await;
|
||||
}
|
||||
|
||||
// NFS mounts. Awaited in parallel so callers that immediately access the
|
||||
// mount path don't race the mount(2). Previously these were detached via
|
||||
// tokio::spawn, which let /init return success before mounts existed.
|
||||
if let Some(ref mounts) = init_req.volume_mounts {
|
||||
let futs = mounts.iter().map(|m| {
|
||||
let target = m.nfs_target.clone();
|
||||
let path = m.path.clone();
|
||||
async move {
|
||||
setup_nfs(&target, &path).await;
|
||||
}
|
||||
});
|
||||
futures::future::join_all(futs).await;
|
||||
}
|
||||
|
||||
// Set sandbox/template metadata from request body. Deliberately NOT
|
||||
// written into envd's own process environment: std::env::set_var is
|
||||
// undefined behavior with the multi-threaded runtime live (concurrent
|
||||
@ -293,45 +271,6 @@ async fn setup_hyperloop(address: &str, env_vars: &dashmap::DashMap<String, Stri
|
||||
env_vars.insert("WRENN_EVENTS_ADDRESS".into(), format!("http://{address}"));
|
||||
}
|
||||
|
||||
async fn setup_nfs(nfs_target: &str, path: &str) {
|
||||
let mkdir = tokio::process::Command::new("mkdir")
|
||||
.args(["-p", path])
|
||||
.output()
|
||||
.await;
|
||||
if let Err(e) = mkdir {
|
||||
tracing::error!(error = %e, path, "nfs: mkdir failed");
|
||||
return;
|
||||
}
|
||||
|
||||
let mount = tokio::process::Command::new("mount")
|
||||
.args([
|
||||
"-v",
|
||||
"-t",
|
||||
"nfs",
|
||||
"-o",
|
||||
"mountproto=tcp,mountport=2049,proto=tcp,port=2049,nfsvers=3,noacl",
|
||||
nfs_target,
|
||||
path,
|
||||
])
|
||||
.output()
|
||||
.await;
|
||||
|
||||
match mount {
|
||||
Ok(output) => {
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
if output.status.success() {
|
||||
tracing::info!(nfs_target, path, stdout = %stdout, "nfs: mount success");
|
||||
} else {
|
||||
tracing::error!(nfs_target, path, stderr = %stderr, "nfs: mount failed");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, nfs_target, path, "nfs: mount command failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_run_file(name: &str, value: &str) {
|
||||
let dir = std::path::Path::new(crate::config::WRENN_RUN_DIR);
|
||||
if let Err(e) = std::fs::create_dir_all(dir) {
|
||||
|
||||
@ -7,6 +7,7 @@ pub mod init;
|
||||
pub mod memory;
|
||||
pub mod metrics;
|
||||
pub mod snapshot;
|
||||
pub mod volumes;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@ -61,6 +62,8 @@ pub fn router(state: Arc<AppState>) -> Router {
|
||||
post(memory::post_memory_preload_cancel),
|
||||
)
|
||||
.route("/files", get(files::get_files).put(files::put_files))
|
||||
.route("/volumes/mount", post(volumes::post_mount))
|
||||
.route("/volumes/unmount", post(volumes::post_unmount))
|
||||
.layer(cors)
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
252
envd-rs/src/http/volumes.rs
Normal file
252
envd-rs/src/http/volumes.rs
Normal file
@ -0,0 +1,252 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct MountRequest {
|
||||
/// virtio-blk serial set by the host in the VM config; used to resolve the
|
||||
/// block device via /sys/block/*/serial regardless of enumeration order.
|
||||
pub serial: String,
|
||||
/// Guest path to mount the volume at (e.g. "/mnt/vol-...").
|
||||
pub mount_path: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct UnmountRequest {
|
||||
pub mount_path: String,
|
||||
}
|
||||
|
||||
fn json_error(status: StatusCode, msg: &str) -> Response {
|
||||
let body = serde_json::json!({ "code": status.as_u16(), "message": msg });
|
||||
(status, Json(body)).into_response()
|
||||
}
|
||||
|
||||
fn no_content() -> Response {
|
||||
(
|
||||
StatusCode::NO_CONTENT,
|
||||
[(header::CACHE_CONTROL, "no-store")],
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// POST /volumes/mount — called by the host agent after boot for each attached
|
||||
/// volume. Resolves the block device by its virtio-blk serial, formats it with
|
||||
/// ext4 only if it has no filesystem yet (existing data is never reformatted),
|
||||
/// then mounts it at the requested path.
|
||||
pub async fn post_mount(
|
||||
State(_state): State<Arc<AppState>>,
|
||||
Json(req): Json<MountRequest>,
|
||||
) -> Response {
|
||||
// Serials are host-generated hex tokens; reject anything else before it
|
||||
// reaches a sysfs comparison / command argument.
|
||||
if req.serial.is_empty() || !req.serial.chars().all(|c| c.is_ascii_alphanumeric()) {
|
||||
return json_error(StatusCode::BAD_REQUEST, "invalid volume serial");
|
||||
}
|
||||
if req.mount_path.is_empty() {
|
||||
return json_error(StatusCode::BAD_REQUEST, "mount_path is required");
|
||||
}
|
||||
|
||||
let device = match resolve_device_by_serial(&req.serial).await {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
return json_error(
|
||||
StatusCode::NOT_FOUND,
|
||||
&format!("no block device with serial {}", req.serial),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// blkid exit status: 0 = a filesystem/signature is present (leave it alone),
|
||||
// 2 = no recognized signature (format it), anything else = probe error.
|
||||
// Tracked so the permissions of an existing volume are never touched — only
|
||||
// a filesystem this call created gets opened up below.
|
||||
let formatted = match has_filesystem(&device).await {
|
||||
Ok(true) => {
|
||||
tracing::info!(device, "volume already has a filesystem; skipping mkfs");
|
||||
false
|
||||
}
|
||||
Ok(false) => {
|
||||
if let Err(e) = mkfs_ext4(&device).await {
|
||||
return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e);
|
||||
}
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
return json_error(StatusCode::INTERNAL_SERVER_ERROR, &e);
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = tokio::fs::create_dir_all(&req.mount_path).await {
|
||||
return json_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&format!("create mount dir: {e}"),
|
||||
);
|
||||
}
|
||||
|
||||
// No -t: let the kernel autodetect the filesystem, so a pre-existing
|
||||
// non-ext4 volume still mounts.
|
||||
match tokio::process::Command::new("mount")
|
||||
.arg(&device)
|
||||
.arg(&req.mount_path)
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
Ok(out) if out.status.success() => {
|
||||
if formatted {
|
||||
open_mount_root(&req.mount_path);
|
||||
}
|
||||
tracing::info!(device, mount_path = %req.mount_path, "volume mounted");
|
||||
no_content()
|
||||
}
|
||||
Ok(out) => {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
// Already mounted at the target is success for an idempotent retry.
|
||||
if stderr.contains("already mounted") {
|
||||
return no_content();
|
||||
}
|
||||
json_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&format!("mount failed: {}", stderr.trim()),
|
||||
)
|
||||
}
|
||||
Err(e) => json_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&format!("mount command failed: {e}"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// open_mount_root relaxes the permissions of a just-formatted volume so every
|
||||
/// user in the capsule can use it.
|
||||
///
|
||||
/// mkfs.ext4 leaves the filesystem root owned by root with mode 0755, which
|
||||
/// locks out every non-root user — including the template's default user, which
|
||||
/// is who the file API and exec run as. Without this a caller has to sudo to
|
||||
/// write to their own volume. 1777 is /tmp's mode: world-writable, with the
|
||||
/// sticky bit so one user cannot remove another's files.
|
||||
///
|
||||
/// Only ever called straight after a first-time format. Re-mounting an existing
|
||||
/// volume leaves its permissions exactly as its owner last set them.
|
||||
///
|
||||
/// Applied to the mounted root rather than the directory created before the
|
||||
/// mount — that one is hidden underneath and changing it would have no effect.
|
||||
///
|
||||
/// Best-effort: a failure here leaves a root-only volume that is still mounted
|
||||
/// and holds its data, which is not worth failing the whole capsule create over.
|
||||
fn open_mount_root(mount_path: &str) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
if let Err(e) = std::fs::set_permissions(mount_path, std::fs::Permissions::from_mode(0o1777)) {
|
||||
tracing::warn!(
|
||||
mount_path,
|
||||
error = %e,
|
||||
"failed to relax volume permissions; only root will be able to write to it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// POST /volumes/unmount — called best-effort before a graceful capsule destroy.
|
||||
/// Flushes buffered writes (sync) and unmounts so the backing file is
|
||||
/// consistent. Idempotent: unmounting an already-unmounted path is fine.
|
||||
pub async fn post_unmount(
|
||||
State(_state): State<Arc<AppState>>,
|
||||
Json(req): Json<UnmountRequest>,
|
||||
) -> Response {
|
||||
if req.mount_path.is_empty() {
|
||||
return json_error(StatusCode::BAD_REQUEST, "mount_path is required");
|
||||
}
|
||||
|
||||
// sync flushes all filesystems, guaranteeing the volume's data reaches its
|
||||
// backing file even if the umount below is skipped or fails.
|
||||
let _ = tokio::task::spawn_blocking(nix::unistd::sync).await;
|
||||
|
||||
let _ = tokio::process::Command::new("umount")
|
||||
.arg(&req.mount_path)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
no_content()
|
||||
}
|
||||
|
||||
/// resolve_device_by_serial finds the /dev path of the virtio-blk device whose
|
||||
/// serial matches. Polls briefly since a boot-time disk may not be fully
|
||||
/// enumerated the instant envd is asked to mount it.
|
||||
async fn resolve_device_by_serial(serial: &str) -> Option<String> {
|
||||
for _ in 0..50 {
|
||||
if let Some(dev) = find_block_by_serial(serial) {
|
||||
return Some(dev);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn find_block_by_serial(serial: &str) -> Option<String> {
|
||||
let entries = std::fs::read_dir("/sys/block").ok()?;
|
||||
for entry in entries.flatten() {
|
||||
let name = entry.file_name();
|
||||
let name = name.to_string_lossy();
|
||||
// Only virtio-blk devices carry a serial we set; skip loop/ram/etc.
|
||||
if !name.starts_with("vd") {
|
||||
continue;
|
||||
}
|
||||
let serial_path = format!("/sys/block/{name}/serial");
|
||||
if let Ok(s) = std::fs::read_to_string(&serial_path) {
|
||||
if s.trim() == serial {
|
||||
return Some(format!("/dev/{name}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// has_filesystem returns true when blkid detects any filesystem signature on
|
||||
/// the device. Guards mkfs so existing data is never reformatted.
|
||||
async fn has_filesystem(device: &str) -> Result<bool, String> {
|
||||
let out = tokio::process::Command::new("blkid")
|
||||
.arg(device)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("blkid spawn failed: {e}"))?;
|
||||
match out.status.code() {
|
||||
// 0 with output = a signature was printed. Guard on the output too so a
|
||||
// blkid variant that always exits 0 (e.g. busybox) is handled: empty
|
||||
// output means no filesystem.
|
||||
Some(0) => Ok(!String::from_utf8_lossy(&out.stdout).trim().is_empty()),
|
||||
// util-linux blkid: 2 = nothing detected.
|
||||
Some(2) => Ok(false),
|
||||
// 4 (usage) / 8 (error) / signal — treat as a probe failure.
|
||||
other => Err(format!(
|
||||
"blkid probe failed (exit {:?}): {}",
|
||||
other,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn mkfs_ext4(device: &str) -> Result<(), String> {
|
||||
// -F: the target is a whole virtio-blk device, not a partition; without it
|
||||
// mkfs.ext4 prompts and would hang. Safe here — only reached when blkid
|
||||
// found no existing signature.
|
||||
let out = tokio::process::Command::new("mkfs.ext4")
|
||||
.args(["-q", "-F", device])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("mkfs.ext4 spawn failed: {e}"))?;
|
||||
if out.status.success() {
|
||||
tracing::info!(device, "formatted volume with ext4");
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"mkfs.ext4 failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
))
|
||||
}
|
||||
}
|
||||
@ -10,7 +10,7 @@ source "$(cd "$(dirname "$0")" && pwd)/build-common.sh"
|
||||
# Alpine is musl-based: the static envd + static tini run fine. bash is added so
|
||||
# wrenn-user has a familiar login shell; wrenn-init itself only needs /bin/sh.
|
||||
PREP="set -e
|
||||
apk add --no-cache socat chrony sudo wget curl ca-certificates git iproute2 tini bash gcc make nano vim
|
||||
apk add --no-cache socat chrony sudo wget curl ca-certificates git iproute2 tini bash make nano vim e2fsprogs util-linux
|
||||
adduser -D wrenn-user
|
||||
${WRENN_SUDOERS_SETUP}"
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ source "$(cd "$(dirname "$0")" && pwd)/build-common.sh"
|
||||
# tini is AUR-only on Arch (not in core/extra), so it is not installed here —
|
||||
# rootfs-from-container.sh injects the static tini binary instead.
|
||||
PREP="set -e
|
||||
pacman -Sy --noconfirm --needed socat chrony sudo wget curl ca-certificates git iproute2 inetutils gcc make nano vim
|
||||
pacman -Sy --noconfirm --needed socat chrony sudo wget curl ca-certificates git iproute2 inetutils make nano vim e2fsprogs
|
||||
useradd -m -s /bin/bash wrenn-user
|
||||
${WRENN_SUDOERS_SETUP}
|
||||
pacman -Scc --noconfirm || true"
|
||||
|
||||
@ -11,7 +11,7 @@ source "$(cd "$(dirname "$0")" && pwd)/build-common.sh"
|
||||
PREP="set -e
|
||||
# install_weak_deps=False keeps the image lean. The guest never runs systemd:
|
||||
# PID 1 is wrenn-init -> tini -> envd.
|
||||
dnf install -y --setopt=install_weak_deps=False socat chrony sudo wget curl ca-certificates git iproute hostname tini gcc make nano vim
|
||||
dnf install -y --setopt=install_weak_deps=False socat chrony sudo wget curl ca-certificates git iproute hostname tini make nano vim e2fsprogs
|
||||
useradd -m -s /bin/bash wrenn-user
|
||||
${WRENN_SUDOERS_SETUP}
|
||||
dnf clean all"
|
||||
|
||||
@ -12,7 +12,7 @@ export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
# --no-install-recommends keeps the image lean (avoids pulling systemd-adjacent
|
||||
# recommends). The guest never runs systemd: PID 1 is wrenn-init -> tini -> envd.
|
||||
apt-get install -y --no-install-recommends socat chrony sudo wget curl ca-certificates git iproute2 hostname tini gcc make nano vim
|
||||
apt-get install -y --no-install-recommends socat chrony sudo wget curl ca-certificates git iproute2 hostname tini make nano vim e2fsprogs
|
||||
# Remove the stock 'ubuntu' user (uid 1000) shipped by the base image; it is
|
||||
# replaced by wrenn-user. Also drop its cloud-init sudoers drop-in.
|
||||
userdel -r ubuntu 2>/dev/null || true
|
||||
|
||||
@ -30,6 +30,10 @@ type createSandboxRequest struct {
|
||||
MemoryMB int32 `json:"memory_mb"`
|
||||
TimeoutSec int32 `json:"timeout_sec"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
// VolumeIDs are external storage volumes to attach at boot, each given as
|
||||
// a volume ID ("vol-...") or a name ("vl-cache"). Each is mounted at
|
||||
// /mnt/<volume-id> inside the guest.
|
||||
VolumeIDs []string `json:"volume_ids"`
|
||||
}
|
||||
|
||||
type sandboxResponse struct {
|
||||
@ -102,6 +106,7 @@ func (h *sandboxHandler) Create(w http.ResponseWriter, r *http.Request) {
|
||||
MemoryMB: req.MemoryMB,
|
||||
TimeoutSec: req.TimeoutSec,
|
||||
Metadata: req.Metadata,
|
||||
VolumeRefs: req.VolumeIDs,
|
||||
})
|
||||
h.audit.LogSandboxCreate(r.Context(), ac, sb.ID, req.Template, err)
|
||||
if err != nil {
|
||||
|
||||
166
internal/api/handlers_volumes.go
Normal file
166
internal/api/handlers_volumes.go
Normal file
@ -0,0 +1,166 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/units"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/apperr"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/audit"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/auth"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/db"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/id"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/service"
|
||||
)
|
||||
|
||||
type volumeHandler struct {
|
||||
svc *service.VolumeService
|
||||
audit *audit.AuditLogger
|
||||
}
|
||||
|
||||
func newVolumeHandler(svc *service.VolumeService, al *audit.AuditLogger) *volumeHandler {
|
||||
return &volumeHandler{svc: svc, audit: al}
|
||||
}
|
||||
|
||||
type createVolumeRequest struct {
|
||||
// Name is optional; omitted, the volume is named after its own ID.
|
||||
Name string `json:"name"`
|
||||
// Size is the human-readable form ("20Gi", "500M"). SizeMB is the plain
|
||||
// megabyte form. Exactly one is needed; Size wins if both are given.
|
||||
Size string `json:"size"`
|
||||
SizeMB int32 `json:"size_mb"`
|
||||
}
|
||||
|
||||
// sizeMB resolves the request's size into megabytes, accepting either field.
|
||||
func (r createVolumeRequest) sizeMB() (int32, error) {
|
||||
if s := strings.TrimSpace(r.Size); s != "" {
|
||||
mb, err := units.ParseSizeToMB(s)
|
||||
if err != nil {
|
||||
return 0, apperr.InvalidRequest.WrapMsg(err, "Invalid volume size.")
|
||||
}
|
||||
return int32(mb), nil
|
||||
}
|
||||
if r.SizeMB <= 0 {
|
||||
return 0, apperr.InvalidRequest.Msg(`A volume size is required (e.g. "size": "20Gi" or "size_mb": 20480).`)
|
||||
}
|
||||
return r.SizeMB, nil
|
||||
}
|
||||
|
||||
type volumeResponse struct {
|
||||
ID string `json:"id"`
|
||||
TeamID string `json:"team_id"`
|
||||
Name string `json:"name"`
|
||||
SizeMB int32 `json:"size_mb"`
|
||||
Status string `json:"status"`
|
||||
HostID *string `json:"host_id,omitempty"`
|
||||
SandboxID *string `json:"sandbox_id,omitempty"`
|
||||
// Always emitted, "" while detached — host_id/sandbox_id above are pointers
|
||||
// because they are genuinely nullable, but mount_path is not.
|
||||
MountPath string `json:"mount_path"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
LastAttachedAt *string `json:"last_attached_at,omitempty"`
|
||||
}
|
||||
|
||||
func volumeToResponse(v db.Volume) volumeResponse {
|
||||
resp := volumeResponse{
|
||||
ID: id.FormatVolumeID(v.ID),
|
||||
TeamID: id.FormatTeamID(v.TeamID),
|
||||
Name: v.Name,
|
||||
SizeMB: v.SizeMb,
|
||||
Status: v.Status,
|
||||
MountPath: v.MountPath,
|
||||
}
|
||||
if v.HostID.Valid {
|
||||
s := id.FormatHostID(v.HostID)
|
||||
resp.HostID = &s
|
||||
}
|
||||
if v.SandboxID.Valid {
|
||||
s := id.FormatSandboxID(v.SandboxID)
|
||||
resp.SandboxID = &s
|
||||
}
|
||||
if v.CreatedAt.Valid {
|
||||
resp.CreatedAt = v.CreatedAt.Time.Format(time.RFC3339)
|
||||
}
|
||||
if v.LastAttachedAt.Valid {
|
||||
s := v.LastAttachedAt.Time.Format(time.RFC3339)
|
||||
resp.LastAttachedAt = &s
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
// Create handles POST /v1/volumes.
|
||||
func (h *volumeHandler) Create(w http.ResponseWriter, r *http.Request) {
|
||||
ac := auth.MustFromContext(r.Context())
|
||||
|
||||
var req createVolumeRequest
|
||||
if err := decodeJSON(r, &req); err != nil {
|
||||
writeErr(w, r, apperr.InvalidRequest.WrapMsg(err, "Invalid JSON body."))
|
||||
return
|
||||
}
|
||||
|
||||
sizeMB, err := req.sizeMB()
|
||||
if err != nil {
|
||||
writeErr(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
vol, err := h.svc.Create(r.Context(), ac.TeamID, req.Name, sizeMB)
|
||||
if err != nil {
|
||||
writeErr(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
h.audit.LogVolumeCreate(r.Context(), ac, vol.ID, vol.Name, int(vol.SizeMb), nil)
|
||||
writeJSON(w, http.StatusCreated, volumeToResponse(vol))
|
||||
}
|
||||
|
||||
// List handles GET /v1/volumes.
|
||||
func (h *volumeHandler) List(w http.ResponseWriter, r *http.Request) {
|
||||
ac := auth.MustFromContext(r.Context())
|
||||
|
||||
vols, err := h.svc.List(r.Context(), ac.TeamID)
|
||||
if err != nil {
|
||||
writeErr(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
resp := make([]volumeResponse, len(vols))
|
||||
for i, v := range vols {
|
||||
resp[i] = volumeToResponse(v)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// Get handles GET /v1/volumes/{id}, where {id} is a volume ID or name.
|
||||
func (h *volumeHandler) Get(w http.ResponseWriter, r *http.Request) {
|
||||
ac := auth.MustFromContext(r.Context())
|
||||
|
||||
vol, err := h.svc.Resolve(r.Context(), ac.TeamID, chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
writeErr(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, volumeToResponse(vol))
|
||||
}
|
||||
|
||||
// Delete handles DELETE /v1/volumes/{id}, where {id} is a volume ID or name.
|
||||
func (h *volumeHandler) Delete(w http.ResponseWriter, r *http.Request) {
|
||||
ac := auth.MustFromContext(r.Context())
|
||||
|
||||
vol, err := h.svc.Resolve(r.Context(), ac.TeamID, chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
writeErr(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.svc.Delete(r.Context(), vol.ID, ac.TeamID); err != nil {
|
||||
writeErr(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
h.audit.LogVolumeDelete(r.Context(), ac, vol.ID, nil)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@ -65,6 +65,19 @@ func NewHostMonitor(queries *db.Queries, pool *lifecycle.HostClientPool, al *aud
|
||||
}
|
||||
}
|
||||
|
||||
// detachVolumes frees any volumes attached to sandboxes the monitor has just
|
||||
// moved to a terminal state, so a volume is never stranded on a capsule that
|
||||
// died via host loss or reconciliation rather than an explicit destroy.
|
||||
// Idempotent — only 'attached' rows change.
|
||||
func (m *HostMonitor) detachVolumes(ctx context.Context, sandboxIDs []pgtype.UUID) {
|
||||
if len(sandboxIDs) == 0 {
|
||||
return
|
||||
}
|
||||
if err := m.db.DetachVolumesBySandboxIDs(ctx, sandboxIDs); err != nil {
|
||||
slog.Warn("host monitor: failed to detach volumes for stopped sandboxes", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the monitor loop until the context is cancelled.
|
||||
func (m *HostMonitor) Start(ctx context.Context) {
|
||||
go func() {
|
||||
@ -229,6 +242,7 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
|
||||
}); err != nil {
|
||||
slog.Warn("host monitor: failed to stop missing sandboxes", "host_id", id.FormatHostID(host.ID), "error", err)
|
||||
} else {
|
||||
m.detachVolumes(ctx, ids)
|
||||
for _, sb := range toStop {
|
||||
m.audit.LogSandboxDestroySystem(ctx, sb.TeamID, sb.ID, "orphaned", nil)
|
||||
}
|
||||
@ -279,6 +293,7 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
|
||||
}); err != nil {
|
||||
slog.Warn("host monitor: failed to mark stopped", "host_id", id.FormatHostID(host.ID), "error", err)
|
||||
} else {
|
||||
m.detachVolumes(ctx, ids)
|
||||
for _, sb := range toStop {
|
||||
m.audit.LogSandboxDestroySystem(ctx, sb.TeamID, sb.ID, "orphaned", nil)
|
||||
}
|
||||
@ -378,6 +393,26 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
|
||||
slog.Info("host monitor: promoted transient sandbox to running", "sandbox_id", sbIDStr, "from", sb.Status)
|
||||
}
|
||||
}
|
||||
// A capsule the host still reports alive while the DB says
|
||||
// "stopping" means its destroy never landed — the control plane
|
||||
// exhausted its retries, or died mid-destroy. Nothing else re-sends
|
||||
// it, so without this the capsule sits in "stopping" forever and its
|
||||
// volumes stay claimed. Re-issue against confirmed-live host state;
|
||||
// the volumes are freed only once the host says it is gone.
|
||||
if sb.Status == "stopping" &&
|
||||
sb.LastUpdated.Valid && time.Since(sb.LastUpdated.Time) >= transientGracePeriod {
|
||||
slog.Info("host monitor: re-issuing destroy for capsule stuck stopping", "sandbox_id", sbIDStr)
|
||||
if _, err := agent.DestroySandbox(ctx, connect.NewRequest(&pb.DestroySandboxRequest{
|
||||
SandboxId: sbIDStr,
|
||||
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
|
||||
slog.Warn("host monitor: destroy retry failed", "sandbox_id", sbIDStr, "error", err)
|
||||
} else if _, err := m.db.UpdateSandboxStatusIf(ctx, db.UpdateSandboxStatusIfParams{
|
||||
ID: sb.ID, Status: "stopping", Status_2: "stopped",
|
||||
}); err == nil {
|
||||
m.detachVolumes(ctx, []pgtype.UUID{sb.ID})
|
||||
m.audit.LogSandboxDestroySystem(ctx, sb.TeamID, sb.ID, "destroy_retry", nil)
|
||||
}
|
||||
}
|
||||
// A snapshot keeps the source sandbox alive throughout, so an alive
|
||||
// sandbox does NOT mean the snapshot finished. Only recover it once
|
||||
// it has been stuck past the snapshot grace period (i.e. the CP
|
||||
@ -427,6 +462,11 @@ func (m *HostMonitor) checkHost(ctx context.Context, host db.Host) {
|
||||
ID: sb.ID, Status: fromStatus, Status_2: finalStatus,
|
||||
}); err == nil {
|
||||
slog.Info("host monitor: resolved transient sandbox", "sandbox_id", sbIDStr, "from", fromStatus, "to", finalStatus)
|
||||
// error/stopped are terminal — free any attached volumes. "paused"
|
||||
// (from pausing) is recoverable, so the volume stays attached.
|
||||
if finalStatus == "error" || finalStatus == "stopped" {
|
||||
m.detachVolumes(ctx, []pgtype.UUID{sb.ID})
|
||||
}
|
||||
inferredErr := errInferredTransientTimeout
|
||||
switch fromStatus {
|
||||
case "starting":
|
||||
|
||||
@ -782,7 +782,9 @@ paths:
|
||||
- sessionAuth: []
|
||||
description: |
|
||||
Owner only. Soft-deletes the team and destroys all running/paused/starting
|
||||
capsules. All DB records are preserved. The team slug is permanently reserved.
|
||||
capsules. Team-owned templates and storage volumes are deleted along with
|
||||
their data; capsule records are preserved. The team slug is permanently
|
||||
reserved.
|
||||
responses:
|
||||
"204":
|
||||
description: Team deleted
|
||||
@ -1582,6 +1584,113 @@ paths:
|
||||
"503":
|
||||
$ref: "#/components/responses/ServiceUnavailable"
|
||||
|
||||
/v1/volumes:
|
||||
post:
|
||||
summary: Create a storage volume
|
||||
operationId: createVolume
|
||||
tags: [volumes]
|
||||
security:
|
||||
- apiKeyAuth: []
|
||||
- sessionAuth: []
|
||||
description: |
|
||||
Create an external storage volume. The volume starts detached and is
|
||||
not placed on any host until it is first attached to a capsule (via
|
||||
`volume_ids` on capsule create), at which point it is pinned to that
|
||||
capsule's host for the rest of its life. Destroying a capsule only frees
|
||||
its volumes, never deletes them; the one case where a volume is removed
|
||||
without an explicit delete is deleting the team that owns it.
|
||||
|
||||
Every volume has a `vl-`-prefixed name that is unique within your team
|
||||
and can be used anywhere a volume ID is accepted. Omit `name` and the
|
||||
volume is named after its own ID.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/CreateVolumeRequest"
|
||||
responses:
|
||||
"201":
|
||||
description: Volume created
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Volume"
|
||||
"400":
|
||||
$ref: "#/components/responses/BadRequest"
|
||||
"409":
|
||||
description: A volume with this name already exists
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
get:
|
||||
summary: List storage volumes
|
||||
operationId: listVolumes
|
||||
tags: [volumes]
|
||||
security:
|
||||
- apiKeyAuth: []
|
||||
- sessionAuth: []
|
||||
responses:
|
||||
"200":
|
||||
description: List of volumes
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/Volume"
|
||||
|
||||
/v1/volumes/{id}:
|
||||
get:
|
||||
summary: Get a storage volume
|
||||
operationId: getVolume
|
||||
tags: [volumes]
|
||||
security:
|
||||
- apiKeyAuth: []
|
||||
- sessionAuth: []
|
||||
description: |
|
||||
Look up a volume by ID (`vol-...`) or by name (`vl-cache`, or plain
|
||||
`cache` — the `vl-` prefix is optional on input).
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/VolumeRef"
|
||||
responses:
|
||||
"200":
|
||||
description: The volume
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Volume"
|
||||
"400":
|
||||
$ref: "#/components/responses/BadRequest"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
delete:
|
||||
summary: Delete a storage volume
|
||||
operationId: deleteVolume
|
||||
tags: [volumes]
|
||||
security:
|
||||
- apiKeyAuth: []
|
||||
- sessionAuth: []
|
||||
description: |
|
||||
Delete a volume and its data. The volume must be detached — destroy any
|
||||
capsule using it first. Accepts an ID or a name.
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/VolumeRef"
|
||||
responses:
|
||||
"204":
|
||||
description: Volume deleted
|
||||
"400":
|
||||
$ref: "#/components/responses/BadRequest"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"409":
|
||||
description: Volume is attached to a capsule
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
|
||||
/v1/snapshots:
|
||||
post:
|
||||
summary: Create a snapshot template
|
||||
@ -3933,6 +4042,19 @@ paths:
|
||||
$ref: "#/components/responses/NotFound"
|
||||
|
||||
components:
|
||||
parameters:
|
||||
VolumeRef:
|
||||
name: id
|
||||
in: path
|
||||
required: true
|
||||
description: >
|
||||
A volume ID (`vol-<base36>`) or name (`vl-cache`). The `vl-` prefix is
|
||||
optional on input, so `cache` resolves the same volume as `vl-cache`.
|
||||
The two namespaces cannot collide — IDs always start with `vol-`.
|
||||
schema:
|
||||
type: string
|
||||
example: vl-cache
|
||||
|
||||
responses:
|
||||
BadRequest:
|
||||
description: Invalid request parameters
|
||||
@ -4132,6 +4254,98 @@ components:
|
||||
are at most 64 characters. Reserved system keys (kernel_version,
|
||||
vmm_version, agent_version, envd_version) are rejected. Metadata is
|
||||
set at create-time only.
|
||||
volume_ids:
|
||||
type: array
|
||||
items: { type: string }
|
||||
nullable: true
|
||||
maxItems: 4
|
||||
example: ["vl-cache", "vol-8f3kq2m1p7x0a9b4c6d8e2f5g"]
|
||||
description: >
|
||||
External storage volumes to attach at boot, each given as a volume
|
||||
ID ("vol-...") or a name ("vl-cache"). Each is mounted at
|
||||
/mnt/<volume-id> inside the guest, formatted on first use and
|
||||
writable by any user in the capsule. At most 4 per capsule. Volumes
|
||||
can only be attached at create time and must be detached and owned
|
||||
by your team. If a volume is already pinned to a host, the capsule is
|
||||
scheduled onto that host; attaching volumes pinned to different hosts
|
||||
fails. Not supported for capsules created from a snapshot template.
|
||||
|
||||
CreateVolumeRequest:
|
||||
type: object
|
||||
description: >
|
||||
Supply the size as either `size` or `size_mb`. `name` is optional.
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
maxLength: 40
|
||||
example: vl-cache
|
||||
description: >
|
||||
Name for the volume, unique within your team and usable anywhere a
|
||||
volume ID is accepted. Lowercase letters, numbers, and dashes only
|
||||
(no leading, trailing, or repeated dashes), at most 40 characters
|
||||
including the `vl-` prefix. The prefix is added if you omit it, so
|
||||
`cache` and `vl-cache` name the same volume. Omit the field entirely
|
||||
and the volume is named after its own ID.
|
||||
size:
|
||||
type: string
|
||||
example: 20Gi
|
||||
description: >
|
||||
Volume size with a G/Gi/M/Mi suffix; a bare number is read as MB.
|
||||
Fixed at creation. Takes precedence over `size_mb` if both are set.
|
||||
size_mb:
|
||||
type: integer
|
||||
minimum: 100
|
||||
example: 5120
|
||||
description: >
|
||||
Volume size in MB, as an alternative to `size`. Fixed at creation.
|
||||
The backing storage is sparse and grows on use, so the size is a
|
||||
ceiling rather than an upfront allocation. The maximum is set per
|
||||
deployment via `WRENN_MAX_VOLUME_SIZE` (default 20Gi); requesting
|
||||
more returns 400.
|
||||
|
||||
Volume:
|
||||
type: object
|
||||
required: [id, team_id, name, size_mb, status, mount_path, created_at]
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
description: Volume ID (e.g. "vol-...").
|
||||
team_id:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
description: >
|
||||
The volume's `vl-`-prefixed name, unique within the team. Accepted
|
||||
in place of the ID on any volume route and in `volume_ids`.
|
||||
example: vl-cache
|
||||
size_mb:
|
||||
type: integer
|
||||
status:
|
||||
type: string
|
||||
enum: [detached, attaching, attached, deleting]
|
||||
description: >
|
||||
`detached` — free to attach or delete. `attaching` — reserved by a
|
||||
capsule that is booting. `attached` — in use by a capsule.
|
||||
`deleting` — a delete is in flight; transient, and reverts to
|
||||
`detached` if the delete fails.
|
||||
host_id:
|
||||
type: string
|
||||
nullable: true
|
||||
description: Host the volume is pinned to; null until first attached.
|
||||
sandbox_id:
|
||||
type: string
|
||||
nullable: true
|
||||
description: Capsule the volume is currently attached to, if any.
|
||||
mount_path:
|
||||
type: string
|
||||
description: Guest mount path while attached (e.g. "/mnt/vol-...").
|
||||
created_at:
|
||||
type: string
|
||||
format: date-time
|
||||
last_attached_at:
|
||||
type: string
|
||||
format: date-time
|
||||
nullable: true
|
||||
|
||||
UsageResponse:
|
||||
type: object
|
||||
|
||||
@ -260,7 +260,20 @@ func (c *SandboxEventConsumer) handleAutoPaused(ctx context.Context, sandboxID p
|
||||
}
|
||||
}
|
||||
|
||||
// detachVolumes frees any volumes still attached to a sandbox that has reached
|
||||
// a terminal state. Idempotent (only 'attached' rows change) and keyed on
|
||||
// sandbox_id, so it runs safely from every terminal path — the explicit destroy
|
||||
// flow, the TTL-reaper fallback, and the crash/failure path — not just
|
||||
// SandboxService.Destroy.
|
||||
func (c *SandboxEventConsumer) detachVolumes(ctx context.Context, sandboxID pgtype.UUID) {
|
||||
if err := c.db.DetachVolumesBySandbox(ctx, sandboxID); err != nil {
|
||||
slog.Warn("sandbox event consumer: failed to detach volumes", "sandbox_id", id.FormatSandboxID(sandboxID), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *SandboxEventConsumer) handleStopped(ctx context.Context, sandboxID pgtype.UUID) {
|
||||
c.detachVolumes(ctx, sandboxID)
|
||||
|
||||
// stopping → stopped (CP-initiated destroy completed). No audit row here;
|
||||
// the handler that issued the destroy already wrote one.
|
||||
if _, err := c.db.UpdateSandboxStatusIf(ctx, db.UpdateSandboxStatusIfParams{
|
||||
@ -289,6 +302,8 @@ func (c *SandboxEventConsumer) handleStopped(ctx context.Context, sandboxID pgty
|
||||
// audit.Log writes the row only — it does NOT republish an event, which would
|
||||
// loop back into this consumer. Do not switch to LogSandboxCreateSystem here.
|
||||
func (c *SandboxEventConsumer) handleFailed(ctx context.Context, sandboxID pgtype.UUID, event events.Event) {
|
||||
c.detachVolumes(ctx, sandboxID)
|
||||
|
||||
for _, fromStatus := range []string{"running", "starting", "pausing", "resuming", "snapshotting"} {
|
||||
if _, err := c.db.UpdateSandboxStatusIf(ctx, db.UpdateSandboxStatusIfParams{
|
||||
ID: sandboxID, Status: fromStatus, Status_2: "error",
|
||||
|
||||
@ -82,7 +82,8 @@ func New(
|
||||
}
|
||||
|
||||
// Shared service layer.
|
||||
sandboxSvc := &service.SandboxService{DB: queries, Pool: pool, Scheduler: sched}
|
||||
volumeSvc := &service.VolumeService{DB: queries, Pool: pool, MaxSizeMB: int32(sctx.Config.MaxVolumeSizeMB)}
|
||||
sandboxSvc := &service.SandboxService{DB: queries, Pool: pool, Scheduler: sched, Volumes: volumeSvc}
|
||||
sandboxSvc.PublishEvent = func(ctx context.Context, event service.SandboxStateEvent) {
|
||||
if evt, ok := serviceEventToCanonical(event); ok {
|
||||
// State-change events are ephemeral UI signals — mirror them to the
|
||||
@ -116,6 +117,7 @@ func New(
|
||||
authH := newAuthHandler(queries, pgPool, sessionSvc, mailer, rdb, oauthRedirectURL, authHooks)
|
||||
oauthH := newOAuthHandler(queries, pgPool, jwtSecret, sessionSvc, oauthRegistry, oauthRedirectURL, authHooks)
|
||||
apiKeys := newAPIKeyHandler(apiKeySvc, al)
|
||||
volumes := newVolumeHandler(volumeSvc, al)
|
||||
hostH := newHostHandler(hostSvc, queries, al, monitor)
|
||||
teamH := newTeamHandler(teamSvc, al, mailer, sessionSvc)
|
||||
usersH := newUsersHandler(queries, userSvc, al, sessionSvc)
|
||||
@ -278,6 +280,16 @@ func New(
|
||||
r.Delete("/{name}", snapshots.Delete)
|
||||
})
|
||||
|
||||
// External storage volumes: API key (SDK) or session (browser).
|
||||
r.Route("/v1/volumes", func(r chi.Router) {
|
||||
r.Use(requireSessionOrAPIKey(queries, sessionSvc))
|
||||
r.Use(csrf)
|
||||
r.Post("/", volumes.Create)
|
||||
r.Get("/", volumes.List)
|
||||
r.Get("/{id}", volumes.Get)
|
||||
r.Delete("/{id}", volumes.Delete)
|
||||
})
|
||||
|
||||
// Host management.
|
||||
r.Route("/v1/hosts", func(r chi.Router) {
|
||||
// Unauthenticated: one-time registration token.
|
||||
@ -480,6 +492,14 @@ func serviceEventToCanonical(e service.SandboxStateEvent) (events.Event, bool) {
|
||||
case "sandbox.stopped":
|
||||
eventType = events.CapsuleDestroy
|
||||
outcome = events.OutcomeSuccess
|
||||
case "sandbox.destroy_failed":
|
||||
// The destroy RPC never reached the host, so the capsule stays in
|
||||
// "stopping" holding its volumes until the host monitor re-issues it.
|
||||
// This is the only lifecycle path that reaches no terminal state of its
|
||||
// own, so the event is what tells anyone it is outstanding.
|
||||
eventType = events.CapsuleDestroy
|
||||
outcome = events.OutcomeError
|
||||
metadata = map[string]string{"reason": "destroy_failed"}
|
||||
case "sandbox.pause_failed":
|
||||
// reason must be non-empty or channels.isRedundantSystemFollowup
|
||||
// filters this system-actor event out of webhook delivery.
|
||||
|
||||
72
internal/api/volume_reaper.go
Normal file
72
internal/api/volume_reaper.go
Normal file
@ -0,0 +1,72 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/db"
|
||||
)
|
||||
|
||||
// VolumeReaper periodically frees volumes stuck in a transient state
|
||||
// (attaching/deleting) whose owning operation died before it could ever reach a
|
||||
// host — a control-plane crash between reserving a volume and inserting the
|
||||
// sandbox row, or a delete that crashed mid-flight. It is the safety net for
|
||||
// the one volume-lifecycle window that no synchronous path can clean up.
|
||||
//
|
||||
// It deliberately only touches rows with no sandbox_id. Once a reservation
|
||||
// names its capsule, it may correspond to a VM that really did boot with the
|
||||
// volume mounted, and no timer can tell the difference — freeing it would let a
|
||||
// second capsule attach the same backing file and corrupt it. Those rows are
|
||||
// resolved by the host monitor instead, which first confirms against the host
|
||||
// that the capsule is gone.
|
||||
//
|
||||
// A volume is released only once it has been stuck longer than staleAfter, which
|
||||
// MUST exceed the capsule create timeout so an in-flight attach is never freed
|
||||
// out from under a booting capsule.
|
||||
type VolumeReaper struct {
|
||||
db *db.Queries
|
||||
interval time.Duration
|
||||
staleAfter time.Duration
|
||||
}
|
||||
|
||||
// NewVolumeReaper creates a VolumeReaper that sweeps every interval and releases
|
||||
// volumes stuck in a transient state for longer than staleAfter.
|
||||
func NewVolumeReaper(queries *db.Queries, interval, staleAfter time.Duration) *VolumeReaper {
|
||||
return &VolumeReaper{db: queries, interval: interval, staleAfter: staleAfter}
|
||||
}
|
||||
|
||||
// Start runs the reaper loop until the context is cancelled.
|
||||
func (r *VolumeReaper) Start(ctx context.Context) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(r.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Run immediately on startup so a crash-orphaned reservation is cleared
|
||||
// without waiting a full interval.
|
||||
r.run(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.run(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (r *VolumeReaper) run(ctx context.Context) {
|
||||
cutoff := pgtype.Timestamptz{Time: time.Now().Add(-r.staleAfter), Valid: true}
|
||||
n, err := r.db.ReleaseStaleVolumeReservations(ctx, cutoff)
|
||||
if err != nil {
|
||||
slog.Warn("volume reaper: failed to release stale reservations", "error", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
slog.Info("volume reaper: released stale volume reservations", "count", n)
|
||||
}
|
||||
}
|
||||
@ -78,12 +78,17 @@ func (s *Server) CreateSandbox(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
volumes, err := volumeSpecsFromProto(msg.Volumes)
|
||||
if err != nil {
|
||||
return nil, apperr.ToConnect(apperr.InvalidRequest.Wrap(err))
|
||||
}
|
||||
|
||||
// disk_size_mb in the request is deprecated and never set by the control
|
||||
// plane; passing 0 lets the manager pick the size (DefaultRootfsSizeMB,
|
||||
// floored at the origin image size).
|
||||
sb, diskSizeBytes, err := s.mgr.Create(ctx, msg.SandboxId, teamID, templateID,
|
||||
int(msg.Vcpus), int(msg.MemoryMb), int(msg.TimeoutSec), 0,
|
||||
msg.DefaultUser, msg.DefaultEnv)
|
||||
msg.DefaultUser, msg.DefaultEnv, volumes)
|
||||
if err != nil {
|
||||
return nil, mapSandboxError(fmt.Errorf("create sandbox: %w", err))
|
||||
}
|
||||
@ -182,6 +187,51 @@ func (s *Server) FlattenRootfs(
|
||||
}), nil
|
||||
}
|
||||
|
||||
// DeleteVolume removes a detached volume's backing file from this host.
|
||||
func (s *Server) DeleteVolume(
|
||||
_ context.Context,
|
||||
req *connect.Request[pb.DeleteVolumeRequest],
|
||||
) (*connect.Response[pb.DeleteVolumeResponse], error) {
|
||||
teamID, err := parseUUIDString(req.Msg.TeamId)
|
||||
if err != nil {
|
||||
return nil, apperr.ToConnect(apperr.InvalidRequest.Wrap(err))
|
||||
}
|
||||
volumeID, err := parseUUIDString(req.Msg.VolumeId)
|
||||
if err != nil {
|
||||
return nil, apperr.ToConnect(apperr.InvalidRequest.Wrap(err))
|
||||
}
|
||||
if err := s.mgr.DeleteVolumeFile(teamID, volumeID); err != nil {
|
||||
return nil, mapSandboxError(err)
|
||||
}
|
||||
return connect.NewResponse(&pb.DeleteVolumeResponse{}), nil
|
||||
}
|
||||
|
||||
// volumeSpecsFromProto converts the wire VolumeSpecs on a create request into
|
||||
// the sandbox layer's attach specs, parsing the hex UUID fields.
|
||||
func volumeSpecsFromProto(pbVols []*pb.VolumeSpec) ([]sandbox.VolumeAttachSpec, error) {
|
||||
if len(pbVols) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
specs := make([]sandbox.VolumeAttachSpec, 0, len(pbVols))
|
||||
for _, v := range pbVols {
|
||||
volumeID, err := parseUUIDString(v.VolumeId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("volume id: %w", err)
|
||||
}
|
||||
teamID, err := parseUUIDString(v.TeamId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("volume team id: %w", err)
|
||||
}
|
||||
specs = append(specs, sandbox.VolumeAttachSpec{
|
||||
VolumeID: volumeID,
|
||||
TeamID: teamID,
|
||||
SizeMB: int(v.SizeMb),
|
||||
MountPath: v.MountPath,
|
||||
})
|
||||
}
|
||||
return specs, nil
|
||||
}
|
||||
|
||||
// mapSandboxError translates sandbox.Manager errors to Connect errors
|
||||
// carrying an apperr ErrorInfo detail, so the control plane surfaces the
|
||||
// precise catalog error instead of guessing from the Connect code. Sentinels
|
||||
@ -204,6 +254,10 @@ func mapSandboxError(err error) error {
|
||||
return apperr.ToConnect(apperr.TemplateNotFound.WrapMsg(err, "The template's rootfs image is not available on this host."))
|
||||
case errors.Is(err, sandbox.ErrEnvdNotReady):
|
||||
return apperr.ToConnect(apperr.SandboxUnresponsive.WrapMsg(err, "The sandbox VM started but its agent did not become ready."))
|
||||
case errors.Is(err, sandbox.ErrVolumesAttached):
|
||||
return apperr.ToConnect(apperr.VolumesAttached.Wrap(err))
|
||||
case errors.Is(err, sandbox.ErrVolumesOnSnapshotTemplate):
|
||||
return apperr.ToConnect(apperr.InvalidRequest.WrapMsg(err, "Volumes can only be attached to capsules created from a base template, not a snapshot template."))
|
||||
case errors.Is(err, network.ErrNoFreeSlots):
|
||||
return apperr.ToConnect(apperr.CapacityUnavailable.WrapMsg(err, "This host has no free network slots. Try again shortly."))
|
||||
default:
|
||||
|
||||
72
internal/units/size.go
Normal file
72
internal/units/size.go
Normal file
@ -0,0 +1,72 @@
|
||||
// Package units parses the human-readable size strings used across wrenn's
|
||||
// configuration and API surfaces (e.g. WRENN_DEFAULT_ROOTFS_SIZE=5Gi, a
|
||||
// volume's "size": "20Gi"). It deliberately holds no dependencies so both the
|
||||
// control plane and the host runtime can share one definition of what "5G"
|
||||
// means.
|
||||
package units
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// maxSizeMB bounds a parsed size at what fits in an int32 megabyte count (~2
|
||||
// PiB). Sizes are stored and carried as int32 throughout — DB columns, proto
|
||||
// fields, request bodies — so a larger value could only reach them by silently
|
||||
// truncating, turning an absurd request into a plausible-looking small volume.
|
||||
// Reject it at the parse instead.
|
||||
const maxSizeMB = 1<<31 - 1
|
||||
|
||||
// ParseSizeToMB parses a human-readable size string into megabytes.
|
||||
// Supported suffixes: G, Gi (gibibytes), M, Mi (mebibytes). A bare number is
|
||||
// read as megabytes.
|
||||
// Examples: "5G" → 5120, "2Gi" → 2048, "1000M" → 1000, "512Mi" → 512.
|
||||
func ParseSizeToMB(s string) (int, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, fmt.Errorf("empty size string")
|
||||
}
|
||||
|
||||
// Find where the numeric part ends.
|
||||
i := 0
|
||||
for i < len(s) && (s[i] == '.' || (s[i] >= '0' && s[i] <= '9')) {
|
||||
i++
|
||||
}
|
||||
if i == 0 {
|
||||
return 0, fmt.Errorf("invalid size %q: no numeric value", s)
|
||||
}
|
||||
|
||||
numStr := s[:i]
|
||||
suffix := strings.TrimSpace(s[i:])
|
||||
|
||||
num, err := strconv.ParseFloat(numStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size %q: %w", s, err)
|
||||
}
|
||||
|
||||
var mb float64
|
||||
switch suffix {
|
||||
case "G", "Gi":
|
||||
mb = num * 1024
|
||||
case "M", "Mi", "":
|
||||
mb = num
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid size %q: unknown suffix %q (use G, Gi, M, or Mi)", s, suffix)
|
||||
}
|
||||
|
||||
// Compare as float: converting first would already have truncated.
|
||||
if mb > maxSizeMB {
|
||||
return 0, fmt.Errorf("invalid size %q: too large (max %d MB)", s, maxSizeMB)
|
||||
}
|
||||
return int(mb), nil
|
||||
}
|
||||
|
||||
// FormatMB renders a megabyte count back into the compact human form used in
|
||||
// error messages, preferring whole gibibytes when the value divides evenly.
|
||||
func FormatMB(mb int) string {
|
||||
if mb > 0 && mb%1024 == 0 {
|
||||
return strconv.Itoa(mb/1024) + "Gi"
|
||||
}
|
||||
return strconv.Itoa(mb) + "Mi"
|
||||
}
|
||||
81
internal/units/size_test.go
Normal file
81
internal/units/size_test.go
Normal file
@ -0,0 +1,81 @@
|
||||
package units
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseSizeToMB(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want int
|
||||
wantErr bool
|
||||
}{
|
||||
{"gibibytes", "5G", 5120, false},
|
||||
{"gibibytes-explicit", "2Gi", 2048, false},
|
||||
{"megabytes", "1000M", 1000, false},
|
||||
{"mebibytes", "512Mi", 512, false},
|
||||
{"bare-number-is-mb", "2048", 2048, false},
|
||||
{"fractional", "1.5G", 1536, false},
|
||||
{"trims-space", " 20Gi ", 20480, false},
|
||||
{"default-volume-cap", "20Gi", 20 * 1024, false},
|
||||
|
||||
{"empty", "", 0, true},
|
||||
{"whitespace-only", " ", 0, true},
|
||||
{"no-number", "Gi", 0, true},
|
||||
{"negative", "-5G", 0, true},
|
||||
{"unknown-suffix", "5T", 0, true},
|
||||
{"unknown-suffix-bytes", "500B", 0, true},
|
||||
|
||||
// Sizes are int32 megabytes everywhere downstream. Without an explicit
|
||||
// bound these truncate: 4194305Gi wraps to exactly 1024 MB, which would
|
||||
// sail past both the minimum and maximum checks as a 1 GiB volume.
|
||||
{"overflows-int32", "4194305Gi", 0, true},
|
||||
{"overflows-int32-exactly", "4194304Gi", 0, true},
|
||||
{"huge", "999999999Gi", 0, true},
|
||||
{"largest-accepted", "2147483647", 1<<31 - 1, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := ParseSizeToMB(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("ParseSizeToMB(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("ParseSizeToMB(%q) = %d, want %d", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatMB(t *testing.T) {
|
||||
tests := []struct {
|
||||
input int
|
||||
want string
|
||||
}{
|
||||
{20 * 1024, "20Gi"},
|
||||
{1024, "1Gi"},
|
||||
{5120, "5Gi"},
|
||||
{100, "100Mi"},
|
||||
{1500, "1500Mi"},
|
||||
{0, "0Mi"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := FormatMB(tt.input); got != tt.want {
|
||||
t.Errorf("FormatMB(%d) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FormatMB's output must feed back into ParseSizeToMB unchanged, so an error
|
||||
// message quoting a limit can be pasted straight into a request or env var.
|
||||
func TestFormatMBRoundTrips(t *testing.T) {
|
||||
for _, mb := range []int{100, 1024, 5120, 20 * 1024, 1500} {
|
||||
got, err := ParseSizeToMB(FormatMB(mb))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseSizeToMB(FormatMB(%d)) unexpected error: %v", mb, err)
|
||||
}
|
||||
if got != mb {
|
||||
t.Errorf("round trip of %d MB produced %d", mb, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -214,6 +214,31 @@ var (
|
||||
Status: http.StatusNotFound,
|
||||
Message: "User not found.",
|
||||
})
|
||||
VolumeNotFound = register(Def{
|
||||
Code: "volume_not_found",
|
||||
Status: http.StatusNotFound,
|
||||
Message: "Volume not found.",
|
||||
})
|
||||
VolumeInUse = register(Def{
|
||||
Code: "volume_in_use",
|
||||
Status: http.StatusConflict,
|
||||
Message: "Volume is attached to a capsule. Destroy the capsule before deleting the volume.",
|
||||
})
|
||||
VolumeHostMismatch = register(Def{
|
||||
Code: "volume_host_mismatch",
|
||||
Status: http.StatusConflict,
|
||||
Message: "This volume is pinned to a different host than the requested capsule can run on.",
|
||||
})
|
||||
VolumesAttached = register(Def{
|
||||
Code: "volumes_attached",
|
||||
Status: http.StatusConflict,
|
||||
Message: "Detach all volumes before creating a template from this capsule.",
|
||||
})
|
||||
VolumeNameTaken = register(Def{
|
||||
Code: "volume_name_taken",
|
||||
Status: http.StatusConflict,
|
||||
Message: "A volume with this name already exists.",
|
||||
})
|
||||
)
|
||||
|
||||
// Lookup returns the catalog Def for a code, or false if unregistered.
|
||||
|
||||
@ -452,6 +452,38 @@ func (l *AuditLogger) LogSnapshotDeleteSystem(ctx context.Context, teamID pgtype
|
||||
})
|
||||
}
|
||||
|
||||
// --- Volume events (scope: team) ---
|
||||
|
||||
// LogVolumeCreate records the creation of an external storage volume.
|
||||
func (l *AuditLogger) LogVolumeCreate(ctx context.Context, ac auth.AuthContext, volumeID pgtype.UUID, name string, sizeMB int, err error) {
|
||||
meta := map[string]any{"name": name, "size_mb": sizeMB}
|
||||
l.Log(ctx, newEntry(ac, ac.TeamID, "team", "volume", id.FormatVolumeID(volumeID), "create", auditStatusFor(err, "success"), mergeMeta(meta, err)))
|
||||
l.publish(ctx, events.Event{
|
||||
Event: events.VolumeCreate,
|
||||
Outcome: outcomeFromErr(err),
|
||||
Timestamp: events.Now(),
|
||||
TeamID: id.FormatTeamID(ac.TeamID),
|
||||
Actor: actorToEvent(ac),
|
||||
Resource: events.Resource{ID: id.FormatVolumeID(volumeID), Type: "volume"},
|
||||
Metadata: map[string]string{"name": name},
|
||||
Error: errString(err),
|
||||
})
|
||||
}
|
||||
|
||||
// LogVolumeDelete records the deletion of an external storage volume.
|
||||
func (l *AuditLogger) LogVolumeDelete(ctx context.Context, ac auth.AuthContext, volumeID pgtype.UUID, err error) {
|
||||
l.Log(ctx, newEntry(ac, ac.TeamID, "team", "volume", id.FormatVolumeID(volumeID), "delete", auditStatusFor(err, "warning"), mergeMeta(nil, err)))
|
||||
l.publish(ctx, events.Event{
|
||||
Event: events.VolumeDelete,
|
||||
Outcome: outcomeFromErr(err),
|
||||
Timestamp: events.Now(),
|
||||
TeamID: id.FormatTeamID(ac.TeamID),
|
||||
Actor: actorToEvent(ac),
|
||||
Resource: events.Resource{ID: id.FormatVolumeID(volumeID), Type: "volume"},
|
||||
Error: errString(err),
|
||||
})
|
||||
}
|
||||
|
||||
// --- Team events (scope: team) ---
|
||||
|
||||
func (l *AuditLogger) LogTeamRename(ctx context.Context, ac auth.AuthContext, teamID pgtype.UUID, oldName, newName string) {
|
||||
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"strconv"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/units"
|
||||
)
|
||||
|
||||
// Config holds the control plane configuration.
|
||||
@ -36,6 +38,14 @@ type Config struct {
|
||||
// deployments that legitimately deliver to internal endpoints.
|
||||
ChannelsAllowPrivateTargets bool // WRENN_CHANNELS_ALLOW_PRIVATE
|
||||
|
||||
// MaxVolumeSizeMB caps how large a single external storage volume may be.
|
||||
// WRENN_MAX_VOLUME_SIZE accepts the same human-readable form as
|
||||
// WRENN_DEFAULT_ROOTFS_SIZE (e.g. "20Gi", "500G", "2048M"). Raise it on
|
||||
// self-hosted deployments with the disk to back it — a volume is a sparse
|
||||
// file, so the cap bounds worst-case growth rather than upfront allocation.
|
||||
// 0 means unset; service.DefaultMaxVolumeSizeMB (20Gi) then applies.
|
||||
MaxVolumeSizeMB int // WRENN_MAX_VOLUME_SIZE
|
||||
|
||||
// SMTP — transactional email. All fields optional; omitting SMTPHost disables email.
|
||||
SMTPHost string // SMTP_HOST
|
||||
SMTPPort int // SMTP_PORT (default 587)
|
||||
@ -69,6 +79,10 @@ func Load() Config {
|
||||
|
||||
ChannelsAllowPrivateTargets: envOrDefaultBool("WRENN_CHANNELS_ALLOW_PRIVATE", false),
|
||||
|
||||
// 0 means "unset" — service.VolumeService owns the default so the
|
||||
// limit is not declared in two places that can drift apart.
|
||||
MaxVolumeSizeMB: envOrDefaultSizeMB("WRENN_MAX_VOLUME_SIZE", 0),
|
||||
|
||||
SMTPHost: os.Getenv("SMTP_HOST"),
|
||||
SMTPPort: envOrDefaultInt("SMTP_PORT", 587),
|
||||
SMTPUsername: os.Getenv("SMTP_USERNAME"),
|
||||
@ -105,6 +119,21 @@ func envOrDefaultInt(key string, def int) int {
|
||||
return n
|
||||
}
|
||||
|
||||
// envOrDefaultSizeMB reads a human-readable size (e.g. "20Gi", "500G", "2048M")
|
||||
// and returns it in megabytes, falling back to def when unset or unparseable —
|
||||
// matching the fail-soft behaviour of the other envOrDefault helpers.
|
||||
func envOrDefaultSizeMB(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
mb, err := units.ParseSizeToMB(v)
|
||||
if err != nil || mb <= 0 {
|
||||
return def
|
||||
}
|
||||
return mb
|
||||
}
|
||||
|
||||
func envOrDefaultBool(key string, def bool) bool {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
|
||||
@ -296,6 +296,12 @@ func Run(opts ...Option) {
|
||||
rollup := api.NewDailyUsageRollup(queries, time.Hour)
|
||||
rollup.Start(ctx)
|
||||
|
||||
// Reap volumes stuck in a transient attaching/deleting state after a crash.
|
||||
// The 30m grace safely exceeds the 10m capsule create timeout, so an
|
||||
// in-flight attach is never freed out from under a booting capsule.
|
||||
volumeReaper := api.NewVolumeReaper(queries, 10*time.Minute, 30*time.Minute)
|
||||
volumeReaper.Start(ctx)
|
||||
|
||||
// Start extension background workers.
|
||||
for _, ext := range o.extensions {
|
||||
for _, worker := range ext.BackgroundWorkers(sctx) {
|
||||
|
||||
@ -239,3 +239,17 @@ type UsersTeam struct {
|
||||
Role string `json:"role"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
}
|
||||
|
||||
type Volume struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
TeamID pgtype.UUID `json:"team_id"`
|
||||
HostID pgtype.UUID `json:"host_id"`
|
||||
Name string `json:"name"`
|
||||
SizeMb int32 `json:"size_mb"`
|
||||
Status string `json:"status"`
|
||||
SandboxID pgtype.UUID `json:"sandbox_id"`
|
||||
MountPath string `json:"mount_path"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
LastAttachedAt pgtype.Timestamptz `json:"last_attached_at"`
|
||||
LastUpdated pgtype.Timestamptz `json:"last_updated"`
|
||||
}
|
||||
|
||||
511
pkg/db/volumes.sql.go
Normal file
511
pkg/db/volumes.sql.go
Normal file
@ -0,0 +1,511 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.30.0
|
||||
// source: volumes.sql
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const abortVolumeDelete = `-- name: AbortVolumeDelete :exec
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'deleting'
|
||||
`
|
||||
|
||||
// Revert a delete claim back to detached when the host-side file removal fails,
|
||||
// so the volume stays usable rather than stuck in 'deleting'.
|
||||
func (q *Queries) AbortVolumeDelete(ctx context.Context, id pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, abortVolumeDelete, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const beginVolumeDelete = `-- name: BeginVolumeDelete :one
|
||||
UPDATE volumes
|
||||
SET status = 'deleting',
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND team_id = $2 AND status IN ('detached', 'deleting')
|
||||
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
|
||||
`
|
||||
|
||||
type BeginVolumeDeleteParams struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
TeamID pgtype.UUID `json:"team_id"`
|
||||
}
|
||||
|
||||
// Atomically claim a volume for deletion. The CAS blocks a concurrent attach
|
||||
// (which needs status='detached') so a volume can never be attached and deleted
|
||||
// at the same time. 'deleting' is also accepted so a delete that crashed
|
||||
// mid-flight can be retried rather than leaving the volume permanently stuck.
|
||||
// Returns the row (for host_id) when claimed; no row means it is not deletable
|
||||
// (attached/attaching, wrong team, or missing).
|
||||
func (q *Queries) BeginVolumeDelete(ctx context.Context, arg BeginVolumeDeleteParams) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, beginVolumeDelete, arg.ID, arg.TeamID)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const countVolumesByHost = `-- name: CountVolumesByHost :one
|
||||
SELECT COUNT(*) FROM volumes WHERE host_id = $1
|
||||
`
|
||||
|
||||
// Guard used before a host can be removed: a host holding pinned volumes must
|
||||
// not be deleted out from under them.
|
||||
func (q *Queries) CountVolumesByHost(ctx context.Context, hostID pgtype.UUID) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, countVolumesByHost, hostID)
|
||||
var count int64
|
||||
err := row.Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
const deleteVolumeRow = `-- name: DeleteVolumeRow :exec
|
||||
DELETE FROM volumes WHERE id = $1 AND team_id = $2 AND status = 'deleting'
|
||||
`
|
||||
|
||||
type DeleteVolumeRowParams struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
TeamID pgtype.UUID `json:"team_id"`
|
||||
}
|
||||
|
||||
// Drop the row only while the delete claim still holds. Without the status
|
||||
// guard a delete that lost its claim mid-flight (reaped, or aborted and then
|
||||
// re-attached by a racing capsule create) would erase the record of a volume
|
||||
// that is once again in use.
|
||||
func (q *Queries) DeleteVolumeRow(ctx context.Context, arg DeleteVolumeRowParams) error {
|
||||
_, err := q.db.Exec(ctx, deleteVolumeRow, arg.ID, arg.TeamID)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteVolumesByTeam = `-- name: DeleteVolumesByTeam :exec
|
||||
DELETE FROM volumes WHERE team_id = $1
|
||||
`
|
||||
|
||||
// Drop every volume record belonging to a team being deleted. This is the one
|
||||
// place volumes are removed without an explicit per-volume delete: the team that
|
||||
// owns them is going away, so there is no longer anyone who could reference,
|
||||
// re-attach, or delete them. Unlike DeleteVolumeRow there is no status guard —
|
||||
// the team's capsules have already been destroyed, so no volume can still be
|
||||
// legitimately in use, and a row left behind would be unreachable forever (and
|
||||
// would keep blocking deletion of the host it is pinned to).
|
||||
func (q *Queries) DeleteVolumesByTeam(ctx context.Context, teamID pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, deleteVolumesByTeam, teamID)
|
||||
return err
|
||||
}
|
||||
|
||||
const detachVolumesByHost = `-- name: DetachVolumesByHost :exec
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
host_id = NULL,
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE host_id = $1
|
||||
`
|
||||
|
||||
// Free and un-pin every volume on a host being force-deleted. The host (and its
|
||||
// backing files) are going away, so the volumes are reset to a fresh detached,
|
||||
// un-pinned state — the rows survive (never auto-deleted) and can be re-attached
|
||||
// on a new host. Also required before DeleteHost so the host_id FK does not
|
||||
// block deletion.
|
||||
func (q *Queries) DetachVolumesByHost(ctx context.Context, hostID pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, detachVolumesByHost, hostID)
|
||||
return err
|
||||
}
|
||||
|
||||
const detachVolumesBySandbox = `-- name: DetachVolumesBySandbox :exec
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE sandbox_id = $1 AND status IN ('attached', 'attaching')
|
||||
`
|
||||
|
||||
// Terminal sweep run when a capsule reaches a terminal state (destroyed,
|
||||
// stopped, errored, or reaped): free every volume it held back to detached
|
||||
// (data and host pin preserved) so it can be reused or deleted. Keyed on
|
||||
// sandbox_id, which is stamped on at reservation time. Never deletes the
|
||||
// volume — removal is always explicit.
|
||||
//
|
||||
// 'attaching' is swept alongside 'attached' so a reservation whose promotion
|
||||
// never landed (lost RPC response, DB blip) is freed here — on confirmed host
|
||||
// state — rather than on a blind timer.
|
||||
func (q *Queries) DetachVolumesBySandbox(ctx context.Context, sandboxID pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, detachVolumesBySandbox, sandboxID)
|
||||
return err
|
||||
}
|
||||
|
||||
const detachVolumesBySandboxIDs = `-- name: DetachVolumesBySandboxIDs :exec
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE sandbox_id = ANY($1::uuid[]) AND status IN ('attached', 'attaching')
|
||||
`
|
||||
|
||||
// Bulk variant of DetachVolumesBySandbox for the host monitor, which stops
|
||||
// many sandboxes at once (e.g. after a host goes offline).
|
||||
func (q *Queries) DetachVolumesBySandboxIDs(ctx context.Context, dollar_1 []pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, detachVolumesBySandboxIDs, dollar_1)
|
||||
return err
|
||||
}
|
||||
|
||||
const getVolume = `-- name: GetVolume :one
|
||||
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetVolume(ctx context.Context, id pgtype.UUID) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, getVolume, id)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getVolumeByTeam = `-- name: GetVolumeByTeam :one
|
||||
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE id = $1 AND team_id = $2
|
||||
`
|
||||
|
||||
type GetVolumeByTeamParams struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
TeamID pgtype.UUID `json:"team_id"`
|
||||
}
|
||||
|
||||
func (q *Queries) GetVolumeByTeam(ctx context.Context, arg GetVolumeByTeamParams) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, getVolumeByTeam, arg.ID, arg.TeamID)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getVolumeByTeamAndName = `-- name: GetVolumeByTeamAndName :one
|
||||
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE team_id = $1 AND name = $2
|
||||
`
|
||||
|
||||
type GetVolumeByTeamAndNameParams struct {
|
||||
TeamID pgtype.UUID `json:"team_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Resolve a volume by its user-facing name. Names are unique per team, so this
|
||||
// is the name-based counterpart to GetVolumeByTeam and is what lets the API
|
||||
// accept "vl-cache" wherever it accepts "vol-<id>".
|
||||
func (q *Queries) GetVolumeByTeamAndName(ctx context.Context, arg GetVolumeByTeamAndNameParams) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, getVolumeByTeamAndName, arg.TeamID, arg.Name)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const insertVolume = `-- name: InsertVolume :one
|
||||
INSERT INTO volumes (id, team_id, name, size_mb, status)
|
||||
VALUES ($1, $2, $3, $4, 'detached')
|
||||
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
|
||||
`
|
||||
|
||||
type InsertVolumeParams struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
TeamID pgtype.UUID `json:"team_id"`
|
||||
Name string `json:"name"`
|
||||
SizeMb int32 `json:"size_mb"`
|
||||
}
|
||||
|
||||
func (q *Queries) InsertVolume(ctx context.Context, arg InsertVolumeParams) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, insertVolume,
|
||||
arg.ID,
|
||||
arg.TeamID,
|
||||
arg.Name,
|
||||
arg.SizeMb,
|
||||
)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const linkVolumeReservation = `-- name: LinkVolumeReservation :exec
|
||||
UPDATE volumes
|
||||
SET sandbox_id = $2,
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'attaching'
|
||||
`
|
||||
|
||||
type LinkVolumeReservationParams struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
SandboxID pgtype.UUID `json:"sandbox_id"`
|
||||
}
|
||||
|
||||
// Stamp the owning sandbox onto a reservation as soon as the sandbox row
|
||||
// exists (ReserveVolumeForAttach runs before the insert, so it cannot set the
|
||||
// FK itself). This makes every in-flight reservation attributable to a capsule,
|
||||
// which is what lets the host monitor decide — against live host state —
|
||||
// whether a stuck 'attaching' row is safe to free.
|
||||
func (q *Queries) LinkVolumeReservation(ctx context.Context, arg LinkVolumeReservationParams) error {
|
||||
_, err := q.db.Exec(ctx, linkVolumeReservation, arg.ID, arg.SandboxID)
|
||||
return err
|
||||
}
|
||||
|
||||
const listVolumesBySandbox = `-- name: ListVolumesBySandbox :many
|
||||
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE sandbox_id = $1 ORDER BY created_at DESC
|
||||
`
|
||||
|
||||
func (q *Queries) ListVolumesBySandbox(ctx context.Context, sandboxID pgtype.UUID) ([]Volume, error) {
|
||||
rows, err := q.db.Query(ctx, listVolumesBySandbox, sandboxID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Volume
|
||||
for rows.Next() {
|
||||
var i Volume
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listVolumesByTeam = `-- name: ListVolumesByTeam :many
|
||||
SELECT id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated FROM volumes WHERE team_id = $1 ORDER BY created_at DESC
|
||||
`
|
||||
|
||||
func (q *Queries) ListVolumesByTeam(ctx context.Context, teamID pgtype.UUID) ([]Volume, error) {
|
||||
rows, err := q.db.Query(ctx, listVolumesByTeam, teamID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Volume
|
||||
for rows.Next() {
|
||||
var i Volume
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markVolumeAttached = `-- name: MarkVolumeAttached :one
|
||||
UPDATE volumes
|
||||
SET status = 'attached',
|
||||
host_id = COALESCE(host_id, $2),
|
||||
sandbox_id = $3,
|
||||
mount_path = $4,
|
||||
last_attached_at = NOW(),
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'attaching'
|
||||
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
|
||||
`
|
||||
|
||||
type MarkVolumeAttachedParams struct {
|
||||
ID pgtype.UUID `json:"id"`
|
||||
HostID pgtype.UUID `json:"host_id"`
|
||||
SandboxID pgtype.UUID `json:"sandbox_id"`
|
||||
MountPath string `json:"mount_path"`
|
||||
}
|
||||
|
||||
// Promote a reserved volume to attached once the capsule has booted with it.
|
||||
// Records the owning sandbox and pins the volume to the sandbox's host on
|
||||
// first-ever attach (COALESCE keeps an existing pin for a re-attached volume).
|
||||
func (q *Queries) MarkVolumeAttached(ctx context.Context, arg MarkVolumeAttachedParams) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, markVolumeAttached,
|
||||
arg.ID,
|
||||
arg.HostID,
|
||||
arg.SandboxID,
|
||||
arg.MountPath,
|
||||
)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const releaseStaleVolumeReservations = `-- name: ReleaseStaleVolumeReservations :execrows
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
mount_path = '',
|
||||
last_updated = NOW()
|
||||
WHERE status IN ('attaching', 'deleting')
|
||||
AND sandbox_id IS NULL
|
||||
AND last_updated < $1
|
||||
`
|
||||
|
||||
// Free volumes stuck in a transient state whose owning operation died before it
|
||||
// could ever reach a host — the control plane crashed between reserving a
|
||||
// volume and inserting the sandbox row, or mid-delete. Run periodically by the
|
||||
// volume reaper.
|
||||
//
|
||||
// Deliberately limited to unattributed rows (sandbox_id IS NULL). A reservation
|
||||
// that already carries a sandbox_id may correspond to a capsule that really did
|
||||
// boot with the volume mounted — and a timer cannot tell the difference. Those
|
||||
// are freed only by the host-monitor sweep, which first confirms against the
|
||||
// host that the capsule is gone. Freeing a live volume here would let a second
|
||||
// capsule attach the same backing file and corrupt it.
|
||||
//
|
||||
// The cutoff ($1 = now - grace) MUST exceed the capsule create timeout so a
|
||||
// legitimately in-flight attach is never freed out from under a booting capsule.
|
||||
func (q *Queries) ReleaseStaleVolumeReservations(ctx context.Context, lastUpdated pgtype.Timestamptz) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, releaseStaleVolumeReservations, lastUpdated)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const releaseVolumeReservation = `-- name: ReleaseVolumeReservation :exec
|
||||
UPDATE volumes
|
||||
SET status = 'detached',
|
||||
sandbox_id = NULL,
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'attaching'
|
||||
`
|
||||
|
||||
// Roll a reserved volume back to detached when the capsule create fails.
|
||||
// host_id is left untouched (a failed create never pins a fresh volume, since
|
||||
// MarkVolumeAttached is what sets the pin). Only call this once the host has
|
||||
// confirmed the capsule is not running — a volume freed while a VM still has
|
||||
// its backing file open can be re-attached elsewhere and corrupted.
|
||||
func (q *Queries) ReleaseVolumeReservation(ctx context.Context, id pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, releaseVolumeReservation, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const reserveVolumeForAttach = `-- name: ReserveVolumeForAttach :one
|
||||
UPDATE volumes
|
||||
SET status = 'attaching',
|
||||
last_updated = NOW()
|
||||
WHERE id = $1 AND status = 'detached'
|
||||
RETURNING id, team_id, host_id, name, size_mb, status, sandbox_id, mount_path, created_at, last_attached_at, last_updated
|
||||
`
|
||||
|
||||
// Atomically claim a detached volume for a capsule about to boot. The CAS on
|
||||
// status = 'detached' enforces single-attach: a second capsule create racing
|
||||
// for the same volume finds no row and fails. sandbox_id is deliberately NOT
|
||||
// set here — the sandbox row does not exist yet, so setting the FK would fail.
|
||||
// It is set in MarkVolumeAttached once the capsule has booted.
|
||||
func (q *Queries) ReserveVolumeForAttach(ctx context.Context, id pgtype.UUID) (Volume, error) {
|
||||
row := q.db.QueryRow(ctx, reserveVolumeForAttach, id)
|
||||
var i Volume
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.TeamID,
|
||||
&i.HostID,
|
||||
&i.Name,
|
||||
&i.SizeMb,
|
||||
&i.Status,
|
||||
&i.SandboxID,
|
||||
&i.MountPath,
|
||||
&i.CreatedAt,
|
||||
&i.LastAttachedAt,
|
||||
&i.LastUpdated,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@ -360,6 +360,62 @@ func (c *Client) PrepareSnapshot(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// MountVolume asks envd to mount a data volume inside the guest. The volume's
|
||||
// block device is resolved by its virtio-blk serial (set in the VM config), so
|
||||
// this is robust to device-enumeration order. envd formats the device with
|
||||
// ext4 only if it has no filesystem yet (existing data is never reformatted),
|
||||
// then mounts it at mountPath.
|
||||
func (c *Client) MountVolume(ctx context.Context, serial, mountPath string) error {
|
||||
payload, err := json.Marshal(map[string]string{"serial": serial, "mount_path": mountPath})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal mount body: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/volumes/mount", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mount volume: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, MaxEnvdControlBytes))
|
||||
return fmt.Errorf("mount volume: status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnmountVolume asks envd to flush (sync) and unmount a data volume. Used
|
||||
// best-effort before a graceful capsule destroy so buffered writes reach the
|
||||
// backing file. Safe to call even if the path is not mounted.
|
||||
func (c *Client) UnmountVolume(ctx context.Context, mountPath string) error {
|
||||
payload, err := json.Marshal(map[string]string{"mount_path": mountPath})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal unmount body: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/volumes/unmount", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unmount volume: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, MaxEnvdControlBytes))
|
||||
return fmt.Errorf("unmount volume: status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MemoryPreloadStatus mirrors envd's /memory/preload response.
|
||||
//
|
||||
// State values: "idle", "running", "done", "failed", "cancelled".
|
||||
|
||||
@ -90,6 +90,12 @@ const (
|
||||
SnapshotCreate = "template.snapshot.create"
|
||||
SnapshotDelete = "template.snapshot.delete"
|
||||
|
||||
// Durable, subscribable. Volume lifecycle. Attach/detach are not separate
|
||||
// events — a volume attaches only as part of capsule.create and frees on
|
||||
// capsule.destroy.
|
||||
VolumeCreate = "volume.create"
|
||||
VolumeDelete = "volume.delete"
|
||||
|
||||
// Durable, no outcome (binary by name).
|
||||
HostUp = "host.up"
|
||||
HostDown = "host.down"
|
||||
@ -108,6 +114,8 @@ var SubscribableEventTypes = []string{
|
||||
CapsuleDestroy,
|
||||
SnapshotCreate,
|
||||
SnapshotDelete,
|
||||
VolumeCreate,
|
||||
VolumeDelete,
|
||||
HostUp,
|
||||
HostDown,
|
||||
}
|
||||
|
||||
20
pkg/id/id.go
20
pkg/id/id.go
@ -39,6 +39,7 @@ func NewAdminPermissionID() pgtype.UUID { return newUUID() }
|
||||
func NewChannelID() pgtype.UUID { return newUUID() }
|
||||
|
||||
func NewTemplateID() pgtype.UUID { return newUUID() }
|
||||
func NewVolumeID() pgtype.UUID { return newUUID() }
|
||||
|
||||
// NewSnapshotName generates a snapshot name: "template-" + 8 hex chars.
|
||||
func NewSnapshotName() string {
|
||||
@ -78,6 +79,7 @@ const (
|
||||
PrefixBuild = "bld-"
|
||||
PrefixAdminPermission = "perm-"
|
||||
PrefixChannel = "ch-"
|
||||
PrefixVolume = "vol-"
|
||||
)
|
||||
|
||||
// UUIDToBase36 encodes 16 UUID bytes as a 25-char base36 string (0-9a-z).
|
||||
@ -127,6 +129,7 @@ func FormatRefreshTokenID(id pgtype.UUID) string { return formatUUID(PrefixRefre
|
||||
func FormatAuditLogID(id pgtype.UUID) string { return formatUUID(PrefixAuditLog, id) }
|
||||
func FormatBuildID(id pgtype.UUID) string { return formatUUID(PrefixBuild, id) }
|
||||
func FormatChannelID(id pgtype.UUID) string { return formatUUID(PrefixChannel, id) }
|
||||
func FormatVolumeID(id pgtype.UUID) string { return formatUUID(PrefixVolume, id) }
|
||||
|
||||
// --- Parsing (prefixed string from API/RPC input → pgtype.UUID) ---
|
||||
|
||||
@ -150,6 +153,23 @@ func ParseHostTokenID(s string) (pgtype.UUID, error) { return parseUUID(PrefixHo
|
||||
func ParseAuditLogID(s string) (pgtype.UUID, error) { return parseUUID(PrefixAuditLog, s) }
|
||||
func ParseBuildID(s string) (pgtype.UUID, error) { return parseUUID(PrefixBuild, s) }
|
||||
func ParseChannelID(s string) (pgtype.UUID, error) { return parseUUID(PrefixChannel, s) }
|
||||
func ParseVolumeID(s string) (pgtype.UUID, error) { return parseUUID(PrefixVolume, s) }
|
||||
|
||||
// DefaultVolumeMountPath returns the guest path a volume is mounted at when no
|
||||
// explicit path is requested. Single source of truth shared by the control
|
||||
// plane (which records it) and the host agent (which mounts there).
|
||||
func DefaultVolumeMountPath(v pgtype.UUID) string {
|
||||
return "/mnt/" + FormatVolumeID(v)
|
||||
}
|
||||
|
||||
// VolumeSerial returns a 16-hex-char (64-bit) token derived from a volume ID.
|
||||
// It is set as the virtio-blk disk serial in the VM config so the guest can
|
||||
// resolve the block device via /sys/block/*/serial regardless of enumeration
|
||||
// order, without relying on positional /dev/vdX guessing. Kept well under the
|
||||
// virtio-blk serial cap (VIRTIO_BLK_ID_BYTES = 20).
|
||||
func VolumeSerial(v pgtype.UUID) string {
|
||||
return hex.EncodeToString(v.Bytes[:8])
|
||||
}
|
||||
|
||||
// --- Well-known IDs ---
|
||||
|
||||
|
||||
@ -100,6 +100,51 @@ func TestMaxUUID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeIDRoundTrip(t *testing.T) {
|
||||
id := NewVolumeID()
|
||||
formatted := FormatVolumeID(id)
|
||||
if formatted[:len(PrefixVolume)] != PrefixVolume {
|
||||
t.Fatalf("expected %s prefix, got %s", PrefixVolume, formatted)
|
||||
}
|
||||
parsed, err := ParseVolumeID(formatted)
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed != id {
|
||||
t.Fatalf("round-trip failed: %v → %s → %v", id, formatted, parsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeSerial(t *testing.T) {
|
||||
// The virtio-blk serial (VIRTIO_BLK_ID_BYTES) caps at 20 bytes; the guest
|
||||
// resolves the device by this value, so it must always fit.
|
||||
for i := 0; i < 1000; i++ {
|
||||
s := VolumeSerial(NewVolumeID())
|
||||
if len(s) > 20 {
|
||||
t.Fatalf("serial %q exceeds 20 bytes (%d)", s, len(s))
|
||||
}
|
||||
if s == "" {
|
||||
t.Fatal("serial must not be empty")
|
||||
}
|
||||
}
|
||||
// Deterministic: the same volume yields the same serial (required so a
|
||||
// resumed capsule rebuilds the identical disk symlink).
|
||||
id := NewVolumeID()
|
||||
first := VolumeSerial(id)
|
||||
second := VolumeSerial(id)
|
||||
if first != second {
|
||||
t.Fatalf("VolumeSerial must be deterministic: %q != %q", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultVolumeMountPath(t *testing.T) {
|
||||
id := NewVolumeID()
|
||||
want := "/mnt/" + FormatVolumeID(id)
|
||||
if got := DefaultVolumeMountPath(id); got != want {
|
||||
t.Fatalf("DefaultVolumeMountPath = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFormatSandboxID(b *testing.B) {
|
||||
id := pgtype.UUID{Bytes: uuid.New(), Valid: true}
|
||||
b.ResetTimer()
|
||||
|
||||
@ -169,6 +169,24 @@ func compareSemver(a, b string) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// VolumeDir returns the on-disk directory for an external storage volume,
|
||||
// mirroring the templates tree but under its own volumes/ root so it can never
|
||||
// be touched by sandbox teardown (which only ever removes SandboxDir):
|
||||
//
|
||||
// {wrennDir}/volumes/teams/{base36(teamID)}/{base36(volumeID)}
|
||||
func VolumeDir(wrennDir string, teamID, volumeID pgtype.UUID) string {
|
||||
return filepath.Join(wrennDir, "volumes", "teams",
|
||||
id.UUIDToBase36(teamID.Bytes),
|
||||
id.UUIDToBase36(volumeID.Bytes))
|
||||
}
|
||||
|
||||
// VolumeDataPath returns the path to a volume's backing image — a plain sparse
|
||||
// file (not a dm-snapshot CoW), formatted with ext4 inside the guest on first
|
||||
// use and handed to Cloud Hypervisor as a Raw disk.
|
||||
func VolumeDataPath(wrennDir string, teamID, volumeID pgtype.UUID) string {
|
||||
return filepath.Join(VolumeDir(wrennDir, teamID, volumeID), "data.img")
|
||||
}
|
||||
|
||||
// SlotsDir returns the directory holding network slot claim files. Each
|
||||
// allocated slot index is an empty file named after the index, created with
|
||||
// O_EXCL so concurrent processes can never claim the same slot.
|
||||
|
||||
@ -6,11 +6,10 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/units"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/id"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/layout"
|
||||
)
|
||||
@ -69,37 +68,13 @@ func EnsureImageSizes(wrennDir string, targetMB int) error {
|
||||
// ParseSizeToMB parses a human-readable size string into megabytes.
|
||||
// Supported suffixes: G, Gi (gibibytes), M, Mi (mebibytes).
|
||||
// Examples: "5G" → 5120, "2Gi" → 2048, "1000M" → 1000, "512Mi" → 512.
|
||||
//
|
||||
// The implementation lives in internal/units so the control plane can share it
|
||||
// without importing the host runtime. That package is unimportable from outside
|
||||
// this module, so this stays the entry point for external consumers of the
|
||||
// sandbox runtime (e.g. the wr CLI).
|
||||
func ParseSizeToMB(s string) (int, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, fmt.Errorf("empty size string")
|
||||
}
|
||||
|
||||
// Find where the numeric part ends.
|
||||
i := 0
|
||||
for i < len(s) && (s[i] == '.' || (s[i] >= '0' && s[i] <= '9')) {
|
||||
i++
|
||||
}
|
||||
if i == 0 {
|
||||
return 0, fmt.Errorf("invalid size %q: no numeric value", s)
|
||||
}
|
||||
|
||||
numStr := s[:i]
|
||||
suffix := strings.TrimSpace(s[i:])
|
||||
|
||||
num, err := strconv.ParseFloat(numStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size %q: %w", s, err)
|
||||
}
|
||||
|
||||
switch suffix {
|
||||
case "G", "Gi":
|
||||
return int(num * 1024), nil
|
||||
case "M", "Mi", "":
|
||||
return int(num), nil
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid size %q: unknown suffix %q (use G, Gi, M, or Mi)", s, suffix)
|
||||
}
|
||||
return units.ParseSizeToMB(s)
|
||||
}
|
||||
|
||||
// ShrinkSystemImages shrinks the built-in system base rootfs images back to
|
||||
|
||||
@ -44,6 +44,13 @@ var (
|
||||
// ErrEnvdNotReady is returned when the VM booted (or resumed) but envd did
|
||||
// not become healthy within the readiness budget.
|
||||
ErrEnvdNotReady = errors.New("envd not ready")
|
||||
// ErrVolumesAttached is returned when an operation (e.g. creating a template
|
||||
// from a capsule) is refused because the sandbox still has volumes attached.
|
||||
ErrVolumesAttached = errors.New("sandbox has volumes attached")
|
||||
// ErrVolumesOnSnapshotTemplate is returned when a create requests volumes on
|
||||
// a sandbox launched from a snapshot template — CH --restore reconstructs
|
||||
// only the snapshot's device set, so a new boot-time disk cannot be added.
|
||||
ErrVolumesOnSnapshotTemplate = errors.New("volumes are not supported when creating from a snapshot template")
|
||||
)
|
||||
|
||||
// MinTimeoutSec is the minimum inactivity TTL accepted by Create/Resume.
|
||||
@ -209,6 +216,12 @@ type sandboxState struct {
|
||||
dmDevice *devicemapper.SnapshotDevice
|
||||
baseImagePath string // path to the base template rootfs (for loop registry release)
|
||||
|
||||
// volumes holds the external storage volumes attached to this sandbox at
|
||||
// boot. Read under m.mu, mutated only while holding lifecycleMu (like
|
||||
// dmDevice). Persisted into runningState + snapshotMeta so re-attach and
|
||||
// resume can rebuild each disk symlink.
|
||||
volumes []*attachedVolume
|
||||
|
||||
// sandboxDirOverride, when non-empty, pins this sandbox's VMConfig.SandboxDir
|
||||
// to a path other than the default vm.SandboxTmpDir(sb.ID). Set when the
|
||||
// sandbox was launched from a snapshot template — CH's saved config.json
|
||||
@ -326,6 +339,7 @@ func (m *Manager) Create(
|
||||
vcpus, memoryMB, timeoutSec, diskSizeMB int,
|
||||
defaultUser string,
|
||||
defaultEnv map[string]string,
|
||||
volumes []VolumeAttachSpec,
|
||||
) (*models.Sandbox, int64, error) {
|
||||
if m.draining.Load() {
|
||||
return nil, 0, ErrDraining
|
||||
@ -385,6 +399,12 @@ func (m *Manager) Create(
|
||||
// the restore path with a confusing error downstream.
|
||||
templateDir := layout.TemplateDir(m.cfg.WrennDir, teamID, templateID)
|
||||
if !layout.IsSystemTemplate(teamID, templateID) && layout.IsSnapshotTemplate(templateDir) {
|
||||
// A snapshot-template launch restores CH's saved device topology via
|
||||
// --restore; a boot-time data disk cannot be added on top of it, and we
|
||||
// no longer hot-plug. Refuse rather than silently dropping the volumes.
|
||||
if len(volumes) > 0 {
|
||||
return nil, 0, ErrVolumesOnSnapshotTemplate
|
||||
}
|
||||
return m.createFromSnapshotTemplate(ctx, sandboxID, teamID, templateID,
|
||||
vcpus, memoryMB, timeoutSec, diskSizeMB, defaultUser, defaultEnv)
|
||||
}
|
||||
@ -449,6 +469,16 @@ func (m *Manager) Create(
|
||||
}
|
||||
res.slot = slot
|
||||
|
||||
// Provision data volumes (sparse backing files) before boot so they can be
|
||||
// baked into vm.create as extra Raw disks. The files persist across the VM
|
||||
// lifecycle; rollback below tears down the VM but deliberately leaves volume
|
||||
// data intact (volumes are owned by their own lifecycle, never auto-deleted).
|
||||
volumeDisks, attachedVols, err := m.prepareVolumeDisks(volumes)
|
||||
if err != nil {
|
||||
res.rollback()
|
||||
return nil, 0, fmt.Errorf("provision volumes: %w", err)
|
||||
}
|
||||
|
||||
// Boot VM — CH gets the dm device path.
|
||||
vmCfg := vm.VMConfig{
|
||||
SandboxID: sandboxID,
|
||||
@ -465,6 +495,7 @@ func (m *Manager) Create(
|
||||
NetMask: slot.GuestNetMask,
|
||||
VMMBin: m.cfg.VMMBin,
|
||||
LogDir: filepath.Join(m.cfg.WrennDir, "logs"),
|
||||
Volumes: volumeDisks,
|
||||
}
|
||||
|
||||
if _, err := m.vm.Create(ctx, vmCfg); err != nil {
|
||||
@ -496,6 +527,14 @@ func (m *Manager) Create(
|
||||
}
|
||||
initCancel()
|
||||
|
||||
// Format (if empty) and mount each attached volume. A mount failure fails
|
||||
// the whole create — a capsule that was asked for a volume must not come up
|
||||
// without it.
|
||||
if err := m.mountVolumes(ctx, client, attachedVols); err != nil {
|
||||
res.rollback()
|
||||
return nil, 0, fmt.Errorf("mount volumes: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
sb := &sandboxState{
|
||||
Sandbox: models.Sandbox{
|
||||
@ -517,6 +556,7 @@ func (m *Manager) Create(
|
||||
connTracker: &ConnTracker{},
|
||||
dmDevice: dmDev,
|
||||
baseImagePath: baseRootfs,
|
||||
volumes: attachedVols,
|
||||
}
|
||||
sb.client.Store(client)
|
||||
|
||||
@ -619,6 +659,10 @@ func (m *Manager) cleanup(ctx context.Context, sb *sandboxState) {
|
||||
}
|
||||
}
|
||||
m.stopSampler(sb)
|
||||
// Flush volumes before killing the VM so buffered guest writes land in the
|
||||
// backing files. Best-effort and only meaningful for a running sandbox
|
||||
// (paused ones were already synced at pause time and have no live envd).
|
||||
m.unmountVolumesBestEffort(sb)
|
||||
if err := m.vm.Destroy(ctx, sb.ID); err != nil {
|
||||
slog.Warn("vm destroy error", "id", sb.ID, "error", err)
|
||||
}
|
||||
|
||||
@ -103,6 +103,11 @@ type snapshotMeta struct {
|
||||
// forward verbatim through the chain.
|
||||
SandboxDir string `json:"sandbox_dir"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Volumes records the data volumes attached at pause time so Resume can
|
||||
// rebuild each disk symlink and CH's restored config.json paths resolve.
|
||||
// Only ever set for pause snapshots — snapshot templates are volume-free
|
||||
// (CreateSnapshot refuses to run while volumes are attached).
|
||||
Volumes []*attachedVolume `json:"volumes,omitempty"`
|
||||
}
|
||||
|
||||
// effectiveSandboxDir returns the tmpfs SandboxDir the running VM uses — the
|
||||
@ -269,6 +274,7 @@ func (m *Manager) Pause(ctx context.Context, sandboxID string) error {
|
||||
CowPath: sb.dmDevice.CowPath,
|
||||
SandboxDir: effectiveSandboxDir(sb),
|
||||
CreatedAt: time.Now(),
|
||||
Volumes: sb.volumes,
|
||||
}
|
||||
if err := writeSnapshotMeta(stageDir, meta); err != nil {
|
||||
// Without meta, Resume cannot reconstruct the sandbox. Treat as fatal.
|
||||
@ -665,6 +671,12 @@ func (m *Manager) resumeFromMeta(ctx context.Context, sb *sandboxState, meta *sn
|
||||
// keeps its original ID/SandboxDir so the disk path baked into
|
||||
// config.json (`/tmp/ch-vm-{originalID}/rootfs.ext4`) resolves to the
|
||||
// re-attached dm device via the tmpfs symlink set up by the launcher.
|
||||
// The attached-volume records from the snapshot meta drive the restore
|
||||
// launch: re-create each disk symlink (CH's saved config.json references
|
||||
// them) so the guest mount — already present in the restored memory image —
|
||||
// resolves to the same device.
|
||||
restoredVols := meta.Volumes
|
||||
|
||||
vmCfg := m.buildRestoreVMConfig(restoreInputs{
|
||||
sandboxID: sb.ID,
|
||||
templateID: id.UUIDString(pgtype.UUID{Bytes: sb.TemplateID, Valid: true}),
|
||||
@ -674,6 +686,7 @@ func (m *Manager) resumeFromMeta(ctx context.Context, sb *sandboxState, meta *sn
|
||||
memoryMB: meta.MemoryMB,
|
||||
slot: slot,
|
||||
sandboxDir: meta.SandboxDir,
|
||||
volumes: volumeDisksFor(restoredVols),
|
||||
})
|
||||
client, err := m.launchRestoredVM(ctx, vmCfg, slot.HostIP.String())
|
||||
if err != nil {
|
||||
@ -697,6 +710,7 @@ func (m *Manager) resumeFromMeta(ctx context.Context, sb *sandboxState, meta *sn
|
||||
// it empty so a Destroy-before-Resume cannot underflow the registry.
|
||||
sb.baseImagePath = meta.BaseTemplate
|
||||
sb.lazyRestore = true
|
||||
sb.volumes = restoredVols
|
||||
sb.connTracker.Reset()
|
||||
sb.HostIP = slot.HostIP
|
||||
sb.RootfsPath = dmDev.DevicePath
|
||||
@ -860,6 +874,15 @@ func (m *Manager) CreateSnapshot(ctx context.Context, sandboxID string, teamID,
|
||||
sb.lifecycleMu.Lock()
|
||||
defer sb.lifecycleMu.Unlock()
|
||||
|
||||
// A template must be volume-free: CH's snapshot config would otherwise
|
||||
// reference the volume disks, producing a template that cannot launch on a
|
||||
// fresh sandbox. Refuse while any volume is attached. (For a paused sandbox
|
||||
// re-attached across an agent restart sb.volumes may be empty; that case is
|
||||
// caught again from the on-disk meta in snapshotPausedToTemplate.)
|
||||
if len(sb.volumes) > 0 {
|
||||
return 0, fmt.Errorf("%w", ErrVolumesAttached)
|
||||
}
|
||||
|
||||
// Refuse silent overwrites: every snapshot must land in a fresh
|
||||
// templateID. Defends against caller bugs and concurrent CreateSnapshot
|
||||
// races for the same destination. User-facing snapshot-name uniqueness
|
||||
@ -1001,6 +1024,11 @@ func (m *Manager) snapshotPausedToTemplate(ctx context.Context, sb *sandboxState
|
||||
|
||||
snapDir := layout.PauseSnapshotDir(m.cfg.WrennDir, sb.ID)
|
||||
meta, err := readSnapshotMeta(snapDir)
|
||||
if err == nil && len(meta.Volumes) > 0 {
|
||||
// A sandbox paused with volumes attached carries them in its snapshot;
|
||||
// promoting it to a template would bake in dangling disk references.
|
||||
return 0, fmt.Errorf("%w", ErrVolumesAttached)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("load pause snapshot meta: %w", err)
|
||||
}
|
||||
@ -1199,6 +1227,11 @@ func (m *Manager) FlattenRootfs(ctx context.Context, sandboxID string, teamID, t
|
||||
return 0, fmt.Errorf("%w: %s (status: %s)", ErrNotRunning, sandboxID, sb.Status)
|
||||
}
|
||||
|
||||
// A flattened image template must be volume-free (see CreateSnapshot).
|
||||
if len(sb.volumes) > 0 {
|
||||
return 0, fmt.Errorf("%w", ErrVolumesAttached)
|
||||
}
|
||||
|
||||
dstDir := layout.TemplateDir(m.cfg.WrennDir, teamID, templateID)
|
||||
stageDir := filepath.Join(layout.SandboxesDir(m.cfg.WrennDir),
|
||||
fmt.Sprintf(".stage-%s-%d", sandboxID, time.Now().UnixNano()))
|
||||
|
||||
@ -31,7 +31,8 @@ type restoreInputs struct {
|
||||
vcpus int
|
||||
memoryMB int
|
||||
slot *network.Slot
|
||||
sandboxDir string // override for VMConfig.SandboxDir; "" = default
|
||||
sandboxDir string // override for VMConfig.SandboxDir; "" = default
|
||||
volumes []vm.VolumeDisk // data volumes to re-symlink so restored disk paths resolve
|
||||
}
|
||||
|
||||
// buildRestoreVMConfig assembles the VMConfig used to launch a CH process in
|
||||
@ -57,6 +58,7 @@ func (m *Manager) buildRestoreVMConfig(in restoreInputs) vm.VMConfig {
|
||||
RestoreFromDir: in.snapDir,
|
||||
RestoreLazyMemory: true,
|
||||
SandboxDir: in.sandboxDir,
|
||||
Volumes: in.volumes,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -202,6 +202,7 @@ func (m *Manager) reattachRunning(st *runningState) error {
|
||||
baseImagePath: st.BaseImagePath,
|
||||
sandboxDirOverride: st.SandboxDirOverride,
|
||||
lazyRestore: st.LazyRestore,
|
||||
volumes: st.Volumes,
|
||||
}
|
||||
sb.client.Store(envdclient.New(slot.HostIP.String()))
|
||||
|
||||
|
||||
@ -67,6 +67,10 @@ type runningState struct {
|
||||
LazyRestore bool `json:"lazy_restore,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
// Volumes records the data volumes attached to this sandbox so a process
|
||||
// re-attaching to the still-live VM (RestoreRunningSandboxes) can rebuild
|
||||
// its in-memory volume state without contacting the control plane.
|
||||
Volumes []*attachedVolume `json:"volumes,omitempty"`
|
||||
}
|
||||
|
||||
// writeRunningState persists sb's re-attach state next to its CoW file.
|
||||
@ -99,6 +103,7 @@ func (m *Manager) writeRunningState(sb *sandboxState) {
|
||||
LazyRestore: sb.lazyRestore,
|
||||
CreatedAt: sb.CreatedAt,
|
||||
Metadata: sb.Metadata,
|
||||
Volumes: sb.volumes,
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(st, "", " ")
|
||||
|
||||
164
pkg/sandbox/volumes.go
Normal file
164
pkg/sandbox/volumes.go
Normal file
@ -0,0 +1,164 @@
|
||||
// Package sandbox: external storage volumes attached to a sandbox at boot.
|
||||
//
|
||||
// A volume is a plain sparse file on the host (never a dm-snapshot CoW — a data
|
||||
// volume has no base image). It is handed to Cloud Hypervisor as an extra Raw
|
||||
// disk in the initial vm.create and symlinked into the VMM's private tmpfs like
|
||||
// the rootfs, so its disk path survives pause/resume via CH's snapshot config.
|
||||
// The guest resolves the device by its virtio-blk serial and envd formats (if
|
||||
// empty) and mounts it. Volumes are only ever attached at create time; they are
|
||||
// freed (not deleted) when the capsule is destroyed.
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/envdclient"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/id"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/layout"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/vm"
|
||||
)
|
||||
|
||||
// VolumeAttachSpec describes an external storage volume to attach to a sandbox
|
||||
// at boot. Passed to Create by the host-agent RPC layer; the host provisions
|
||||
// the backing file on first use.
|
||||
type VolumeAttachSpec struct {
|
||||
VolumeID pgtype.UUID
|
||||
TeamID pgtype.UUID
|
||||
SizeMB int
|
||||
MountPath string // "" => default /mnt/<vol-id>
|
||||
}
|
||||
|
||||
// attachedVolume is the host's lean record of a volume attached to a sandbox.
|
||||
// It doubles as the on-disk form (JSON tags) stored in runningState
|
||||
// (cross-process re-attach) and snapshotMeta (pause/resume) — self-contained so
|
||||
// a re-attaching or resuming process needs no re-derivation from IDs.
|
||||
type attachedVolume struct {
|
||||
VolumeID string `json:"volume_id"` // formatted "vol-..." (logging + default mount path)
|
||||
HostPath string `json:"host_path"` // backing sparse file on the host
|
||||
Serial string `json:"serial"` // virtio-blk serial; guest resolves the device by this
|
||||
MountPath string `json:"mount_path"` // effective guest mount path
|
||||
}
|
||||
|
||||
// volumeDisksFor returns the vm.VolumeDisk list for a set of attached volumes,
|
||||
// used to rebuild the VM config on resume so CH's restored disk paths resolve.
|
||||
func volumeDisksFor(vols []*attachedVolume) []vm.VolumeDisk {
|
||||
if len(vols) == 0 {
|
||||
return nil
|
||||
}
|
||||
disks := make([]vm.VolumeDisk, 0, len(vols))
|
||||
for _, v := range vols {
|
||||
disks = append(disks, vm.VolumeDisk{HostPath: v.HostPath, Serial: v.Serial})
|
||||
}
|
||||
return disks
|
||||
}
|
||||
|
||||
// ensureVolumeFile makes sure the backing sparse file exists at path, creating
|
||||
// it (and its parent dir) and truncating to sizeBytes on first use. Idempotent:
|
||||
// an existing file is left untouched so previously-written data survives across
|
||||
// detach and re-attach (size is fixed at first provision).
|
||||
func ensureVolumeFile(path string, sizeBytes int64) error {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return nil // already provisioned; preserve existing data
|
||||
} else if !os.IsNotExist(err) {
|
||||
return fmt.Errorf("stat volume file: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return fmt.Errorf("create volume dir: %w", err)
|
||||
}
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create volume file: %w", err)
|
||||
}
|
||||
if err := f.Truncate(sizeBytes); err != nil {
|
||||
f.Close()
|
||||
os.Remove(path)
|
||||
return fmt.Errorf("truncate volume file: %w", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return fmt.Errorf("close volume file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareVolumeDisks provisions the backing file for each spec and returns the
|
||||
// vm.VolumeDisk list to bake into the VM config plus the attachedVolume records
|
||||
// to track in sandbox state. Called before the VM boots.
|
||||
func (m *Manager) prepareVolumeDisks(specs []VolumeAttachSpec) ([]vm.VolumeDisk, []*attachedVolume, error) {
|
||||
if len(specs) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
disks := make([]vm.VolumeDisk, 0, len(specs))
|
||||
attached := make([]*attachedVolume, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
volIDStr := id.FormatVolumeID(spec.VolumeID)
|
||||
hostPath := layout.VolumeDataPath(m.cfg.WrennDir, spec.TeamID, spec.VolumeID)
|
||||
if err := ensureVolumeFile(hostPath, int64(spec.SizeMB)*1024*1024); err != nil {
|
||||
return nil, nil, fmt.Errorf("provision volume %s: %w", volIDStr, err)
|
||||
}
|
||||
serial := id.VolumeSerial(spec.VolumeID)
|
||||
mountPath := spec.MountPath
|
||||
if mountPath == "" {
|
||||
mountPath = id.DefaultVolumeMountPath(spec.VolumeID)
|
||||
}
|
||||
disks = append(disks, vm.VolumeDisk{HostPath: hostPath, Serial: serial})
|
||||
attached = append(attached, &attachedVolume{
|
||||
VolumeID: volIDStr,
|
||||
HostPath: hostPath,
|
||||
Serial: serial,
|
||||
MountPath: mountPath,
|
||||
})
|
||||
}
|
||||
return disks, attached, nil
|
||||
}
|
||||
|
||||
// mountVolumes asks envd to format (if empty) and mount every attached volume.
|
||||
// Called once after a fresh boot. On resume the guest mount survives inside the
|
||||
// restored memory image, so this is not called again.
|
||||
func (m *Manager) mountVolumes(ctx context.Context, client *envdclient.Client, vols []*attachedVolume) error {
|
||||
for _, v := range vols {
|
||||
mctx, cancel := context.WithTimeout(ctx, m.cfg.EnvdTimeout)
|
||||
err := client.MountVolume(mctx, v.Serial, v.MountPath)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("mount volume %s: %w", v.VolumeID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unmountVolumesBestEffort asks envd to sync+unmount each volume before a
|
||||
// graceful destroy so buffered guest writes reach the backing file. Best-effort:
|
||||
// on a crash envd is already gone and every call fails harmlessly.
|
||||
func (m *Manager) unmountVolumesBestEffort(sb *sandboxState) {
|
||||
if len(sb.volumes) == 0 {
|
||||
return
|
||||
}
|
||||
client := sb.client.Load()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
for _, v := range sb.volumes {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), m.cfg.EnvdTimeout)
|
||||
if err := client.UnmountVolume(ctx, v.MountPath); err != nil {
|
||||
slog.Warn("volume unmount on destroy failed", "id", sb.ID, "volume", v.VolumeID, "error", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteVolumeFile removes an external storage volume's backing file and its
|
||||
// (now-empty) directory tree from this host. Called by the control plane when a
|
||||
// detached volume is deleted. Idempotent: a missing file is not an error.
|
||||
func (m *Manager) DeleteVolumeFile(teamID, volumeID pgtype.UUID) error {
|
||||
dir := layout.VolumeDir(m.cfg.WrennDir, teamID, volumeID)
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return fmt.Errorf("remove volume dir %s: %w", id.FormatVolumeID(volumeID), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
42
pkg/sandbox/volumes_test.go
Normal file
42
pkg/sandbox/volumes_test.go
Normal file
@ -0,0 +1,42 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEnsureVolumeFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "team", "vol", "data.img")
|
||||
const size = 5 * 1024 * 1024 // 5 MiB
|
||||
|
||||
// First call creates the parent dirs and a sparse file of the right size.
|
||||
if err := ensureVolumeFile(path, size); err != nil {
|
||||
t.Fatalf("first ensureVolumeFile: %v", err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat after create: %v", err)
|
||||
}
|
||||
if info.Size() != size {
|
||||
t.Fatalf("size = %d, want %d", info.Size(), size)
|
||||
}
|
||||
|
||||
// Write data, then call again: an existing file must be left untouched so
|
||||
// previously-written data survives detach/re-attach.
|
||||
marker := []byte("volume-data-must-survive")
|
||||
if err := os.WriteFile(path, marker, 0o644); err != nil {
|
||||
t.Fatalf("write marker: %v", err)
|
||||
}
|
||||
if err := ensureVolumeFile(path, size); err != nil {
|
||||
t.Fatalf("second ensureVolumeFile: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read back: %v", err)
|
||||
}
|
||||
if string(got) != string(marker) {
|
||||
t.Fatalf("existing data was clobbered: got %q, want %q", got, marker)
|
||||
}
|
||||
}
|
||||
@ -14,6 +14,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/apperr"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/auth"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/db"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/id"
|
||||
@ -510,6 +511,19 @@ func (s *HostService) Delete(ctx context.Context, hostID, userID, teamID pgtype.
|
||||
return &HostHasSandboxesError{SandboxIDs: ids}
|
||||
}
|
||||
|
||||
// Volumes are pinned to a host; deleting it orphans their data. Refuse
|
||||
// unless forced (data on a host being intentionally destroyed is expected
|
||||
// to go with it).
|
||||
if !force {
|
||||
volCount, err := s.DB.CountVolumesByHost(ctx, hostID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count volumes: %w", err)
|
||||
}
|
||||
if volCount > 0 {
|
||||
return apperr.Conflict.Msgf("This host has %d volume(s) pinned to it. Deleting it will orphan their data. Pass ?force=true to proceed.", volCount)
|
||||
}
|
||||
}
|
||||
|
||||
hostIDStr := id.FormatHostID(hostID)
|
||||
|
||||
// Gracefully destroy running sandboxes and terminate the agent (best-effort).
|
||||
@ -552,6 +566,15 @@ func (s *HostService) Delete(ctx context.Context, hostID, userID, teamID pgtype.
|
||||
slog.Warn("delete host: failed to revoke refresh tokens", "host_id", hostIDStr, "error", err)
|
||||
}
|
||||
|
||||
// Free and un-pin any volumes on this host. The host (and its backing files)
|
||||
// are going away, so the volumes reset to a fresh detached, un-pinned state —
|
||||
// the rows survive (volumes are never auto-deleted) and can be re-attached on
|
||||
// a new host. This must also run before DeleteHost so the volumes.host_id FK
|
||||
// does not block the delete.
|
||||
if err := s.DB.DetachVolumesByHost(ctx, hostID); err != nil {
|
||||
return fmt.Errorf("release volumes on host: %w", err)
|
||||
}
|
||||
|
||||
// Evict the client from the pool so no further RPCs are sent.
|
||||
if s.Pool != nil {
|
||||
s.Pool.Evict(id.FormatHostID(hostID))
|
||||
|
||||
@ -37,9 +37,12 @@ type SandboxStateEvent struct {
|
||||
// SandboxService provides sandbox lifecycle operations shared between the
|
||||
// REST API and the dashboard.
|
||||
type SandboxService struct {
|
||||
DB *db.Queries
|
||||
Pool *lifecycle.HostClientPool
|
||||
Scheduler scheduler.HostScheduler
|
||||
DB *db.Queries
|
||||
Pool *lifecycle.HostClientPool
|
||||
Scheduler scheduler.HostScheduler
|
||||
// Volumes resolves the volume references passed at create time. Required
|
||||
// only when callers attach volumes.
|
||||
Volumes *VolumeService
|
||||
PublishEvent SandboxEventPublisher
|
||||
}
|
||||
|
||||
@ -53,8 +56,18 @@ type SandboxCreateParams struct {
|
||||
// Metadata holds user-supplied key/value labels attached at create-time.
|
||||
// Reserved system keys (kernel_version, etc.) are rejected by validation.
|
||||
Metadata map[string]string
|
||||
// VolumeRefs are external storage volumes to attach at boot, each given as
|
||||
// a volume ID ("vol-...") or a name ("vl-cache"). All must be detached and
|
||||
// owned by TeamID; if any is pinned to a host, the capsule is scheduled onto
|
||||
// that host. Volumes are only ever attached at create time.
|
||||
VolumeRefs []string
|
||||
}
|
||||
|
||||
// MaxVolumesPerSandbox caps how many volumes one capsule may attach. Each
|
||||
// becomes a virtio-blk device on the VM, so an unbounded list would fail late
|
||||
// and expensively — at boot, after every volume had already been reserved.
|
||||
const MaxVolumesPerSandbox = 4
|
||||
|
||||
// MinTimeoutSec mirrors pkg/sandbox.MinTimeoutSec. Sub-minute TTLs race
|
||||
// the post-create startup window (DB insert → /init → memory loader); the
|
||||
// agent silently clamps anyway, but the CP must clamp too so the DB record
|
||||
@ -106,6 +119,188 @@ func (s *SandboxService) publishEvent(ctx context.Context, event SandboxStateEve
|
||||
}
|
||||
}
|
||||
|
||||
// reservedVolume is a volume claimed for an in-flight capsule create.
|
||||
type reservedVolume struct {
|
||||
ID pgtype.UUID
|
||||
TeamID pgtype.UUID
|
||||
SizeMB int32
|
||||
MountPath string
|
||||
}
|
||||
|
||||
// reserveVolumesForCreate validates the requested volumes, computes the host
|
||||
// they pin the capsule to (if any), and atomically claims each one. On any
|
||||
// failure it releases whatever it already claimed so the caller can abort
|
||||
// cleanly. The claim records only the 'attaching' status — the owning
|
||||
// sandbox_id is set later in markVolumesAttached, because the sandbox row does
|
||||
// not exist yet (its FK would reject an early write). Returns the reserved
|
||||
// volumes and, when at least one volume is already pinned, that host.
|
||||
func (s *SandboxService) reserveVolumesForCreate(ctx context.Context, teamID pgtype.UUID, volumeRefs []string) ([]reservedVolume, pgtype.UUID, error) {
|
||||
if len(volumeRefs) == 0 {
|
||||
return nil, pgtype.UUID{}, nil
|
||||
}
|
||||
if len(volumeRefs) > MaxVolumesPerSandbox {
|
||||
return nil, pgtype.UUID{}, apperr.InvalidRequest.Msgf(
|
||||
"A capsule may attach at most %d volumes.", MaxVolumesPerSandbox)
|
||||
}
|
||||
if s.Volumes == nil {
|
||||
return nil, pgtype.UUID{}, apperr.InvalidRequest.Msg("Volumes are not available on this deployment.")
|
||||
}
|
||||
|
||||
var pinnedHost pgtype.UUID
|
||||
seen := make(map[[16]byte]bool, len(volumeRefs))
|
||||
reserved := make([]reservedVolume, 0, len(volumeRefs))
|
||||
|
||||
release := func() { s.releaseVolumeReservations(ctx, reserved) }
|
||||
|
||||
for _, ref := range volumeRefs {
|
||||
vol, err := s.Volumes.Resolve(ctx, teamID, ref)
|
||||
if err != nil {
|
||||
release()
|
||||
return nil, pgtype.UUID{}, err
|
||||
}
|
||||
if seen[vol.ID.Bytes] {
|
||||
continue // a volume attaches at most once per capsule
|
||||
}
|
||||
seen[vol.ID.Bytes] = true
|
||||
|
||||
// All pinned volumes must share one host — a capsule runs on a single
|
||||
// host and cannot straddle hosts.
|
||||
if vol.HostID.Valid {
|
||||
if pinnedHost.Valid && pinnedHost.Bytes != vol.HostID.Bytes {
|
||||
release()
|
||||
return nil, pgtype.UUID{}, apperr.VolumeHostMismatch.New()
|
||||
}
|
||||
pinnedHost = vol.HostID
|
||||
}
|
||||
|
||||
if _, err := s.DB.ReserveVolumeForAttach(ctx, vol.ID); err != nil {
|
||||
// CAS missed: the volume is not detached (already attached, or being
|
||||
// claimed by a racing create).
|
||||
release()
|
||||
return nil, pgtype.UUID{}, apperr.VolumeInUse.Msgf("Volume %q is not available to attach.", vol.Name)
|
||||
}
|
||||
reserved = append(reserved, reservedVolume{
|
||||
ID: vol.ID,
|
||||
TeamID: teamID,
|
||||
SizeMB: vol.SizeMb,
|
||||
MountPath: id.DefaultVolumeMountPath(vol.ID),
|
||||
})
|
||||
}
|
||||
return reserved, pinnedHost, nil
|
||||
}
|
||||
|
||||
func (s *SandboxService) releaseVolumeReservations(ctx context.Context, reserved []reservedVolume) {
|
||||
for _, rv := range reserved {
|
||||
if err := s.DB.ReleaseVolumeReservation(ctx, rv.ID); err != nil {
|
||||
slog.Warn("failed to release volume reservation", "volume", id.FormatVolumeID(rv.ID), "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// linkVolumeReservations stamps the owning sandbox onto each reservation once
|
||||
// the sandbox row exists. Reservations are claimed before the insert (the FK
|
||||
// would reject an earlier write), which would otherwise leave them anonymous —
|
||||
// and an anonymous reservation is indistinguishable from one whose capsule
|
||||
// really did boot, so nothing could safely reclaim it.
|
||||
func (s *SandboxService) linkVolumeReservations(ctx context.Context, reserved []reservedVolume, sandboxID pgtype.UUID) {
|
||||
for _, rv := range reserved {
|
||||
if err := s.DB.LinkVolumeReservation(ctx, db.LinkVolumeReservationParams{
|
||||
ID: rv.ID, SandboxID: sandboxID,
|
||||
}); err != nil {
|
||||
slog.Warn("failed to link volume reservation to sandbox", "volume", id.FormatVolumeID(rv.ID), "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// markVolumesAttachedRetries bounds how hard we try to record a successful
|
||||
// attach. The capsule is already running with the volume mounted at this point,
|
||||
// so losing this write leaves the row stuck at 'attaching' until the capsule
|
||||
// ends — recoverable, but worth retrying through a transient DB blip.
|
||||
const markVolumesAttachedRetries = 3
|
||||
|
||||
// destroySandboxRetries bounds how many times a destroy is re-sent before the
|
||||
// capsule is handed to the host monitor. Worth retrying because the failure
|
||||
// path deliberately reaches no terminal state — it leaves the capsule in
|
||||
// "stopping" with its volumes still claimed.
|
||||
const destroySandboxRetries = 3
|
||||
|
||||
func (s *SandboxService) markVolumesAttached(ctx context.Context, reserved []reservedVolume, hostID, sandboxID pgtype.UUID) {
|
||||
for _, rv := range reserved {
|
||||
var err error
|
||||
for attempt := range markVolumesAttachedRetries {
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Duration(attempt) * 500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if _, err = s.DB.MarkVolumeAttached(ctx, db.MarkVolumeAttachedParams{
|
||||
ID: rv.ID, HostID: hostID, SandboxID: sandboxID, MountPath: rv.MountPath,
|
||||
}); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
// The row keeps its sandbox_id, so the host-monitor sweep still
|
||||
// frees it when the capsule reaches a terminal state. Loud because
|
||||
// the volume is unusable by anything else until then.
|
||||
slog.Error("failed to mark volume attached; volume stays reserved until its capsule ends",
|
||||
"volume", id.FormatVolumeID(rv.ID), "sandbox_id", id.FormatSandboxID(sandboxID), "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sandboxConfirmedGone asks the host to destroy a capsule and reports whether
|
||||
// the host is now known not to be running it. Used on failure paths before
|
||||
// freeing resources the capsule may still hold: a NotFound means it never
|
||||
// existed, a success means it has been torn down, and anything else (including
|
||||
// an unreachable host) means we cannot tell and must not assume.
|
||||
func (s *SandboxService) sandboxConfirmedGone(ctx context.Context, agent hostagentClient, sandboxIDStr string) bool {
|
||||
_, err := agent.DestroySandbox(ctx, connect.NewRequest(&pb.DestroySandboxRequest{
|
||||
SandboxId: sandboxIDStr,
|
||||
}))
|
||||
return err == nil || connect.CodeOf(err) == connect.CodeNotFound
|
||||
}
|
||||
|
||||
func volumeSpecsToProto(reserved []reservedVolume) []*pb.VolumeSpec {
|
||||
if len(reserved) == 0 {
|
||||
return nil
|
||||
}
|
||||
specs := make([]*pb.VolumeSpec, 0, len(reserved))
|
||||
for _, rv := range reserved {
|
||||
specs = append(specs, &pb.VolumeSpec{
|
||||
VolumeId: id.UUIDString(rv.ID),
|
||||
TeamId: id.UUIDString(rv.TeamID),
|
||||
SizeMb: rv.SizeMB,
|
||||
MountPath: rv.MountPath,
|
||||
})
|
||||
}
|
||||
return specs
|
||||
}
|
||||
|
||||
// resolvePinnedHost returns the host a set of already-pinned volumes forces the
|
||||
// capsule onto, verifying it is online and eligible for the team. It bypasses
|
||||
// the scheduler: a pinned volume has exactly one valid host, so there is no
|
||||
// placement choice — only a check that the host can still run the capsule.
|
||||
func (s *SandboxService) resolvePinnedHost(ctx context.Context, hostID pgtype.UUID, isByoc bool, teamID pgtype.UUID) (db.Host, error) {
|
||||
host, err := s.DB.GetHost(ctx, hostID)
|
||||
if err != nil {
|
||||
return db.Host{}, apperr.VolumeHostMismatch.WrapMsg(err, "The host holding this volume no longer exists.")
|
||||
}
|
||||
if host.Status != "online" {
|
||||
return db.Host{}, apperr.VolumeHostMismatch.Msg("The host holding this volume is not online. Try again once it recovers.")
|
||||
}
|
||||
if isByoc {
|
||||
if host.Type != "byoc" || !host.TeamID.Valid || host.TeamID.Bytes != teamID.Bytes {
|
||||
return db.Host{}, apperr.VolumeHostMismatch.New()
|
||||
}
|
||||
} else if host.Type != "regular" {
|
||||
return db.Host{}, apperr.VolumeHostMismatch.New()
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
// hostagentClient is a local alias to avoid the full package path in signatures.
|
||||
type hostagentClient = interface {
|
||||
CreateSandbox(ctx context.Context, req *connect.Request[pb.CreateSandboxRequest]) (*connect.Response[pb.CreateSandboxResponse], error)
|
||||
@ -116,6 +311,7 @@ type hostagentClient = interface {
|
||||
GetSandboxMetrics(ctx context.Context, req *connect.Request[pb.GetSandboxMetricsRequest]) (*connect.Response[pb.GetSandboxMetricsResponse], error)
|
||||
FlushSandboxMetrics(ctx context.Context, req *connect.Request[pb.FlushSandboxMetricsRequest]) (*connect.Response[pb.FlushSandboxMetricsResponse], error)
|
||||
CreateSnapshot(ctx context.Context, req *connect.Request[pb.CreateSnapshotRequest]) (*connect.Response[pb.CreateSnapshotResponse], error)
|
||||
DeleteVolume(ctx context.Context, req *connect.Request[pb.DeleteVolumeRequest]) (*connect.Response[pb.DeleteVolumeResponse], error)
|
||||
}
|
||||
|
||||
// resolveTemplateRef resolves a parsed template reference to its owning
|
||||
@ -207,9 +403,34 @@ func (s *SandboxService) Create(ctx context.Context, p SandboxCreateParams) (db.
|
||||
return db.Sandbox{}, apperr.TeamNotFound.Wrap(err)
|
||||
}
|
||||
|
||||
host, err := s.Scheduler.SelectHost(ctx, p.TeamID, team.IsByoc, p.MemoryMB, 0)
|
||||
sandboxID := id.NewSandboxID()
|
||||
|
||||
// Claim any requested volumes and learn the host they pin the capsule to.
|
||||
// The reservations are released by the deferred cleanup below unless the
|
||||
// create hands them off to the background goroutine (committed = true).
|
||||
reserved, pinnedHost, err := s.reserveVolumesForCreate(ctx, p.TeamID, p.VolumeRefs)
|
||||
if err != nil {
|
||||
return db.Sandbox{}, fmt.Errorf("select host: %w", err)
|
||||
return db.Sandbox{}, err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
s.releaseVolumeReservations(ctx, reserved)
|
||||
}
|
||||
}()
|
||||
|
||||
// A pinned volume forces the host; otherwise the scheduler picks one.
|
||||
var host db.Host
|
||||
if pinnedHost.Valid {
|
||||
host, err = s.resolvePinnedHost(ctx, pinnedHost, team.IsByoc, p.TeamID)
|
||||
} else {
|
||||
host, err = s.Scheduler.SelectHost(ctx, p.TeamID, team.IsByoc, p.MemoryMB, 0)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("select host: %w", err)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return db.Sandbox{}, err
|
||||
}
|
||||
|
||||
agent, err := s.Pool.GetForHost(host)
|
||||
@ -217,7 +438,6 @@ func (s *SandboxService) Create(ctx context.Context, p SandboxCreateParams) (db.
|
||||
return db.Sandbox{}, fmt.Errorf("get agent client: %w", err)
|
||||
}
|
||||
|
||||
sandboxID := id.NewSandboxID()
|
||||
sandboxIDStr := id.FormatSandboxID(sandboxID)
|
||||
hostIDStr := id.FormatHostID(host.ID)
|
||||
|
||||
@ -246,9 +466,18 @@ func (s *SandboxService) Create(ctx context.Context, p SandboxCreateParams) (db.
|
||||
return db.Sandbox{}, fmt.Errorf("insert sandbox: %w", err)
|
||||
}
|
||||
|
||||
// The sandbox row now exists, so the reservations can name their owner.
|
||||
// Do this before handing off to the background goroutine: from here on,
|
||||
// every claimed volume is attributable to a capsule the host monitor can
|
||||
// reconcile against live host state.
|
||||
s.linkVolumeReservations(ctx, reserved, sandboxID)
|
||||
|
||||
teamIDStr := id.FormatTeamID(p.TeamID)
|
||||
s.publishStateChanged(ctx, sandboxIDStr, teamIDStr, hostIDStr, "", "starting")
|
||||
go s.createInBackground(sandboxID, sandboxIDStr, hostIDStr, teamIDStr, agent, p, templateTeamID, templateID, templateDefaultUser, templateDefaultEnv)
|
||||
// The background goroutine now owns the reservations (it marks them attached
|
||||
// on success or releases them on failure).
|
||||
committed = true
|
||||
go s.createInBackground(sandboxID, sandboxIDStr, hostIDStr, teamIDStr, agent, p, templateTeamID, templateID, templateDefaultUser, templateDefaultEnv, reserved, host.ID)
|
||||
|
||||
return sb, nil
|
||||
}
|
||||
@ -258,6 +487,7 @@ func (s *SandboxService) createInBackground(
|
||||
agent hostagentClient, p SandboxCreateParams,
|
||||
templateTeamID, templateID pgtype.UUID,
|
||||
defaultUser string, defaultEnv map[string]string,
|
||||
reserved []reservedVolume, hostID pgtype.UUID,
|
||||
) {
|
||||
bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
@ -273,23 +503,46 @@ func (s *SandboxService) createInBackground(
|
||||
DiskSizeMb: 0,
|
||||
DefaultUser: defaultUser,
|
||||
DefaultEnv: defaultEnv,
|
||||
Volumes: volumeSpecsToProto(reserved),
|
||||
}))
|
||||
if err != nil {
|
||||
slog.Warn("background create failed", "sandbox_id", sandboxIDStr, "error", err)
|
||||
errCtx, errCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
errCtx, errCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer errCancel()
|
||||
if _, dbErr := s.DB.UpdateSandboxStatusIf(errCtx, db.UpdateSandboxStatusIfParams{
|
||||
ID: sandboxID, Status: "starting", Status_2: "error",
|
||||
}); dbErr != nil {
|
||||
slog.Warn("failed to update sandbox to error after create failure", "id", sandboxIDStr, "error", dbErr)
|
||||
|
||||
// A failed RPC does not prove the capsule never booted — a lost or
|
||||
// timed-out response leaves a live VM with the volumes mounted. Tear it
|
||||
// down on the host first, and only free the volumes once the host
|
||||
// confirms it is gone. Releasing a volume that a running VM still has
|
||||
// open would let the next capsule attach the same backing file.
|
||||
if s.sandboxConfirmedGone(errCtx, agent, sandboxIDStr) {
|
||||
s.releaseVolumeReservations(errCtx, reserved)
|
||||
if _, dbErr := s.DB.UpdateSandboxStatusIf(errCtx, db.UpdateSandboxStatusIfParams{
|
||||
ID: sandboxID, Status: "starting", Status_2: "error",
|
||||
}); dbErr != nil {
|
||||
slog.Warn("failed to update sandbox to error after create failure", "id", sandboxIDStr, "error", dbErr)
|
||||
}
|
||||
s.publishEvent(errCtx, SandboxStateEvent{
|
||||
Event: "sandbox.failed", SandboxID: sandboxIDStr, TeamID: teamIDStr, HostID: hostIDStr,
|
||||
Error: err.Error(), Timestamp: time.Now().Unix(),
|
||||
})
|
||||
return
|
||||
}
|
||||
s.publishEvent(errCtx, SandboxStateEvent{
|
||||
Event: "sandbox.failed", SandboxID: sandboxIDStr, TeamID: teamIDStr, HostID: hostIDStr,
|
||||
Error: err.Error(), Timestamp: time.Now().Unix(),
|
||||
})
|
||||
|
||||
// The host could not confirm the teardown, so the capsule may still be
|
||||
// running with its volumes. Leave the sandbox in "starting" and the
|
||||
// volumes reserved: the host monitor owns both from here, and resolves
|
||||
// them against live host state once the host is reachable again.
|
||||
slog.Error("create failed and host could not confirm teardown; leaving capsule for host monitor",
|
||||
"sandbox_id", sandboxIDStr, "host_id", hostIDStr, "volumes", len(reserved))
|
||||
return
|
||||
}
|
||||
|
||||
// The capsule booted with its volumes attached; promote each reservation to
|
||||
// attached, record the owning sandbox, and pin the volume to this host
|
||||
// (COALESCE keeps an existing pin).
|
||||
s.markVolumesAttached(bgCtx, reserved, hostID, sandboxID)
|
||||
|
||||
if resp.Msg.DiskSizeMb > 0 {
|
||||
if err := s.DB.UpdateSandboxDiskSize(bgCtx, db.UpdateSandboxDiskSizeParams{
|
||||
ID: sandboxID,
|
||||
@ -714,10 +967,49 @@ func (s *SandboxService) destroyInBackground(sandboxID pgtype.UUID, sandboxIDStr
|
||||
s.flushAndPersistMetrics(bgCtx, agent, sandboxID, false)
|
||||
}
|
||||
|
||||
if _, err := agent.DestroySandbox(bgCtx, connect.NewRequest(&pb.DestroySandboxRequest{
|
||||
SandboxId: sandboxIDStr,
|
||||
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
|
||||
slog.Warn("background destroy failed", "sandbox_id", sandboxIDStr, "error", err)
|
||||
// Retry before giving up: most non-NotFound failures here are transport
|
||||
// hiccups, and the alternative — leaving the capsule mid-destroy for the
|
||||
// host monitor — costs the user a reconciliation cycle.
|
||||
var destroyErr error
|
||||
for attempt := range destroySandboxRetries {
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-bgCtx.Done():
|
||||
return
|
||||
case <-time.After(time.Duration(attempt) * time.Second):
|
||||
}
|
||||
}
|
||||
_, destroyErr = agent.DestroySandbox(bgCtx, connect.NewRequest(&pb.DestroySandboxRequest{
|
||||
SandboxId: sandboxIDStr,
|
||||
}))
|
||||
// NotFound means the host does not have it — the outcome we wanted.
|
||||
if destroyErr == nil || connect.CodeOf(destroyErr) == connect.CodeNotFound {
|
||||
destroyErr = nil
|
||||
break
|
||||
}
|
||||
}
|
||||
if destroyErr != nil {
|
||||
// The host never confirmed the teardown, so the VM may still be running
|
||||
// with its volumes mounted. Leave the capsule in "stopping" and its
|
||||
// volumes held — freeing a volume here could hand its backing file to a
|
||||
// second capsule while the first still has it open. The host monitor
|
||||
// re-issues the destroy against live host state.
|
||||
//
|
||||
// Publish the failure so the operation is not silently open-ended: this
|
||||
// is the one path that reaches no terminal state of its own.
|
||||
slog.Error("background destroy failed; leaving capsule for host monitor",
|
||||
"sandbox_id", sandboxIDStr, "error", destroyErr)
|
||||
s.publishEvent(bgCtx, SandboxStateEvent{
|
||||
Event: "sandbox.destroy_failed", SandboxID: sandboxIDStr, TeamID: teamIDStr, HostID: hostIDStr,
|
||||
Error: destroyErr.Error(), Timestamp: time.Now().Unix(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Free any attached volumes back to detached (data and host pin preserved).
|
||||
// Volumes are never auto-deleted — only an explicit delete removes them.
|
||||
if err := s.DB.DetachVolumesBySandbox(bgCtx, sandboxID); err != nil {
|
||||
slog.Warn("failed to detach volumes on destroy", "sandbox_id", sandboxIDStr, "error", err)
|
||||
}
|
||||
|
||||
if prevStatus == "paused" {
|
||||
|
||||
@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
@ -346,6 +347,15 @@ func (s *TeamService) deleteTeamCore(ctx context.Context, teamID pgtype.UUID) er
|
||||
// as that would leave orphaned "running" records for a deleted team.
|
||||
return fmt.Errorf("update sandbox statuses: %w", err)
|
||||
}
|
||||
|
||||
// Free any volumes those capsules held. The destroys above already ran
|
||||
// against the host, so this is the same confirmed-gone signal the host
|
||||
// monitor uses — no VM is left holding a backing file open. Doing it here
|
||||
// (rather than relying on the volume cleanup below) keeps volume state
|
||||
// consistent even if that cleanup fails.
|
||||
if err := s.DB.DetachVolumesBySandboxIDs(ctx, stopIDs); err != nil {
|
||||
slog.Warn("team delete: failed to detach volumes", "team_id", id.FormatTeamID(teamID), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete sandbox metrics for this team.
|
||||
@ -366,8 +376,10 @@ func (s *TeamService) deleteTeamCore(ctx context.Context, teamID pgtype.UUID) er
|
||||
slog.Warn("team delete: failed to delete channels", "team_id", id.FormatTeamID(teamID), "error", err)
|
||||
}
|
||||
|
||||
// Clean up team-owned templates from all hosts in the background.
|
||||
// Clean up team-owned templates and storage volumes from all hosts in the
|
||||
// background.
|
||||
go s.cleanupTeamTemplates(context.Background(), teamID)
|
||||
go s.cleanupTeamVolumes(context.Background(), teamID)
|
||||
|
||||
if err := s.DB.SoftDeleteTeam(ctx, teamID); err != nil {
|
||||
return fmt.Errorf("soft delete team: %w", err)
|
||||
@ -421,6 +433,65 @@ func (s *TeamService) cleanupTeamTemplates(ctx context.Context, teamID pgtype.UU
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupTeamVolumes deletes every storage volume owned by a team being deleted:
|
||||
// the backing file on each volume's pinned host, then the DB records. Called
|
||||
// asynchronously during team deletion.
|
||||
//
|
||||
// Normal volume deletion is always explicit (VolumeService.Delete) — a destroyed
|
||||
// capsule only frees its volumes, it never removes them. Team deletion is the
|
||||
// exception: the owner of the volumes is going away, so leaving the data behind
|
||||
// would orphan it on the host with no way left to reach it.
|
||||
//
|
||||
// Unlike VolumeService.Delete there is no per-volume status claim: the team's
|
||||
// capsules were destroyed before this ran, so nothing can be attaching or
|
||||
// re-attaching. Records are dropped even when a host is unreachable, matching
|
||||
// template cleanup — a row kept for a deleted team is unreachable forever and
|
||||
// would keep blocking deletion of the host it is pinned to.
|
||||
func (s *TeamService) cleanupTeamVolumes(ctx context.Context, teamID pgtype.UUID) {
|
||||
volumes, err := s.DB.ListVolumesByTeam(ctx, teamID)
|
||||
if err != nil {
|
||||
slog.Warn("team delete: failed to list volumes for cleanup", "team_id", id.FormatTeamID(teamID), "error", err)
|
||||
return
|
||||
}
|
||||
if len(volumes) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, vol := range volumes {
|
||||
// A volume that was never attached has no file on any host, and one whose
|
||||
// pinned host is gone lost its data with that host's disk. Either way
|
||||
// there is nothing to remove — just drop the record below.
|
||||
if !vol.HostID.Valid {
|
||||
continue
|
||||
}
|
||||
host, err := s.DB.GetHost(ctx, vol.HostID)
|
||||
if err != nil {
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
slog.Warn("team delete: failed to load volume host",
|
||||
"volume_id", id.FormatVolumeID(vol.ID), "error", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
agent, err := s.HostPool.GetForHost(host)
|
||||
if err != nil {
|
||||
slog.Warn("team delete: failed to reach volume host",
|
||||
"host_id", id.FormatHostID(host.ID), "volume_id", id.FormatVolumeID(vol.ID), "error", err)
|
||||
continue
|
||||
}
|
||||
if _, err := agent.DeleteVolume(ctx, connect.NewRequest(&pb.DeleteVolumeRequest{
|
||||
TeamId: id.UUIDString(teamID),
|
||||
VolumeId: id.UUIDString(vol.ID),
|
||||
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
|
||||
slog.Warn("team delete: failed to delete volume on host",
|
||||
"host_id", id.FormatHostID(host.ID), "volume_id", id.FormatVolumeID(vol.ID), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.DB.DeleteVolumesByTeam(ctx, teamID); err != nil {
|
||||
slog.Warn("team delete: failed to delete volume records", "team_id", id.FormatTeamID(teamID), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetMembers returns all members of the team with their emails and roles.
|
||||
func (s *TeamService) GetMembers(ctx context.Context, teamID pgtype.UUID) ([]MemberInfo, error) {
|
||||
rows, err := s.DB.GetTeamMembers(ctx, teamID)
|
||||
|
||||
197
pkg/service/volume.go
Normal file
197
pkg/service/volume.go
Normal file
@ -0,0 +1,197 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/units"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/apperr"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/db"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/id"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/lifecycle"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/validate"
|
||||
pb "git.omukk.dev/wrenn/wrenn/proto/hostagent/gen"
|
||||
)
|
||||
|
||||
// MinVolumeSizeMB is the smallest volume that can be created. Below roughly
|
||||
// this size an ext4 filesystem's own metadata dominates the usable space.
|
||||
const MinVolumeSizeMB int32 = 100
|
||||
|
||||
// DefaultMaxVolumeSizeMB is the per-volume ceiling used when a VolumeService is
|
||||
// constructed without one. Deployments override it with WRENN_MAX_VOLUME_SIZE.
|
||||
const DefaultMaxVolumeSizeMB int32 = 20 * 1024 // 20 GiB
|
||||
|
||||
// VolumeService provides external-storage-volume lifecycle operations. Volume
|
||||
// metadata lives entirely in Postgres; the only host interaction is deleting a
|
||||
// pinned volume's backing file. Unlike SandboxService there is no async state
|
||||
// machine — create/list/get are pure DB ops and delete is synchronous.
|
||||
//
|
||||
// Attach/detach are not offered here: a volume is attached only at capsule
|
||||
// create (see SandboxService.Create) and freed when the capsule is destroyed.
|
||||
type VolumeService struct {
|
||||
DB *db.Queries
|
||||
Pool *lifecycle.HostClientPool
|
||||
|
||||
// MaxSizeMB caps the size of a single volume. Zero means
|
||||
// DefaultMaxVolumeSizeMB.
|
||||
MaxSizeMB int32
|
||||
}
|
||||
|
||||
func (s *VolumeService) maxSizeMB() int32 {
|
||||
if s.MaxSizeMB > 0 {
|
||||
return s.MaxSizeMB
|
||||
}
|
||||
return DefaultMaxVolumeSizeMB
|
||||
}
|
||||
|
||||
// Create records a new detached volume. No host is chosen yet — a volume is
|
||||
// pinned to a host only when first attached to a capsule.
|
||||
//
|
||||
// name is optional: an empty name yields "vl-<volume id>", so a caller that
|
||||
// does not care about naming still gets a stable, addressable handle. A
|
||||
// supplied name is normalized to its "vl-"-prefixed slug form.
|
||||
func (s *VolumeService) Create(ctx context.Context, teamID pgtype.UUID, name string, sizeMB int32) (db.Volume, error) {
|
||||
if !teamID.Valid {
|
||||
return db.Volume{}, apperr.InvalidRequest.Msg("A team_id is required.")
|
||||
}
|
||||
if sizeMB < MinVolumeSizeMB {
|
||||
return db.Volume{}, apperr.InvalidRequest.Msgf("Volume size must be at least %s.", units.FormatMB(int(MinVolumeSizeMB)))
|
||||
}
|
||||
if max := s.maxSizeMB(); sizeMB > max {
|
||||
return db.Volume{}, apperr.InvalidRequest.Msgf("Volume size must not exceed %s.", units.FormatMB(int(max)))
|
||||
}
|
||||
|
||||
volumeID := id.NewVolumeID()
|
||||
// A nameless volume is named after itself. id.FormatVolumeID carries the
|
||||
// "vol-" ID prefix, so use the bare base36 body to keep the name in the
|
||||
// "vl-" namespace.
|
||||
if strings.TrimSpace(name) == "" {
|
||||
name = validate.VolumeNamePrefix + id.UUIDToBase36(volumeID.Bytes)
|
||||
}
|
||||
name, err := validate.VolumeName(name)
|
||||
if err != nil {
|
||||
return db.Volume{}, apperr.ValidationFailed.Msgf("Invalid volume name: %v.", err)
|
||||
}
|
||||
|
||||
vol, err := s.DB.InsertVolume(ctx, db.InsertVolumeParams{
|
||||
ID: volumeID,
|
||||
TeamID: teamID,
|
||||
Name: name,
|
||||
SizeMb: sizeMB,
|
||||
})
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return db.Volume{}, apperr.VolumeNameTaken.Msgf("A volume named %q already exists in this team.", name)
|
||||
}
|
||||
return db.Volume{}, fmt.Errorf("insert volume: %w", err)
|
||||
}
|
||||
return vol, nil
|
||||
}
|
||||
|
||||
// Resolve turns a user-supplied reference into the team's volume. A reference
|
||||
// is either a volume ID ("vol-<base36>") or a name ("vl-cache", or plain
|
||||
// "cache" — the prefix is optional on input). The two namespaces cannot
|
||||
// collide: an ID always carries the "vol-" prefix, a name always "vl-".
|
||||
func (s *VolumeService) Resolve(ctx context.Context, teamID pgtype.UUID, ref string) (db.Volume, error) {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return db.Volume{}, apperr.InvalidRequest.Msg("A volume ID or name is required.")
|
||||
}
|
||||
|
||||
if strings.HasPrefix(ref, id.PrefixVolume) {
|
||||
volumeID, err := id.ParseVolumeID(ref)
|
||||
if err != nil {
|
||||
return db.Volume{}, apperr.InvalidRequest.WrapMsg(err, "Invalid volume ID.")
|
||||
}
|
||||
return s.Get(ctx, volumeID, teamID)
|
||||
}
|
||||
|
||||
name, err := validate.VolumeName(ref)
|
||||
if err != nil {
|
||||
return db.Volume{}, apperr.InvalidRequest.Msgf("Invalid volume reference: %v.", err)
|
||||
}
|
||||
vol, err := s.DB.GetVolumeByTeamAndName(ctx, db.GetVolumeByTeamAndNameParams{TeamID: teamID, Name: name})
|
||||
if err != nil {
|
||||
return db.Volume{}, apperr.VolumeNotFound.WrapMsg(err, fmt.Sprintf("Volume %q not found.", name))
|
||||
}
|
||||
return vol, nil
|
||||
}
|
||||
|
||||
// List returns all volumes owned by the team, newest first.
|
||||
func (s *VolumeService) List(ctx context.Context, teamID pgtype.UUID) ([]db.Volume, error) {
|
||||
return s.DB.ListVolumesByTeam(ctx, teamID)
|
||||
}
|
||||
|
||||
// Get returns a single volume owned by the team.
|
||||
func (s *VolumeService) Get(ctx context.Context, volumeID, teamID pgtype.UUID) (db.Volume, error) {
|
||||
vol, err := s.DB.GetVolumeByTeam(ctx, db.GetVolumeByTeamParams{ID: volumeID, TeamID: teamID})
|
||||
if err != nil {
|
||||
return db.Volume{}, apperr.VolumeNotFound.Wrap(err)
|
||||
}
|
||||
return vol, nil
|
||||
}
|
||||
|
||||
// Delete removes a detached volume: it deletes the backing file on the volume's
|
||||
// pinned host (if any), then the DB row. An attached volume cannot be deleted —
|
||||
// the capsule using it must be destroyed first.
|
||||
func (s *VolumeService) Delete(ctx context.Context, volumeID, teamID pgtype.UUID) error {
|
||||
// Atomically claim the volume for deletion (detached → deleting). The CAS
|
||||
// blocks a concurrent attach — which needs status='detached' — closing the
|
||||
// TOCTOU window between checking the status and removing the file/row.
|
||||
vol, err := s.DB.BeginVolumeDelete(ctx, db.BeginVolumeDeleteParams{ID: volumeID, TeamID: teamID})
|
||||
if err != nil {
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return fmt.Errorf("claim volume for delete: %w", err)
|
||||
}
|
||||
// Not claimable: either the volume is missing / not this team's, or it
|
||||
// is currently attached/attaching. Distinguish for a precise error.
|
||||
if _, gerr := s.DB.GetVolumeByTeam(ctx, db.GetVolumeByTeamParams{ID: volumeID, TeamID: teamID}); gerr != nil {
|
||||
return apperr.VolumeNotFound.Wrap(gerr)
|
||||
}
|
||||
return apperr.VolumeInUse.New()
|
||||
}
|
||||
|
||||
// Remove the backing file from the host that holds it. A volume that was
|
||||
// never attached (host_id NULL) has no file on any host — just drop the row.
|
||||
if vol.HostID.Valid {
|
||||
host, err := s.DB.GetHost(ctx, vol.HostID)
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
// The pinned host was deleted; its disk (and the volume's file) are
|
||||
// gone with it. Nothing to remove — proceed to drop the row so the
|
||||
// volume isn't left permanently un-deletable.
|
||||
case err != nil:
|
||||
_ = s.DB.AbortVolumeDelete(ctx, volumeID) // revert to detached
|
||||
return apperr.HostNotFound.Wrap(err)
|
||||
default:
|
||||
agent, err := s.Pool.GetForHost(host)
|
||||
if err != nil {
|
||||
_ = s.DB.AbortVolumeDelete(ctx, volumeID)
|
||||
return fmt.Errorf("get agent client: %w", err)
|
||||
}
|
||||
// NotFound is fine — the file may already be gone. Any other error
|
||||
// must fail the delete (reverting the claim) so we never drop the
|
||||
// row while the file lingers on a reachable host.
|
||||
if _, err := agent.DeleteVolume(ctx, connect.NewRequest(&pb.DeleteVolumeRequest{
|
||||
TeamId: id.UUIDString(teamID),
|
||||
VolumeId: id.UUIDString(volumeID),
|
||||
})); err != nil && connect.CodeOf(err) != connect.CodeNotFound {
|
||||
_ = s.DB.AbortVolumeDelete(ctx, volumeID)
|
||||
return fmt.Errorf("delete volume file on host: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.DB.DeleteVolumeRow(ctx, db.DeleteVolumeRowParams{ID: volumeID, TeamID: teamID}); err != nil {
|
||||
return fmt.Errorf("delete volume row: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
44
pkg/validate/volume.go
Normal file
44
pkg/validate/volume.go
Normal file
@ -0,0 +1,44 @@
|
||||
package validate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// VolumeNamePrefix is the mandatory leading marker on every volume name. It
|
||||
// makes a name self-describing in logs, mount paths, and SDK code, and — since
|
||||
// volume IDs carry the distinct "vol-" prefix — lets a single API path segment
|
||||
// be resolved as either an ID or a name without ambiguity.
|
||||
const VolumeNamePrefix = "vl-"
|
||||
|
||||
// VolumeName normalizes and validates a user-supplied volume name. The name
|
||||
// body follows the same rules as a team slug (lowercase alphanumerics in
|
||||
// dash-separated groups) so names stay safe in paths, URLs, and shell contexts.
|
||||
//
|
||||
// The prefix is optional on input: both "cache" and "vl-cache" normalize to
|
||||
// "vl-cache", so an SDK caller who forgets it still gets the volume they meant.
|
||||
// The returned name is always the prefixed, lower-cased form and is what
|
||||
// callers should store and compare against.
|
||||
func VolumeName(name string) (string, error) {
|
||||
normalized := strings.ToLower(strings.TrimSpace(name))
|
||||
if normalized == "" {
|
||||
return "", fmt.Errorf("volume name must not be empty")
|
||||
}
|
||||
normalized = strings.TrimPrefix(normalized, VolumeNamePrefix)
|
||||
if normalized == "" {
|
||||
return "", fmt.Errorf("volume name must have something after %q", VolumeNamePrefix)
|
||||
}
|
||||
|
||||
normalized = VolumeNamePrefix + normalized
|
||||
// The whole prefixed name is capped at 40 characters, which comfortably
|
||||
// fits the default "vl-" + 25-char base36 ID form (28 characters).
|
||||
if len(normalized) > 40 {
|
||||
return "", fmt.Errorf("volume name must be at most 40 characters including the %q prefix", VolumeNamePrefix)
|
||||
}
|
||||
// slugRe is shared with team slugs: lowercase alphanumerics in
|
||||
// dash-separated groups, no leading, trailing, or repeated dashes.
|
||||
if !slugRe.MatchString(normalized) {
|
||||
return "", fmt.Errorf("volume name may only contain lowercase letters, numbers, and dashes (no leading, trailing, or repeated dashes)")
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
78
pkg/validate/volume_test.go
Normal file
78
pkg/validate/volume_test.go
Normal file
@ -0,0 +1,78 @@
|
||||
package validate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVolumeName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"bare-name-gets-prefix", "cache", "vl-cache", false},
|
||||
{"already-prefixed", "vl-cache", "vl-cache", false},
|
||||
{"multi-group", "vl-build-cache-2", "vl-build-cache-2", false},
|
||||
{"digits", "vl-1", "vl-1", false},
|
||||
{"uppercase-lowered", "VL-Cache", "vl-cache", false},
|
||||
{"trims-space", " cache ", "vl-cache", false},
|
||||
{"default-id-form", "vl-" + strings.Repeat("a", 25), "vl-" + strings.Repeat("a", 25), false},
|
||||
{"max-length", "vl-" + strings.Repeat("a", 37), "vl-" + strings.Repeat("a", 37), false},
|
||||
// A second "vl-" is part of the name, not a repeated prefix — only one
|
||||
// is ever stripped, so this stays distinct from "vl-cache".
|
||||
{"double-prefix-kept", "vl-vl-cache", "vl-vl-cache", false},
|
||||
|
||||
{"empty", "", "", true},
|
||||
{"prefix-only", "vl-", "", true},
|
||||
{"whitespace-only", " ", "", true},
|
||||
{"too-long", "vl-" + strings.Repeat("a", 38), "", true},
|
||||
{"underscore", "my_cache", "", true},
|
||||
{"dot", "my.cache", "", true},
|
||||
{"slash", "team/cache", "", true},
|
||||
{"space-inside", "my cache", "", true},
|
||||
{"leading-dash", "-cache", "", true},
|
||||
{"trailing-dash", "cache-", "", true},
|
||||
{"double-dash", "my--cache", "", true},
|
||||
{"non-ascii", "café", "", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := VolumeName(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("VolumeName(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("VolumeName(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Normalizing an already-normalized name must be a no-op, otherwise a name
|
||||
// would drift every time it round-trips through the API.
|
||||
func TestVolumeNameIsIdempotent(t *testing.T) {
|
||||
for _, input := range []string{"cache", "vl-cache", "VL-Build-Cache", "vl-vl-cache"} {
|
||||
once, err := VolumeName(input)
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeName(%q) unexpected error: %v", input, err)
|
||||
}
|
||||
twice, err := VolumeName(once)
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeName(%q) unexpected error: %v", once, err)
|
||||
}
|
||||
if once != twice {
|
||||
t.Errorf("VolumeName not idempotent for %q: %q then %q", input, once, twice)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A volume ID prefix must never be mistaken for a name prefix, which is what
|
||||
// lets one path segment carry either without ambiguity.
|
||||
func TestVolumeNamePrefixDoesNotCollideWithIDPrefix(t *testing.T) {
|
||||
if strings.HasPrefix("vol-", VolumeNamePrefix) || strings.HasPrefix(VolumeNamePrefix, "vol-") {
|
||||
t.Fatalf("volume name prefix %q collides with the vol- ID prefix", VolumeNamePrefix)
|
||||
}
|
||||
}
|
||||
27
pkg/vm/ch.go
27
pkg/vm/ch.go
@ -107,6 +107,9 @@ type chDisk struct {
|
||||
Path string `json:"path"`
|
||||
Readonly bool `json:"readonly,omitempty"`
|
||||
ImageType string `json:"image_type,omitempty"`
|
||||
// Serial is exposed to the guest as the virtio-blk serial so envd can
|
||||
// resolve the device via /sys/block/*/serial. Only set for data volumes.
|
||||
Serial string `json:"serial,omitempty"`
|
||||
}
|
||||
|
||||
type chNet struct {
|
||||
@ -141,6 +144,23 @@ type chCreatePayload struct {
|
||||
func (c *chClient) createVM(ctx context.Context, cfg *VMConfig) error {
|
||||
memBytes := uint64(cfg.MemoryMB) * 1024 * 1024
|
||||
|
||||
// Rootfs is always disk 0 (guest /dev/vda, referenced by root=/dev/vda in
|
||||
// the kernel cmdline). Data volumes follow as vdb, vdc, ... — but the guest
|
||||
// resolves each by serial, not by enumeration order.
|
||||
disks := []chDisk{
|
||||
{
|
||||
Path: cfg.SandboxDir + "/rootfs.ext4",
|
||||
ImageType: "Raw",
|
||||
},
|
||||
}
|
||||
for _, v := range cfg.Volumes {
|
||||
disks = append(disks, chDisk{
|
||||
Path: cfg.SandboxDir + "/" + v.symlinkName(),
|
||||
ImageType: "Raw",
|
||||
Serial: v.Serial,
|
||||
})
|
||||
}
|
||||
|
||||
payload := chCreatePayload{
|
||||
Payload: chPayload{
|
||||
Kernel: cfg.KernelPath,
|
||||
@ -155,12 +175,7 @@ func (c *chClient) createVM(ctx context.Context, cfg *VMConfig) error {
|
||||
Shared: true,
|
||||
Thp: boolPtr(false),
|
||||
},
|
||||
Disks: []chDisk{
|
||||
{
|
||||
Path: cfg.SandboxDir + "/rootfs.ext4",
|
||||
ImageType: "Raw",
|
||||
},
|
||||
},
|
||||
Disks: disks,
|
||||
Net: []chNet{
|
||||
{
|
||||
Tap: cfg.TapDevice,
|
||||
|
||||
@ -15,6 +15,27 @@ func SandboxSocketPath(sandboxID string) string {
|
||||
return fmt.Sprintf("/tmp/ch-%s.sock", sandboxID)
|
||||
}
|
||||
|
||||
// VolumeDisk is an external storage volume attached to a VM at boot.
|
||||
type VolumeDisk struct {
|
||||
// HostPath is the backing sparse file on the host filesystem. It is
|
||||
// visible inside the VMM's private mount namespace (unshare -m isolates
|
||||
// the mount table, not the filesystem tree), so CH can open it via the
|
||||
// stable symlink created in SandboxDir.
|
||||
HostPath string
|
||||
// Serial is the virtio-blk serial (derived from the volume ID) the guest
|
||||
// uses to resolve the block device via /sys/block/*/serial. It also names
|
||||
// the in-namespace symlink so the disk path is stable across restore.
|
||||
Serial string
|
||||
}
|
||||
|
||||
// symlinkName returns the volume's stable filename inside SandboxDir. CH bakes
|
||||
// this path into its snapshot config.json, so it must be reconstructable on
|
||||
// every restore — hence keyed by the (deterministic) serial, not enumeration
|
||||
// order.
|
||||
func (v VolumeDisk) symlinkName() string {
|
||||
return "volume-" + v.Serial + ".img"
|
||||
}
|
||||
|
||||
// VMConfig holds the configuration for creating a Cloud Hypervisor microVM.
|
||||
type VMConfig struct {
|
||||
// SandboxID is the unique identifier for this sandbox (e.g., "cl-a1b2c3d4").
|
||||
@ -30,6 +51,12 @@ type VMConfig struct {
|
||||
// Typically a dm-snapshot device (e.g., /dev/mapper/wrenn-cl-a1b2c3d4).
|
||||
RootfsPath string
|
||||
|
||||
// Volumes are external storage disks attached at boot, after the rootfs.
|
||||
// Each is symlinked into SandboxDir (like the rootfs) so the disk path CH
|
||||
// records in its snapshot config.json stays stable across restore, and is
|
||||
// exposed to the guest as an additional virtio-blk device (vdb, vdc, ...).
|
||||
Volumes []VolumeDisk
|
||||
|
||||
// VCPUs is the number of virtual CPUs to allocate (default: 1).
|
||||
VCPUs int
|
||||
|
||||
|
||||
@ -137,6 +137,15 @@ func buildLaunchScript(cfg *VMConfig, extraArgs string) string {
|
||||
if extraArgs != "" {
|
||||
chCmd += " " + extraArgs
|
||||
}
|
||||
|
||||
// One symlink per data volume, pointing the stable in-namespace path (baked
|
||||
// into CH's config.json) at the real backing file on the host. Recreated
|
||||
// identically on every restore so the snapshot's disk paths resolve.
|
||||
var volumeLinks strings.Builder
|
||||
for _, v := range cfg.Volumes {
|
||||
fmt.Fprintf(&volumeLinks, "ln -s %s %s/%s\n", v.HostPath, cfg.SandboxDir, v.symlinkName())
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`
|
||||
set -euo pipefail
|
||||
|
||||
@ -147,13 +156,14 @@ mount -t tmpfs tmpfs %[1]s
|
||||
|
||||
ln -s %[2]s %[1]s/vmlinux
|
||||
ln -s %[3]s %[1]s/rootfs.ext4
|
||||
|
||||
%[5]s
|
||||
exec %[4]s
|
||||
`,
|
||||
cfg.SandboxDir, // 1
|
||||
cfg.KernelPath, // 2
|
||||
cfg.RootfsPath, // 3
|
||||
chCmd, // 4
|
||||
cfg.SandboxDir, // 1
|
||||
cfg.KernelPath, // 2
|
||||
cfg.RootfsPath, // 3
|
||||
chCmd, // 4
|
||||
volumeLinks.String(), // 5
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -122,6 +122,9 @@ const (
|
||||
// HostAgentServiceGetTemplateSizeProcedure is the fully-qualified name of the HostAgentService's
|
||||
// GetTemplateSize RPC.
|
||||
HostAgentServiceGetTemplateSizeProcedure = "/hostagent.v1.HostAgentService/GetTemplateSize"
|
||||
// HostAgentServiceDeleteVolumeProcedure is the fully-qualified name of the HostAgentService's
|
||||
// DeleteVolume RPC.
|
||||
HostAgentServiceDeleteVolumeProcedure = "/hostagent.v1.HostAgentService/DeleteVolume"
|
||||
)
|
||||
|
||||
// HostAgentServiceClient is a client for the hostagent.v1.HostAgentService service.
|
||||
@ -201,6 +204,10 @@ type HostAgentServiceClient interface {
|
||||
// size 0 in the database (e.g. system base templates seeded before the
|
||||
// rootfs was built).
|
||||
GetTemplateSize(context.Context, *connect.Request[gen.GetTemplateSizeRequest]) (*connect.Response[gen.GetTemplateSizeResponse], error)
|
||||
// DeleteVolume removes an external storage volume's backing file from this
|
||||
// host. Called by the control plane when a detached volume is deleted; the
|
||||
// volume must not be attached to any sandbox on this host.
|
||||
DeleteVolume(context.Context, *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error)
|
||||
}
|
||||
|
||||
// NewHostAgentServiceClient constructs a client for the hostagent.v1.HostAgentService service. By
|
||||
@ -394,6 +401,12 @@ func NewHostAgentServiceClient(httpClient connect.HTTPClient, baseURL string, op
|
||||
connect.WithSchema(hostAgentServiceMethods.ByName("GetTemplateSize")),
|
||||
connect.WithClientOptions(opts...),
|
||||
),
|
||||
deleteVolume: connect.NewClient[gen.DeleteVolumeRequest, gen.DeleteVolumeResponse](
|
||||
httpClient,
|
||||
baseURL+HostAgentServiceDeleteVolumeProcedure,
|
||||
connect.WithSchema(hostAgentServiceMethods.ByName("DeleteVolume")),
|
||||
connect.WithClientOptions(opts...),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@ -429,6 +442,7 @@ type hostAgentServiceClient struct {
|
||||
killProcess *connect.Client[gen.KillProcessRequest, gen.KillProcessResponse]
|
||||
connectProcess *connect.Client[gen.ConnectProcessRequest, gen.ConnectProcessResponse]
|
||||
getTemplateSize *connect.Client[gen.GetTemplateSizeRequest, gen.GetTemplateSizeResponse]
|
||||
deleteVolume *connect.Client[gen.DeleteVolumeRequest, gen.DeleteVolumeResponse]
|
||||
}
|
||||
|
||||
// CreateSandbox calls hostagent.v1.HostAgentService.CreateSandbox.
|
||||
@ -581,6 +595,11 @@ func (c *hostAgentServiceClient) GetTemplateSize(ctx context.Context, req *conne
|
||||
return c.getTemplateSize.CallUnary(ctx, req)
|
||||
}
|
||||
|
||||
// DeleteVolume calls hostagent.v1.HostAgentService.DeleteVolume.
|
||||
func (c *hostAgentServiceClient) DeleteVolume(ctx context.Context, req *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error) {
|
||||
return c.deleteVolume.CallUnary(ctx, req)
|
||||
}
|
||||
|
||||
// HostAgentServiceHandler is an implementation of the hostagent.v1.HostAgentService service.
|
||||
type HostAgentServiceHandler interface {
|
||||
// CreateSandbox boots a new microVM with the given configuration.
|
||||
@ -658,6 +677,10 @@ type HostAgentServiceHandler interface {
|
||||
// size 0 in the database (e.g. system base templates seeded before the
|
||||
// rootfs was built).
|
||||
GetTemplateSize(context.Context, *connect.Request[gen.GetTemplateSizeRequest]) (*connect.Response[gen.GetTemplateSizeResponse], error)
|
||||
// DeleteVolume removes an external storage volume's backing file from this
|
||||
// host. Called by the control plane when a detached volume is deleted; the
|
||||
// volume must not be attached to any sandbox on this host.
|
||||
DeleteVolume(context.Context, *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error)
|
||||
}
|
||||
|
||||
// NewHostAgentServiceHandler builds an HTTP handler from the service implementation. It returns the
|
||||
@ -847,6 +870,12 @@ func NewHostAgentServiceHandler(svc HostAgentServiceHandler, opts ...connect.Han
|
||||
connect.WithSchema(hostAgentServiceMethods.ByName("GetTemplateSize")),
|
||||
connect.WithHandlerOptions(opts...),
|
||||
)
|
||||
hostAgentServiceDeleteVolumeHandler := connect.NewUnaryHandler(
|
||||
HostAgentServiceDeleteVolumeProcedure,
|
||||
svc.DeleteVolume,
|
||||
connect.WithSchema(hostAgentServiceMethods.ByName("DeleteVolume")),
|
||||
connect.WithHandlerOptions(opts...),
|
||||
)
|
||||
return "/hostagent.v1.HostAgentService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case HostAgentServiceCreateSandboxProcedure:
|
||||
@ -909,6 +938,8 @@ func NewHostAgentServiceHandler(svc HostAgentServiceHandler, opts ...connect.Han
|
||||
hostAgentServiceConnectProcessHandler.ServeHTTP(w, r)
|
||||
case HostAgentServiceGetTemplateSizeProcedure:
|
||||
hostAgentServiceGetTemplateSizeHandler.ServeHTTP(w, r)
|
||||
case HostAgentServiceDeleteVolumeProcedure:
|
||||
hostAgentServiceDeleteVolumeHandler.ServeHTTP(w, r)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
@ -1037,3 +1068,7 @@ func (UnimplementedHostAgentServiceHandler) ConnectProcess(context.Context, *con
|
||||
func (UnimplementedHostAgentServiceHandler) GetTemplateSize(context.Context, *connect.Request[gen.GetTemplateSizeRequest]) (*connect.Response[gen.GetTemplateSizeResponse], error) {
|
||||
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("hostagent.v1.HostAgentService.GetTemplateSize is not implemented"))
|
||||
}
|
||||
|
||||
func (UnimplementedHostAgentServiceHandler) DeleteVolume(context.Context, *connect.Request[gen.DeleteVolumeRequest]) (*connect.Response[gen.DeleteVolumeResponse], error) {
|
||||
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("hostagent.v1.HostAgentService.DeleteVolume is not implemented"))
|
||||
}
|
||||
|
||||
@ -109,6 +109,26 @@ service HostAgentService {
|
||||
// size 0 in the database (e.g. system base templates seeded before the
|
||||
// rootfs was built).
|
||||
rpc GetTemplateSize(GetTemplateSizeRequest) returns (GetTemplateSizeResponse);
|
||||
|
||||
// DeleteVolume removes an external storage volume's backing file from this
|
||||
// host. Called by the control plane when a detached volume is deleted; the
|
||||
// volume must not be attached to any sandbox on this host.
|
||||
rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse);
|
||||
}
|
||||
|
||||
// VolumeSpec describes an external storage volume to attach to a sandbox at
|
||||
// boot time. The host provisions the backing sparse file on first use, hands
|
||||
// it to Cloud Hypervisor as a Raw disk, and asks envd to format (if empty) and
|
||||
// mount it in the guest.
|
||||
message VolumeSpec {
|
||||
// Volume UUID (hex string).
|
||||
string volume_id = 1;
|
||||
// Team UUID that owns the volume (hex string) — locates the backing file.
|
||||
string team_id = 2;
|
||||
// Size of the volume in MB (used to size the sparse file on first attach).
|
||||
int32 size_mb = 3;
|
||||
// Guest mount path. Empty means the default /mnt/<volume-id>.
|
||||
string mount_path = 4;
|
||||
}
|
||||
|
||||
message CreateSandboxRequest {
|
||||
@ -142,6 +162,9 @@ message CreateSandboxRequest {
|
||||
|
||||
// Default environment variables (set in envd via PostInit).
|
||||
map<string, string> default_env = 10;
|
||||
|
||||
// External storage volumes to attach at boot time and mount in the guest.
|
||||
repeated VolumeSpec volumes = 11;
|
||||
}
|
||||
|
||||
message CreateSandboxResponse {
|
||||
@ -464,6 +487,15 @@ message GetTemplateSizeResponse {
|
||||
int64 size_bytes = 1;
|
||||
}
|
||||
|
||||
message DeleteVolumeRequest {
|
||||
// Team UUID that owns the volume (hex string).
|
||||
string team_id = 1;
|
||||
// Volume UUID (hex string).
|
||||
string volume_id = 2;
|
||||
}
|
||||
|
||||
message DeleteVolumeResponse {}
|
||||
|
||||
// ── PTY ─────────────────────────────────────────────────────────────
|
||||
|
||||
message PtyAttachRequest {
|
||||
|
||||
Reference in New Issue
Block a user