forked from wrenn/wrenn
Add tini as PID 1, guest clock sync, and fix PATH in guest VMs
- Use tini as PID 1 in wrenn-init.sh so zombie processes are reaped and signals are forwarded correctly to envd - Set standard PATH in wrenn-init.sh so child processes spawned by envd can find common binaries (fixes "nice: ls command not found") - Add envdclient.Init() to POST /init on envd after every boot/resume, syncing the guest clock via unix.ClockSettime — critical after snapshot resume where the guest clock is frozen - Run Init in a background goroutine so it doesn't block the CreateSandbox RPC response; a slow Init (vCPU busy with envd startup) was causing the RPC context to be canceled before the response reached the control plane - Update rootfs-from-container.sh and update-debug-rootfs.sh to inject tini into the rootfs, checking the container image and host first, downloading from GitHub releases as fallback
This commit is contained in:
@ -23,5 +23,8 @@ hostname sandbox
|
|||||||
echo "nameserver 8.8.8.8" > /etc/resolv.conf
|
echo "nameserver 8.8.8.8" > /etc/resolv.conf
|
||||||
echo "nameserver 8.8.4.4" >> /etc/resolv.conf
|
echo "nameserver 8.8.4.4" >> /etc/resolv.conf
|
||||||
|
|
||||||
# Exec envd as the main process (replaces this script, keeps PID 1).
|
# Set a standard PATH so envd and all child processes can find common binaries.
|
||||||
exec /usr/local/bin/envd
|
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||||
|
|
||||||
|
# Exec tini as PID 1 — it reaps zombie processes and forwards signals to envd.
|
||||||
|
exec /sbin/tini -- /usr/local/bin/envd
|
||||||
|
|||||||
@ -3,12 +3,14 @@ package envdclient
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"time"
|
||||||
|
|
||||||
"connectrpc.com/connect"
|
"connectrpc.com/connect"
|
||||||
|
|
||||||
@ -47,6 +49,35 @@ func (c *Client) BaseURL() string {
|
|||||||
return c.base
|
return c.base
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Init calls POST /init on envd to sync the guest clock with the host.
|
||||||
|
// This is important after snapshot resume where the guest clock is frozen.
|
||||||
|
func (c *Client) Init(ctx context.Context) error {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
body, err := json.Marshal(map[string]any{"timestamp": now})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal init body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/init", bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create init request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := c.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("init request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
respBody, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("init: status %d: %s", resp.StatusCode, string(respBody))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ExecResult holds the output of a command execution.
|
// ExecResult holds the output of a command execution.
|
||||||
type ExecResult struct {
|
type ExecResult struct {
|
||||||
Stdout []byte
|
Stdout []byte
|
||||||
|
|||||||
@ -197,6 +197,16 @@ func (m *Manager) Create(ctx context.Context, sandboxID, template string, vcpus,
|
|||||||
return nil, fmt.Errorf("wait for envd: %w", err)
|
return nil, fmt.Errorf("wait for envd: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sync guest clock in background. Non-fatal — sandbox is usable before this completes.
|
||||||
|
// Run in a goroutine so Init latency doesn't block the RPC response back to the control plane.
|
||||||
|
go func() {
|
||||||
|
initCtx, initCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer initCancel()
|
||||||
|
if err := client.Init(initCtx); err != nil {
|
||||||
|
slog.Warn("envd init (clock sync) failed", "sandbox", sandboxID, "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
sb := &sandboxState{
|
sb := &sandboxState{
|
||||||
Sandbox: models.Sandbox{
|
Sandbox: models.Sandbox{
|
||||||
@ -617,6 +627,16 @@ func (m *Manager) Resume(ctx context.Context, sandboxID string, timeoutSec int)
|
|||||||
return nil, fmt.Errorf("wait for envd: %w", err)
|
return nil, fmt.Errorf("wait for envd: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sync guest clock in background. Non-fatal — sandbox is usable before this completes.
|
||||||
|
// Run in a goroutine so Init latency doesn't block the RPC response back to the control plane.
|
||||||
|
go func() {
|
||||||
|
initCtx, initCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer initCancel()
|
||||||
|
if err := client.Init(initCtx); err != nil {
|
||||||
|
slog.Warn("envd init (clock sync) failed", "sandbox", sandboxID, "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
sb := &sandboxState{
|
sb := &sandboxState{
|
||||||
Sandbox: models.Sandbox{
|
Sandbox: models.Sandbox{
|
||||||
@ -926,6 +946,16 @@ func (m *Manager) createFromSnapshot(ctx context.Context, sandboxID, snapshotNam
|
|||||||
return nil, fmt.Errorf("wait for envd: %w", err)
|
return nil, fmt.Errorf("wait for envd: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sync guest clock in background. Non-fatal — sandbox is usable before this completes.
|
||||||
|
// Run in a goroutine so Init latency doesn't block the RPC response back to the control plane.
|
||||||
|
go func() {
|
||||||
|
initCtx, initCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer initCancel()
|
||||||
|
if err := client.Init(initCtx); err != nil {
|
||||||
|
slog.Warn("envd init (clock sync) failed", "sandbox", sandboxID, "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
sb := &sandboxState{
|
sb := &sandboxState{
|
||||||
Sandbox: models.Sandbox{
|
Sandbox: models.Sandbox{
|
||||||
|
|||||||
@ -15,7 +15,7 @@
|
|||||||
# Output:
|
# Output:
|
||||||
# ${AGENT_FILES_ROOTDIR}/images/<image_name>/rootfs.ext4
|
# ${AGENT_FILES_ROOTDIR}/images/<image_name>/rootfs.ext4
|
||||||
#
|
#
|
||||||
# Requires: docker, mkfs.ext4, resize2fs, e2fsck, make (for building envd)
|
# Requires: docker, mkfs.ext4, resize2fs, e2fsck, make (for building envd), curl (for tini download)
|
||||||
# Sudo is used only for mount/umount/copy-into-image operations.
|
# Sudo is used only for mount/umount/copy-into-image operations.
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
@ -98,10 +98,42 @@ echo "==> Installing wrenn-init..."
|
|||||||
sudo cp "${PROJECT_ROOT}/images/wrenn-init.sh" "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
sudo cp "${PROJECT_ROOT}/images/wrenn-init.sh" "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
||||||
sudo chmod 755 "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
sudo chmod 755 "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
||||||
|
|
||||||
|
echo "==> Installing tini..."
|
||||||
|
TINI_BIN=""
|
||||||
|
# 1. Already in the exported container image?
|
||||||
|
for p in "${MOUNT_DIR}/usr/bin/tini" "${MOUNT_DIR}/sbin/tini" "${MOUNT_DIR}/usr/local/bin/tini"; do
|
||||||
|
if [ -f "$p" ]; then TINI_BIN="$p"; break; fi
|
||||||
|
done
|
||||||
|
# 2. Available on the host?
|
||||||
|
if [ -z "${TINI_BIN}" ]; then
|
||||||
|
for p in /usr/bin/tini /usr/local/bin/tini /sbin/tini; do
|
||||||
|
if [ -f "$p" ]; then TINI_BIN="$p"; break; fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
# 3. Download from GitHub releases.
|
||||||
|
if [ -z "${TINI_BIN}" ]; then
|
||||||
|
ARCH="$(uname -m)"
|
||||||
|
case "${ARCH}" in
|
||||||
|
x86_64) TINI_ARCH="amd64" ;;
|
||||||
|
aarch64) TINI_ARCH="arm64" ;;
|
||||||
|
*) echo "ERROR: Unsupported architecture: ${ARCH}"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
TINI_VERSION="v0.19.0"
|
||||||
|
TINI_URL="https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TINI_ARCH}"
|
||||||
|
TINI_TMP="/tmp/tini-${TINI_ARCH}"
|
||||||
|
echo " Downloading tini ${TINI_VERSION} (${TINI_ARCH})..."
|
||||||
|
curl -fsSL "${TINI_URL}" -o "${TINI_TMP}"
|
||||||
|
chmod +x "${TINI_TMP}"
|
||||||
|
TINI_BIN="${TINI_TMP}"
|
||||||
|
fi
|
||||||
|
sudo mkdir -p "${MOUNT_DIR}/sbin"
|
||||||
|
sudo cp "${TINI_BIN}" "${MOUNT_DIR}/sbin/tini"
|
||||||
|
sudo chmod 755 "${MOUNT_DIR}/sbin/tini"
|
||||||
|
|
||||||
# Step 6: Verify.
|
# Step 6: Verify.
|
||||||
echo ""
|
echo ""
|
||||||
echo "==> Installed guest binaries:"
|
echo "==> Installed guest binaries:"
|
||||||
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init" "${MOUNT_DIR}/sbin/tini"
|
||||||
|
|
||||||
# Unmount before shrinking.
|
# Unmount before shrinking.
|
||||||
sudo umount "${MOUNT_DIR}"
|
sudo umount "${MOUNT_DIR}"
|
||||||
|
|||||||
@ -11,13 +11,13 @@
|
|||||||
# Usage:
|
# Usage:
|
||||||
# bash scripts/update-debug-rootfs.sh [rootfs_path]
|
# bash scripts/update-debug-rootfs.sh [rootfs_path]
|
||||||
#
|
#
|
||||||
# Defaults to /var/lib/wrenn/images/minimal.ext4
|
# Defaults to /var/lib/wrenn/images/minimal/rootfs.ext4
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
ROOTFS="${1:-/var/lib/wrenn/images/minimal.ext4}"
|
ROOTFS="${1:-/var/lib/wrenn/images/minimal/rootfs.ext4}"
|
||||||
MOUNT_DIR="/tmp/wrenn-rootfs-update"
|
MOUNT_DIR="/tmp/wrenn-rootfs-update"
|
||||||
|
|
||||||
if [ ! -f "${ROOTFS}" ]; then
|
if [ ! -f "${ROOTFS}" ]; then
|
||||||
@ -64,10 +64,42 @@ echo "==> Installing wrenn-init..."
|
|||||||
sudo cp "${PROJECT_ROOT}/images/wrenn-init.sh" "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
sudo cp "${PROJECT_ROOT}/images/wrenn-init.sh" "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
||||||
sudo chmod 755 "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
sudo chmod 755 "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
||||||
|
|
||||||
|
echo "==> Installing tini..."
|
||||||
|
TINI_BIN=""
|
||||||
|
# 1. Already in the rootfs?
|
||||||
|
for p in "${MOUNT_DIR}/usr/bin/tini" "${MOUNT_DIR}/sbin/tini" "${MOUNT_DIR}/usr/local/bin/tini"; do
|
||||||
|
if [ -f "$p" ]; then TINI_BIN="$p"; break; fi
|
||||||
|
done
|
||||||
|
# 2. Available on the host?
|
||||||
|
if [ -z "${TINI_BIN}" ]; then
|
||||||
|
for p in /usr/bin/tini /usr/local/bin/tini /sbin/tini; do
|
||||||
|
if [ -f "$p" ]; then TINI_BIN="$p"; break; fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
# 3. Download from GitHub releases.
|
||||||
|
if [ -z "${TINI_BIN}" ]; then
|
||||||
|
ARCH="$(uname -m)"
|
||||||
|
case "${ARCH}" in
|
||||||
|
x86_64) TINI_ARCH="amd64" ;;
|
||||||
|
aarch64) TINI_ARCH="arm64" ;;
|
||||||
|
*) echo "ERROR: Unsupported architecture: ${ARCH}"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
TINI_VERSION="v0.19.0"
|
||||||
|
TINI_URL="https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TINI_ARCH}"
|
||||||
|
TINI_TMP="/tmp/tini-${TINI_ARCH}"
|
||||||
|
echo " Downloading tini ${TINI_VERSION} (${TINI_ARCH})..."
|
||||||
|
curl -fsSL "${TINI_URL}" -o "${TINI_TMP}"
|
||||||
|
chmod +x "${TINI_TMP}"
|
||||||
|
TINI_BIN="${TINI_TMP}"
|
||||||
|
fi
|
||||||
|
sudo mkdir -p "${MOUNT_DIR}/sbin"
|
||||||
|
sudo cp "${TINI_BIN}" "${MOUNT_DIR}/sbin/tini"
|
||||||
|
sudo chmod 755 "${MOUNT_DIR}/sbin/tini"
|
||||||
|
|
||||||
# Step 4: Verify.
|
# Step 4: Verify.
|
||||||
echo ""
|
echo ""
|
||||||
echo "==> Installed files:"
|
echo "==> Installed files:"
|
||||||
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init"
|
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init" "${MOUNT_DIR}/sbin/tini"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "==> Done. Rootfs updated: ${ROOTFS}"
|
echo "==> Done. Rootfs updated: ${ROOTFS}"
|
||||||
|
|||||||
Reference in New Issue
Block a user