forked from wrenn/wrenn
Add USER, COPY, ENV persistence to template build system
Implement three new recipe commands for the admin template builder: - USER <name>: creates the user (adduser + passwordless sudo), switches execution context so subsequent RUN/START commands run as that user via su wrapping. Last USER becomes the template's default_user. - COPY <src> <dst>: copies files from an uploaded build archive (tar/tar.gz/zip) into the sandbox. Source paths validated against traversal. Ownership set to the current USER. - ENV persistence: accumulated env vars stored in templates.default_env (JSONB) and injected via PostInit when sandboxes are created from the template, mirroring Docker's image metadata approach. Supporting changes: - Pre-build creates wrenn-user as default (via USER command) - WORKDIR now creates the directory if it doesn't exist (mkdir -p) - Per-step progress updates (ProgressFunc callback) for live UI - Multipart form support on POST /v1/admin/builds for archive upload - Proto: default_user/default_env fields on Create/ResumeSandboxRequest - Host agent: SetDefaults calls PostInitWithDefaults on envd - Control plane: reads template defaults, passes on sandbox create/resume - Frontend: file upload widget, recipe copy button, keyword colors for USER/COPY, fixed Svelte whitespace stripping in step display - Admin panel defaults to /admin/templates instead of /admin/hosts - Migration adds default_user and default_env to templates and template_builds tables
This commit is contained in:
@ -7,10 +7,11 @@ import (
|
||||
)
|
||||
|
||||
// ExecContext holds mutable state that persists across recipe steps.
|
||||
// It is initialized empty and updated by ENV and WORKDIR steps.
|
||||
// It is initialized empty and updated by ENV, WORKDIR, and USER steps.
|
||||
type ExecContext struct {
|
||||
WorkDir string
|
||||
EnvVars map[string]string
|
||||
User string // Current unix user for command execution.
|
||||
}
|
||||
|
||||
// This regex matches:
|
||||
@ -25,7 +26,20 @@ var envRegex = regexp.MustCompile(`\$\$|\$\{([a-zA-Z0-9_]*)\}|\$([a-zA-Z0-9_]+)`
|
||||
// If WORKDIR and/or ENV are set, they are prepended as a shell preamble:
|
||||
//
|
||||
// cd '/the/dir' && KEY='val' /bin/sh -c 'original command'
|
||||
//
|
||||
// If USER is set to a non-root user, the entire command is wrapped with su:
|
||||
//
|
||||
// su <user> -s /bin/sh -c '<preamble + command>'
|
||||
func (c *ExecContext) WrappedCommand(cmd string) string {
|
||||
inner := c.innerCommand(cmd)
|
||||
if c.User != "" && c.User != "root" {
|
||||
return "su " + shellescape(c.User) + " -s /bin/sh -c " + shellescape(inner)
|
||||
}
|
||||
return inner
|
||||
}
|
||||
|
||||
// innerCommand builds the command with workdir/env preamble but without user wrapping.
|
||||
func (c *ExecContext) innerCommand(cmd string) string {
|
||||
prefix := c.shellPrefix()
|
||||
if prefix == "" {
|
||||
return cmd
|
||||
@ -42,7 +56,11 @@ func (c *ExecContext) WrappedCommand(cmd string) string {
|
||||
// simultaneously before a healthcheck is evaluated.
|
||||
func (c *ExecContext) StartCommand(cmd string) string {
|
||||
prefix := c.shellPrefix()
|
||||
return prefix + "nohup /bin/sh -c " + shellescape(cmd) + " >/dev/null 2>&1 &"
|
||||
inner := prefix + "nohup /bin/sh -c " + shellescape(cmd) + " >/dev/null 2>&1 &"
|
||||
if c.User != "" && c.User != "root" {
|
||||
return "su " + shellescape(c.User) + " -s /bin/sh -c " + shellescape(inner)
|
||||
}
|
||||
return inner
|
||||
}
|
||||
|
||||
// shellPrefix builds the "cd ... && KEY=val " preamble for a shell command.
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -16,6 +17,10 @@ import (
|
||||
// explicit --timeout flag.
|
||||
const DefaultStepTimeout = 30 * time.Second
|
||||
|
||||
// BuildFilesDir is the directory inside the sandbox where uploaded build
|
||||
// archives are extracted. COPY instructions reference paths relative to this.
|
||||
const BuildFilesDir = "/tmp/build-files"
|
||||
|
||||
// BuildLogEntry is the per-step record stored in template_builds.logs (JSONB).
|
||||
type BuildLogEntry struct {
|
||||
Step int `json:"step"`
|
||||
@ -32,13 +37,18 @@ type BuildLogEntry struct {
|
||||
// the method on the hostagent Connect RPC client.
|
||||
type ExecFunc func(ctx context.Context, req *connect.Request[pb.ExecRequest]) (*connect.Response[pb.ExecResponse], error)
|
||||
|
||||
// ProgressFunc is called after each step with the current step counter and
|
||||
// accumulated log entries. Used for per-step DB progress updates.
|
||||
type ProgressFunc func(step int, entries []BuildLogEntry)
|
||||
|
||||
// Execute runs steps sequentially against sandboxID using execFn.
|
||||
//
|
||||
// - phase labels the log entries (e.g., "pre-build", "recipe", "post-build").
|
||||
// - startStep is the 1-based offset so entries are globally numbered across phases.
|
||||
// - defaultTimeout applies to RUN steps with no per-step --timeout; 0 → 10 minutes.
|
||||
// - bctx is mutated in place as ENV/WORKDIR steps execute, and carries forward
|
||||
// - bctx is mutated in place as ENV/WORKDIR/USER steps execute, and carries forward
|
||||
// into subsequent phases when the caller passes the same pointer.
|
||||
// - onProgress is called after each step for live progress updates (may be nil).
|
||||
//
|
||||
// Returns all log entries appended during this call, the next step counter
|
||||
// value, and whether all steps succeeded. On false the last entry contains
|
||||
@ -53,6 +63,7 @@ func Execute(
|
||||
defaultTimeout time.Duration,
|
||||
bctx *ExecContext,
|
||||
execFn ExecFunc,
|
||||
onProgress ProgressFunc,
|
||||
) (entries []BuildLogEntry, nextStep int, ok bool) {
|
||||
if defaultTimeout <= 0 {
|
||||
defaultTimeout = 10 * time.Minute
|
||||
@ -72,19 +83,30 @@ func Execute(
|
||||
entries = append(entries, BuildLogEntry{Step: step, Phase: phase, Cmd: st.Raw, Ok: true})
|
||||
|
||||
case KindWORKDIR:
|
||||
// Create the directory if it doesn't exist.
|
||||
mkdirEntry := execRawShell(ctx, st.Raw, sandboxID, phase, step, 10*time.Second, execFn,
|
||||
"mkdir -p "+shellescape(st.Path))
|
||||
if !mkdirEntry.Ok {
|
||||
entries = append(entries, mkdirEntry)
|
||||
return entries, step, false
|
||||
}
|
||||
bctx.WorkDir = st.Path
|
||||
entries = append(entries, BuildLogEntry{Step: step, Phase: phase, Cmd: st.Raw, Ok: true})
|
||||
mkdirEntry.Ok = true
|
||||
entries = append(entries, mkdirEntry)
|
||||
|
||||
case KindUSER, KindCOPY:
|
||||
verb := strings.ToUpper(strings.Fields(st.Raw)[0])
|
||||
entries = append(entries, BuildLogEntry{
|
||||
Step: step,
|
||||
Phase: phase,
|
||||
Cmd: st.Raw,
|
||||
Stderr: verb + " is not yet supported",
|
||||
Ok: false,
|
||||
})
|
||||
return entries, step, false
|
||||
case KindUSER:
|
||||
entry, succeeded := execUser(ctx, st, sandboxID, phase, step, bctx, execFn)
|
||||
entries = append(entries, entry)
|
||||
if !succeeded {
|
||||
return entries, step, false
|
||||
}
|
||||
|
||||
case KindCOPY:
|
||||
entry, succeeded := execCopy(ctx, st, sandboxID, phase, step, bctx, execFn)
|
||||
entries = append(entries, entry)
|
||||
if !succeeded {
|
||||
return entries, step, false
|
||||
}
|
||||
|
||||
case KindSTART:
|
||||
entry, succeeded := execStart(ctx, st, sandboxID, phase, step, bctx, execFn)
|
||||
@ -104,6 +126,10 @@ func Execute(
|
||||
return entries, step, false
|
||||
}
|
||||
}
|
||||
|
||||
if onProgress != nil {
|
||||
onProgress(step, entries)
|
||||
}
|
||||
}
|
||||
return entries, step, true
|
||||
}
|
||||
@ -145,6 +171,106 @@ func execRun(
|
||||
return entry, entry.Ok
|
||||
}
|
||||
|
||||
// execUser creates a unix user (if not exists), grants passwordless sudo,
|
||||
// and updates bctx.User for subsequent steps.
|
||||
func execUser(
|
||||
ctx context.Context,
|
||||
st Step,
|
||||
sandboxID, phase string,
|
||||
step int,
|
||||
bctx *ExecContext,
|
||||
execFn ExecFunc,
|
||||
) (BuildLogEntry, bool) {
|
||||
username := st.Key
|
||||
// Create user if not exists, with home directory and bash shell.
|
||||
// Grant passwordless sudo access (E2B convention).
|
||||
// Uses printf %s to avoid shell injection in the sudoers line.
|
||||
script := fmt.Sprintf(
|
||||
"id %s >/dev/null 2>&1 || (adduser --disabled-password --gecos '' --shell /bin/bash %s && printf '%%s ALL=(ALL) NOPASSWD:ALL\\n' %s >> /etc/sudoers)",
|
||||
shellescape(username), shellescape(username), shellescape(username),
|
||||
)
|
||||
|
||||
entry := execRawShell(ctx, st.Raw, sandboxID, phase, step, 30*time.Second, execFn, script)
|
||||
if entry.Ok {
|
||||
bctx.User = username
|
||||
}
|
||||
return entry, entry.Ok
|
||||
}
|
||||
|
||||
// execCopy copies a file or directory from the build archive (extracted at
|
||||
// BuildFilesDir) to the destination path inside the sandbox. Ownership is
|
||||
// set to the current user from bctx.
|
||||
func execCopy(
|
||||
ctx context.Context,
|
||||
st Step,
|
||||
sandboxID, phase string,
|
||||
step int,
|
||||
bctx *ExecContext,
|
||||
execFn ExecFunc,
|
||||
) (BuildLogEntry, bool) {
|
||||
// Validate source path: must be relative and not escape the archive directory.
|
||||
cleaned := path.Clean(st.Src)
|
||||
if strings.HasPrefix(cleaned, "..") || strings.HasPrefix(cleaned, "/") {
|
||||
return BuildLogEntry{
|
||||
Step: step,
|
||||
Phase: phase,
|
||||
Cmd: st.Raw,
|
||||
Stderr: "COPY source must be a relative path within the archive",
|
||||
}, false
|
||||
}
|
||||
src := BuildFilesDir + "/" + cleaned
|
||||
dst := st.Dst
|
||||
owner := "root"
|
||||
if bctx.User != "" {
|
||||
owner = bctx.User
|
||||
}
|
||||
script := fmt.Sprintf(
|
||||
"cp -r %s %s && chown -R %s:%s %s",
|
||||
shellescape(src), shellescape(dst), shellescape(owner), shellescape(owner), shellescape(dst),
|
||||
)
|
||||
|
||||
entry := execRawShell(ctx, st.Raw, sandboxID, phase, step, 60*time.Second, execFn, script)
|
||||
return entry, entry.Ok
|
||||
}
|
||||
|
||||
// execRawShell runs a shell command directly (as root) without ExecContext
|
||||
// wrapping. Used for internal operations like user creation and file copy.
|
||||
func execRawShell(
|
||||
ctx context.Context,
|
||||
raw, sandboxID, phase string,
|
||||
step int,
|
||||
timeout time.Duration,
|
||||
execFn ExecFunc,
|
||||
shellCmd string,
|
||||
) BuildLogEntry {
|
||||
execCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
resp, err := execFn(execCtx, connect.NewRequest(&pb.ExecRequest{
|
||||
SandboxId: sandboxID,
|
||||
Cmd: "/bin/sh",
|
||||
Args: []string{"-c", shellCmd},
|
||||
TimeoutSec: int32(timeout.Seconds()),
|
||||
}))
|
||||
|
||||
entry := BuildLogEntry{
|
||||
Step: step,
|
||||
Phase: phase,
|
||||
Cmd: raw,
|
||||
Elapsed: time.Since(start).Milliseconds(),
|
||||
}
|
||||
if err != nil {
|
||||
entry.Stderr = fmt.Sprintf("exec error: %v", err)
|
||||
return entry
|
||||
}
|
||||
entry.Stdout = string(resp.Msg.Stdout)
|
||||
entry.Stderr = string(resp.Msg.Stderr)
|
||||
entry.Exit = resp.Msg.ExitCode
|
||||
entry.Ok = resp.Msg.ExitCode == 0
|
||||
return entry
|
||||
}
|
||||
|
||||
func execStart(
|
||||
ctx context.Context,
|
||||
st Step,
|
||||
|
||||
@ -24,9 +24,11 @@ type Step struct {
|
||||
Raw string // original string, preserved for logging
|
||||
Shell string // KindRUN, KindSTART: the shell command text
|
||||
Timeout time.Duration // KindRUN: 0 means use caller's default
|
||||
Key string // KindENV: variable name
|
||||
Key string // KindENV: variable name; KindUSER: username
|
||||
Value string // KindENV: variable value
|
||||
Path string // KindWORKDIR: directory path
|
||||
Src string // KindCOPY: source path (relative to build archive)
|
||||
Dst string // KindCOPY: destination path inside sandbox
|
||||
}
|
||||
|
||||
// ParseStep parses a single recipe instruction string into a Step.
|
||||
@ -61,9 +63,9 @@ func ParseStep(s string) (Step, error) {
|
||||
case "WORKDIR":
|
||||
return parseWORKDIR(s, rest)
|
||||
case "USER":
|
||||
return Step{Kind: KindUSER, Raw: s}, nil
|
||||
return parseUSER(s, rest)
|
||||
case "COPY":
|
||||
return Step{Kind: KindCOPY, Raw: s}, nil
|
||||
return parseCOPY(s, rest)
|
||||
default:
|
||||
return Step{}, fmt.Errorf("unknown instruction %q (expected RUN, START, ENV, WORKDIR, USER, or COPY)", keyword)
|
||||
}
|
||||
@ -127,3 +129,31 @@ func parseWORKDIR(raw, path string) (Step, error) {
|
||||
}
|
||||
return Step{Kind: KindWORKDIR, Raw: raw, Path: path}, nil
|
||||
}
|
||||
|
||||
func parseUSER(raw, username string) (Step, error) {
|
||||
if username == "" {
|
||||
return Step{}, fmt.Errorf("USER requires a username: %q", raw)
|
||||
}
|
||||
// Validate: alphanumeric, hyphens, underscores only; must start with a letter or underscore.
|
||||
for i, c := range username {
|
||||
if i == 0 && !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') {
|
||||
return Step{}, fmt.Errorf("USER username must start with a letter or underscore: %q", raw)
|
||||
}
|
||||
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') {
|
||||
return Step{}, fmt.Errorf("USER username contains invalid character %q: %q", string(c), raw)
|
||||
}
|
||||
}
|
||||
return Step{Kind: KindUSER, Raw: raw, Key: username}, nil
|
||||
}
|
||||
|
||||
func parseCOPY(raw, rest string) (Step, error) {
|
||||
if rest == "" {
|
||||
return Step{}, fmt.Errorf("COPY requires <src> <dst>: %q", raw)
|
||||
}
|
||||
src, dst, found := strings.Cut(rest, " ")
|
||||
dst = strings.TrimSpace(dst)
|
||||
if !found || dst == "" {
|
||||
return Step{}, fmt.Errorf("COPY requires <src> <dst>: %q", raw)
|
||||
}
|
||||
return Step{Kind: KindCOPY, Raw: raw, Src: src, Dst: dst}, nil
|
||||
}
|
||||
|
||||
@ -111,16 +111,37 @@ func TestParseStep(t *testing.T) {
|
||||
input: "WORKDIR",
|
||||
wantErr: true,
|
||||
},
|
||||
// USER and COPY stubs
|
||||
// USER
|
||||
{
|
||||
name: "USER stub",
|
||||
name: "USER basic",
|
||||
input: "USER www-data",
|
||||
want: Step{Kind: KindUSER, Raw: "USER www-data"},
|
||||
want: Step{Kind: KindUSER, Raw: "USER www-data", Key: "www-data"},
|
||||
},
|
||||
{
|
||||
name: "COPY stub",
|
||||
name: "USER empty",
|
||||
input: "USER",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "USER invalid chars",
|
||||
input: "USER bad user",
|
||||
wantErr: true,
|
||||
},
|
||||
// COPY
|
||||
{
|
||||
name: "COPY basic",
|
||||
input: "COPY config.yaml /etc/app/config.yaml",
|
||||
want: Step{Kind: KindCOPY, Raw: "COPY config.yaml /etc/app/config.yaml"},
|
||||
want: Step{Kind: KindCOPY, Raw: "COPY config.yaml /etc/app/config.yaml", Src: "config.yaml", Dst: "/etc/app/config.yaml"},
|
||||
},
|
||||
{
|
||||
name: "COPY missing dst",
|
||||
input: "COPY config.yaml",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "COPY empty",
|
||||
input: "COPY",
|
||||
wantErr: true,
|
||||
},
|
||||
// Unknown keyword
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user