forked from wrenn/wrenn
Implement host registration, JWT refresh tokens, and multi-host scheduling
Replaces the hardcoded CP_HOST_AGENT_ADDR single-agent setup with a DB-driven registration system supporting multiple host agents (BYOC). Key changes: - Host agents register via one-time token, receive a 7-day JWT + 60-day refresh token; heartbeat loop auto-refreshes on 401/403 and pauses all sandboxes if refresh fails - HostClientPool: lazy Connect RPC client cache keyed by host ID, replacing the single static agent client throughout the API and service layers - RoundRobinScheduler: picks an online host for each new sandbox via ListActiveHosts; extensible for future scheduling strategies - HostMonitor (replaces Reconciler): passive heartbeat staleness check marks hosts unreachable and sandboxes missing after 90s; active reconciliation per online host restores missing-but-alive sandboxes and stops orphans - Graceful host delete: returns 409 with affected sandbox list without ?force=true; force-delete destroys sandboxes then evicts pool client - Snapshot delete broadcasts to all online hosts (templates have no host_id) - sandbox.Manager.PauseAll: pauses all running VMs on CP connectivity loss - New migration: host_refresh_tokens table with token rotation (issue-then- revoke ordering to prevent lockout on mid-rotation crash) - New sandbox status 'missing' (reversible, unlike 'stopped') and host status 'unreachable'; both reflected in OpenAPI spec - Fix: refresh token auth failure now returns 401 (was 400 via generic 'invalid' substring match in serviceErrToHTTP)
This commit is contained in:
92
internal/db/host_refresh_tokens.sql.go
Normal file
92
internal/db/host_refresh_tokens.sql.go
Normal file
@ -0,0 +1,92 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.30.0
|
||||
// source: host_refresh_tokens.sql
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const deleteExpiredHostRefreshTokens = `-- name: DeleteExpiredHostRefreshTokens :exec
|
||||
DELETE FROM host_refresh_tokens
|
||||
WHERE expires_at < NOW() OR revoked_at IS NOT NULL
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteExpiredHostRefreshTokens(ctx context.Context) error {
|
||||
_, err := q.db.Exec(ctx, deleteExpiredHostRefreshTokens)
|
||||
return err
|
||||
}
|
||||
|
||||
const getHostRefreshTokenByHash = `-- name: GetHostRefreshTokenByHash :one
|
||||
SELECT id, host_id, token_hash, expires_at, created_at, revoked_at FROM host_refresh_tokens
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > NOW()
|
||||
`
|
||||
|
||||
func (q *Queries) GetHostRefreshTokenByHash(ctx context.Context, tokenHash string) (HostRefreshToken, error) {
|
||||
row := q.db.QueryRow(ctx, getHostRefreshTokenByHash, tokenHash)
|
||||
var i HostRefreshToken
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.HostID,
|
||||
&i.TokenHash,
|
||||
&i.ExpiresAt,
|
||||
&i.CreatedAt,
|
||||
&i.RevokedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const insertHostRefreshToken = `-- name: InsertHostRefreshToken :one
|
||||
INSERT INTO host_refresh_tokens (id, host_id, token_hash, expires_at)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING id, host_id, token_hash, expires_at, created_at, revoked_at
|
||||
`
|
||||
|
||||
type InsertHostRefreshTokenParams struct {
|
||||
ID string `json:"id"`
|
||||
HostID string `json:"host_id"`
|
||||
TokenHash string `json:"token_hash"`
|
||||
ExpiresAt pgtype.Timestamptz `json:"expires_at"`
|
||||
}
|
||||
|
||||
func (q *Queries) InsertHostRefreshToken(ctx context.Context, arg InsertHostRefreshTokenParams) (HostRefreshToken, error) {
|
||||
row := q.db.QueryRow(ctx, insertHostRefreshToken,
|
||||
arg.ID,
|
||||
arg.HostID,
|
||||
arg.TokenHash,
|
||||
arg.ExpiresAt,
|
||||
)
|
||||
var i HostRefreshToken
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.HostID,
|
||||
&i.TokenHash,
|
||||
&i.ExpiresAt,
|
||||
&i.CreatedAt,
|
||||
&i.RevokedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const revokeHostRefreshToken = `-- name: RevokeHostRefreshToken :exec
|
||||
UPDATE host_refresh_tokens SET revoked_at = NOW() WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) RevokeHostRefreshToken(ctx context.Context, id string) error {
|
||||
_, err := q.db.Exec(ctx, revokeHostRefreshToken, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const revokeHostRefreshTokensByHost = `-- name: RevokeHostRefreshTokensByHost :exec
|
||||
UPDATE host_refresh_tokens SET revoked_at = NOW()
|
||||
WHERE host_id = $1 AND revoked_at IS NULL
|
||||
`
|
||||
|
||||
func (q *Queries) RevokeHostRefreshTokensByHost(ctx context.Context, hostID string) error {
|
||||
_, err := q.db.Exec(ctx, revokeHostRefreshTokensByHost, hostID)
|
||||
return err
|
||||
}
|
||||
@ -234,6 +234,50 @@ func (q *Queries) InsertHostToken(ctx context.Context, arg InsertHostTokenParams
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listActiveHosts = `-- name: ListActiveHosts :many
|
||||
SELECT id, type, team_id, provider, availability_zone, arch, cpu_cores, memory_mb, disk_gb, address, status, last_heartbeat_at, metadata, created_by, created_at, updated_at, cert_fingerprint, mtls_enabled FROM hosts WHERE status NOT IN ('pending', 'offline') ORDER BY created_at
|
||||
`
|
||||
|
||||
// Returns all hosts that have completed registration (not pending/offline).
|
||||
func (q *Queries) ListActiveHosts(ctx context.Context) ([]Host, error) {
|
||||
rows, err := q.db.Query(ctx, listActiveHosts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Host
|
||||
for rows.Next() {
|
||||
var i Host
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Type,
|
||||
&i.TeamID,
|
||||
&i.Provider,
|
||||
&i.AvailabilityZone,
|
||||
&i.Arch,
|
||||
&i.CpuCores,
|
||||
&i.MemoryMb,
|
||||
&i.DiskGb,
|
||||
&i.Address,
|
||||
&i.Status,
|
||||
&i.LastHeartbeatAt,
|
||||
&i.Metadata,
|
||||
&i.CreatedBy,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.CertFingerprint,
|
||||
&i.MtlsEnabled,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listHosts = `-- name: ListHosts :many
|
||||
SELECT id, type, team_id, provider, availability_zone, arch, cpu_cores, memory_mb, disk_gb, address, status, last_heartbeat_at, metadata, created_by, created_at, updated_at, cert_fingerprint, mtls_enabled FROM hosts ORDER BY created_at DESC
|
||||
`
|
||||
@ -461,6 +505,15 @@ func (q *Queries) MarkHostTokenUsed(ctx context.Context, id string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
const markHostUnreachable = `-- name: MarkHostUnreachable :exec
|
||||
UPDATE hosts SET status = 'unreachable', updated_at = NOW() WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) MarkHostUnreachable(ctx context.Context, id string) error {
|
||||
_, err := q.db.Exec(ctx, markHostUnreachable, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const registerHost = `-- name: RegisterHost :execrows
|
||||
UPDATE hosts
|
||||
SET arch = $2,
|
||||
@ -521,6 +574,20 @@ func (q *Queries) UpdateHostHeartbeat(ctx context.Context, id string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
const updateHostHeartbeatAndStatus = `-- name: UpdateHostHeartbeatAndStatus :exec
|
||||
UPDATE hosts
|
||||
SET last_heartbeat_at = NOW(),
|
||||
status = CASE WHEN status = 'unreachable' THEN 'online' ELSE status END,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
// Updates last_heartbeat_at and transitions unreachable hosts back to online.
|
||||
func (q *Queries) UpdateHostHeartbeatAndStatus(ctx context.Context, id string) error {
|
||||
_, err := q.db.Exec(ctx, updateHostHeartbeatAndStatus, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const updateHostStatus = `-- name: UpdateHostStatus :exec
|
||||
UPDATE hosts SET status = $2, updated_at = NOW() WHERE id = $1
|
||||
`
|
||||
|
||||
@ -36,6 +36,15 @@ type Host struct {
|
||||
MtlsEnabled bool `json:"mtls_enabled"`
|
||||
}
|
||||
|
||||
type HostRefreshToken struct {
|
||||
ID string `json:"id"`
|
||||
HostID string `json:"host_id"`
|
||||
TokenHash string `json:"token_hash"`
|
||||
ExpiresAt pgtype.Timestamptz `json:"expires_at"`
|
||||
CreatedAt pgtype.Timestamptz `json:"created_at"`
|
||||
RevokedAt pgtype.Timestamptz `json:"revoked_at"`
|
||||
}
|
||||
|
||||
type HostTag struct {
|
||||
HostID string `json:"host_id"`
|
||||
Tag string `json:"tag"`
|
||||
|
||||
@ -11,6 +11,20 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const bulkRestoreRunning = `-- name: BulkRestoreRunning :exec
|
||||
UPDATE sandboxes
|
||||
SET status = 'running',
|
||||
last_updated = NOW()
|
||||
WHERE id = ANY($1::text[]) AND status = 'missing'
|
||||
`
|
||||
|
||||
// Called by the reconciler when a host comes back online and its sandboxes are
|
||||
// confirmed alive. Restores only sandboxes that are in 'missing' state.
|
||||
func (q *Queries) BulkRestoreRunning(ctx context.Context, dollar_1 []string) error {
|
||||
_, err := q.db.Exec(ctx, bulkRestoreRunning, dollar_1)
|
||||
return err
|
||||
}
|
||||
|
||||
const bulkUpdateStatusByIDs = `-- name: BulkUpdateStatusByIDs :exec
|
||||
UPDATE sandboxes
|
||||
SET status = $2,
|
||||
@ -300,6 +314,21 @@ func (q *Queries) ListSandboxesByTeam(ctx context.Context, teamID string) ([]San
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markSandboxesMissingByHost = `-- name: MarkSandboxesMissingByHost :exec
|
||||
UPDATE sandboxes
|
||||
SET status = 'missing',
|
||||
last_updated = NOW()
|
||||
WHERE host_id = $1 AND status IN ('running', 'starting', 'pending')
|
||||
`
|
||||
|
||||
// Called when the host monitor marks a host unreachable.
|
||||
// Marks running/starting/pending sandboxes on that host as 'missing' so users see
|
||||
// the sandbox is not currently reachable, without permanently losing the record.
|
||||
func (q *Queries) MarkSandboxesMissingByHost(ctx context.Context, hostID string) error {
|
||||
_, err := q.db.Exec(ctx, markSandboxesMissingByHost, hostID)
|
||||
return err
|
||||
}
|
||||
|
||||
const updateLastActive = `-- name: UpdateLastActive :exec
|
||||
UPDATE sandboxes
|
||||
SET last_active_at = $2,
|
||||
|
||||
Reference in New Issue
Block a user