xinmi/internal/application/service/tenant_sandbox_config.go

// Package service: sandbox backend config management.
//
// The load-bearing rule: some fields decide whether a config can still OPERATE
// the sandboxes it already created. Overwriting them in place while sandboxes
// are alive breaks those sandboxes in one of two ways, and both are bad enough
// to refuse the save.
//
// Losing the control plane (provider, API URL, API key) means the new
// credentials have no authority over the old sandboxes, so they can no longer be
// listed or deleted. Session sandboxes are created with onTimeout=pause, so the
// provider TTL never reclaims them either - the leak is permanent and it bills.
// A paused sandbox is also what a session expects to RESUME, so conversations
// break on top.
//
// Losing the data plane (E2B sandbox domain; Cube proxy URL and sandbox domain)
// keeps cleanup possible, but every envd request now goes to the wrong host.
// Every live session on this config fails at once - skill execution, attachment
// staging, artifact collection - while the sandboxes stay alive and keep billing
// until their sessions are deleted.
//
// Hence both groups are refused while sandboxes exist. The admin ends the owning
// sessions or creates a second config and re-points agents; both keep the old
// values intact, which is what keeps cleanup possible.
package service

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	stderrors "errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/google/uuid"
	"golang.org/x/sync/singleflight"

	"github.com/Tencent/WeKnora/internal/application/repository"
	apperrors "github.com/Tencent/WeKnora/internal/errors"
	"github.com/Tencent/WeKnora/internal/logger"
	"github.com/Tencent/WeKnora/internal/sandbox"
	"github.com/Tencent/WeKnora/internal/types"
	"github.com/Tencent/WeKnora/internal/types/interfaces"
	"github.com/Tencent/WeKnora/internal/utils"
)

const sandboxConfigCleanupTimeout = 20 * time.Second

// SandboxIdentityChanged reports whether sandboxes already created under oldCfg
// would stop being operable under newCfg. Names, templates, TTLs, HTTP timeouts
// and env vars deliberately do not count: they only shape FUTURE sandboxes.
//
// newCfg must already have been merged with the stored config (see
// types.MergeSandboxConfigForUpdate). The API masks secrets on read, so
// comparing a raw payload would report a key rotation on every single save and
// leave the config permanently uneditable.
//
// The comparison needs no deployment baseline: named configs inherit nothing, so
// the stored fields are the whole identity. It is also deliberately free of
// validation — see sandbox.IdentityOf for why an unreachable old endpoint must
// still be judgeable.
func SandboxIdentityChanged(oldCfg, newCfg *types.TenantSandboxConfig) bool {
	if oldCfg == nil {
		// Nothing exists yet, so nothing can be stranded.
		return false
	}
	if newCfg == nil {
		return true
	}
	return sandbox.IdentityOf(oldCfg) != sandbox.IdentityOf(newCfg)
}

func configHasSkillSnapshot(cfg *types.TenantSandboxConfig) bool {
	return cfg != nil && cfg.SkillImage != nil && strings.TrimSpace(cfg.SkillImage.SnapshotID) != ""
}

// skillRetargetWouldChange reports edits that would retarget the environment
// a skill snapshot is built from: identity, spawn template, or Cube DNS.
func skillRetargetWouldChange(stored, merged *types.TenantSandboxConfig) bool {
	if SandboxIdentityChanged(stored, merged) {
		return true
	}
	if spawnTemplateID(stored) != spawnTemplateID(merged) {
		return true
	}
	return !sameStrings(cubeDNSServers(stored), cubeDNSServers(merged))
}

// spawnTemplateID is the template/image this config would boot without a
// skill snapshot. Cube and E2B store it as template_id; Docker as the image.
func spawnTemplateID(cfg *types.TenantSandboxConfig) string {
	if cfg == nil {
		return ""
	}
	switch sandbox.SandboxType(cfg.SandboxType) {
	case sandbox.SandboxTypeCube:
		if cfg.Cube != nil {
			return strings.TrimSpace(cfg.Cube.TemplateID)
		}
	case sandbox.SandboxTypeE2B:
		if cfg.E2B != nil {
			return strings.TrimSpace(cfg.E2B.TemplateID)
		}
	case sandbox.SandboxTypeDocker:
		if cfg.Docker != nil {
			return strings.TrimSpace(cfg.Docker.Image)
		}
	}
	return ""
}

func cubeDNSServers(cfg *types.TenantSandboxConfig) []string {
	if cfg == nil || cfg.Cube == nil {
		return nil
	}
	return cfg.Cube.DNSServers
}

func sameStrings(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

// skillSnapshotBlocksConnectionChange reports edits that would retarget the
// environment a skill snapshot was built from: identity, spawn template, or
// Cube DNS (which only applies if the template is rebuilt).
func skillSnapshotBlocksConnectionChange(stored, merged *types.TenantSandboxConfig) bool {
	return configHasSkillSnapshot(stored) && skillRetargetWouldChange(stored, merged)
}

// ErrSandboxesStillLive is returned when an identity change or deletion is
// refused because the current credentials still own provider resources.
var ErrSandboxesStillLive = stderrors.New("sandbox config still owns live sandboxes")

// SandboxesStillLiveError carries the provider inventory the handler must show
// in its 409 response; recomputing it could race and disagree with the refusal.
type SandboxesStillLiveError struct {
	Inventory SandboxInventory
}

func (e *SandboxesStillLiveError) Error() string {
	return fmt.Sprintf("%s: %d", ErrSandboxesStillLive, e.Inventory.SandboxCount)
}

func (e *SandboxesStillLiveError) Unwrap() error {
	return ErrSandboxesStillLive
}

// ErrSandboxInventoryUnverifiable is returned when the provider cannot be
// reached to answer "does this config still own sandboxes?".
//
// It is deliberately distinct from ErrSandboxesStillLive: one means "there ARE
// sandboxes, deal with them", the other means "we cannot tell". The second is
// the only case a force delete may override, because an endpoint whose DNS
// record is gone would otherwise make its config permanently undeletable.
var ErrSandboxInventoryUnverifiable = stderrors.New(
	"cannot verify whether the sandbox config still owns sandboxes")

// ErrSandboxConfigNameRequired is a sentinel so transports can classify this as
// bad input without matching on the message text.
var ErrSandboxConfigNameRequired = stderrors.New("sandbox config name is required")

// ErrNamedSandboxBackendUnsupported marks a sandbox type that cannot be stored
// as a user-facing named backend config.
var ErrNamedSandboxBackendUnsupported = stderrors.New(
	"named sandbox configs only support cube, e2b and docker backends",
)

// ErrSkillSnapshotBlocksTemplateChange is returned when this config already
// has a skill snapshot and the caller tries to retarget its connection, DNS,
// or spawn template, or to rebuild the cluster's WeKnora template. Sessions
// boot the snapshot, so those edits would not reach them. The way out is a
// second config.
var ErrSkillSnapshotBlocksTemplateChange = stderrors.New(
	"sandbox connection cannot change while this config has a skill snapshot")

// ErrSkillSnapshotReleaseFailed is returned when deleting a sandbox config
// cannot destroy every provider snapshot on that config's ledger. Leaving the
// row is recoverable; leaking a billable snapshot is not.
var ErrSkillSnapshotReleaseFailed = stderrors.New("failed to release skill snapshots")

// SkillSnapshotReleaseFailedError names the snapshots that are still on the
// provider so the handler can show them and a retry can skip already-deleted
// ones.
type SkillSnapshotReleaseFailedError struct {
	Remaining []string
}

func (e *SkillSnapshotReleaseFailedError) Error() string {
	if e == nil || len(e.Remaining) == 0 {
		return ErrSkillSnapshotReleaseFailed.Error()
	}
	return fmt.Sprintf("%s: %s", ErrSkillSnapshotReleaseFailed.Error(), strings.Join(e.Remaining, ", "))
}

func (e *SkillSnapshotReleaseFailedError) Unwrap() error {
	return ErrSkillSnapshotReleaseFailed
}

// sandboxSnapshotReleaser is the provider capability Delete needs. It is a
// named slice of RemoteSnapshotManager so tests do not have to fake Create or
// List, and so the type assertion is not anonymous.
type sandboxSnapshotReleaser interface {
	DeleteSnapshot(ctx context.Context, snapshotID string) error
}

// sandboxConfigSkillStore is the skill/ledger slice Delete needs to destroy
// snapshots and drop rows that would dangle after SoftDelete.
type sandboxConfigSkillStore interface {
	ListSnapshotsByConfig(
		ctx context.Context, tenantID uint64, configID string,
	) ([]*types.TenantSkillSnapshotEntity, error)
	MarkSnapshotState(ctx context.Context, tenantID uint64, id, state, snapshotID string) error
	ListSkillsByConfig(ctx context.Context, tenantID uint64, configID string) ([]*types.TenantSkillEntity, error)
	DeleteSkill(ctx context.Context, tenantID uint64, configID, skillID string) error
	// The tenant-wide lists answer who else still reads an archive the
	// deleted install rows had pinned.
	ListSkillsByTenant(ctx context.Context, tenantID uint64) ([]*types.TenantSkillEntity, error)
	ListCatalogsByTenant(ctx context.Context, tenantID uint64) ([]*types.TenantSkillCatalogEntity, error)
	DeleteSnapshotRowsByConfig(ctx context.Context, tenantID uint64, configID string) error
	DeleteUserEnvVarsByConfig(ctx context.Context, tenantID uint64, configID string) error
}

// sandboxConfigBundleResolver locates the tenant file service so config
// deletion can drop skill archives after the ledger is released.
type sandboxConfigBundleResolver interface {
	ResolveFileService(
		ctx context.Context, tenant *types.Tenant, backendID, provider, localBaseDir string,
	) (interfaces.FileService, string, error)
}

// SandboxInventory describes what a config holds and who a change disturbs.
type SandboxInventory struct {
	SandboxCount int      `json:"sandbox_count"`
	SessionIDs   []string `json:"session_ids,omitempty"`
	AgentNames   []string `json:"agent_names,omitempty"`

	// Unverifiable reports that SandboxCount is unknown rather than zero
	// because the provider could not be reached. The management page must say
	// so instead of rendering a reassuring "0 sandboxes".
	Unverifiable bool `json:"unverifiable,omitempty"`
}

// CreateSandboxConfigInput is the create payload.
type CreateSandboxConfigInput struct {
	Name        string
	Description string
	Config      *types.TenantSandboxConfig
}

// UpdateSandboxConfigInput is the update payload. Config may carry redacted
// placeholders that resolve against the stored row.
type UpdateSandboxConfigInput struct {
	Name        string
	Description string
	Config      *types.TenantSandboxConfig
}

// SandboxTemplateQueryInput describes an unsaved connection from the settings
// drawer. ConfigID is optional and lets masked credentials resolve against the
// stored row while editing.
type SandboxTemplateQueryInput struct {
	Config          *types.TenantSandboxConfig
	ConfigID        string
	EnsureStandard  bool
	ReplaceStandard bool
}

type SandboxTemplateCatalog struct {
	Templates          []sandbox.RemoteTemplate `json:"templates"`
	StandardTemplateID string                   `json:"standard_template_id,omitempty"`
	Provisioned        bool                     `json:"provisioned"`
}

// SandboxConfigAgentRepo is the slice of the agent repository this service
// needs. Agent references are warnings, never grounds for refusing a change.
type SandboxConfigAgentRepo interface {
	ListNamesBySandboxConfigID(ctx context.Context, tenantID uint64, configID string) ([]string, error)
}

// TenantSandboxConfigService owns the sandbox config lifecycle.
type TenantSandboxConfigService struct {
	repo      repository.TenantSandboxConfigRepository
	agents    SandboxConfigAgentRepo
	globalCfg *sandbox.Config
	now       func() time.Time

	// skills is used when deleting a config, to destroy the snapshot ledger
	// and drop install rows. Catalog archives are owned by the skill
	// definition and are not deleted here.
	skills sandboxConfigSkillStore
	files  sandboxConfigBundleResolver

	// newClient is injectable so tests can supply a provider inventory.
	newClient func(*sandbox.Config) (sandbox.ConfigSandboxClient, error)

	// ensureTemplate collapses concurrent "make sure this cluster has our
	// template" requests per cluster. Provisioning is idempotent only once the
	// build shows up in the provider's catalog, and a double-click on refresh
	// is fast enough to slip in before that.
	ensureTemplate singleflight.Group
}

// NewTenantSandboxConfigService wires the config service.
func NewTenantSandboxConfigService(
	repo repository.TenantSandboxConfigRepository,
	agents SandboxConfigAgentRepo,
	globalCfg *sandbox.Config,
	skills repository.TenantSkillRepository,
	files sandboxConfigBundleResolver,
) *TenantSandboxConfigService {
	return &TenantSandboxConfigService{
		repo:      repo,
		agents:    agents,
		globalCfg: globalCfg,
		now:       time.Now,
		skills:    skills,
		files:     files,
		newClient: func(cfg *sandbox.Config) (sandbox.ConfigSandboxClient, error) {
			return sandbox.NewRemoteClientForCheck(cfg)
		},
	}
}

// SanitizeSandboxConfig resolves redacted secrets and validates the payload
// before it can be persisted.
func SanitizeSandboxConfig(
	incoming, existing *types.TenantSandboxConfig,
) (*types.TenantSandboxConfig, error) {
	if incoming == nil {
		return nil, nil
	}
	merged := types.MergeSandboxConfigForUpdate(incoming, existing)

	if merged.SandboxType != "" {
		parsed, err := sandbox.ParseSandboxType(merged.SandboxType)
		if err != nil {
			return nil, err
		}
		if err := sandbox.EnsureDockerBackendAllowed(parsed); err != nil {
			return nil, err
		}
	}
	if merged.Cube != nil {
		dns, err := sandbox.NormalizeCubeDNSServers(merged.Cube.DNSServers)
		if err != nil {
			return nil, apperrors.NewBadRequestError(err.Error())
		}
		merged.Cube.DNSServers = dns
	}
	// Rejected here rather than at sandbox-create time: the provider's own
	// error arrives minutes later, on a screen the admin has already left.
	if err := types.ValidateSandboxNetworkPolicy(merged); err != nil {
		return nil, apperrors.NewBadRequestError(err.Error())
	}
	for _, endpoint := range sandboxConfigEndpoints(merged) {
		if err := sandbox.ValidateOutboundURLWithPolicy(endpoint, sandbox.OutboundURLPolicy{
			AllowPrivate: merged.AllowPrivateEndpoints,
		}); err != nil {
			return nil, err
		}
	}
	// Without an AES key the Value() hook would persist these secrets in
	// plaintext. Refuse instead of silently downgrading storage security.
	if sandboxConfigHasSecrets(merged) && utils.GetAESKey() == nil {
		return nil, apperrors.NewBadRequestError(
			"SYSTEM_AES_KEY is not configured; refusing to store sandbox credentials in plaintext",
		)
	}
	// Reject an incomplete config here rather than at first sandbox allocation.
	// Resolving is what the runtime does, so both paths agree by construction.
	// The baseline passed in is irrelevant to the outcome: named configs inherit
	// no provider field, so only merged decides what is missing.
	// Returned unwrapped so respondSandboxConfigServiceError can classify the
	// sentinel; wrapping it in an AppError here would hide the chain.
	if _, err := sandbox.ResolveEffectiveConfig(merged, sandbox.DefaultConfig()); err != nil {
		return nil, err
	}
	if err := validateSkillRollout(merged.SkillRollout); err != nil {
		return nil, err
	}
	return merged, nil
}

func validateSkillRollout(value string) error {
	switch strings.TrimSpace(value) {
	case "", types.SkillRolloutNextTurn, types.SkillRolloutNewSession:
		return nil
	default:
		return apperrors.NewBadRequestError("invalid skill_rollout")
	}
}

func validateNamedSandboxBackend(cfg *types.TenantSandboxConfig) error {
	if cfg == nil || strings.TrimSpace(cfg.SandboxType) == "" {
		return apperrors.NewBadRequestError("sandbox backend type is required")
	}
	if !sandbox.IsNamedSandboxBackendType(cfg.SandboxType) {
		return fmt.Errorf("%w", ErrNamedSandboxBackendUnsupported)
	}
	return sandbox.EnsureDockerBackendAllowed(sandbox.SandboxType(cfg.SandboxType))
}

func filterPublicSandboxConfigs(
	list []*types.TenantSandboxConfigEntity,
) []*types.TenantSandboxConfigEntity {
	if len(list) == 0 {
		return list
	}
	out := make([]*types.TenantSandboxConfigEntity, 0, len(list))
	for _, e := range list {
		if types.IsSandboxWorkspacePolicyRow(e) {
			continue
		}
		out = append(out, e)
	}
	return out
}

func findWorkspacePolicyRow(
	list []*types.TenantSandboxConfigEntity,
) *types.TenantSandboxConfigEntity {
	for _, e := range list {
		if types.IsSandboxWorkspacePolicyRow(e) {
			return e
		}
	}
	return nil
}

// WorkspaceScriptsDisabled reports whether the workspace-wide kill switch is
// active, regardless of which named backend an agent selected.
func (s *TenantSandboxConfigService) WorkspaceScriptsDisabled(
	ctx context.Context, tenantID uint64,
) (bool, error) {
	list, err := s.repo.ListByTenant(ctx, tenantID)
	if err != nil {
		return false, err
	}
	return findWorkspacePolicyRow(list) != nil, nil
}

// SetWorkspaceScriptsDisabled toggles script execution for the entire
// workspace, across all named backend types.
func (s *TenantSandboxConfigService) SetWorkspaceScriptsDisabled(
	ctx context.Context, tenantID uint64, disabled bool,
) error {
	list, err := s.repo.ListByTenant(ctx, tenantID)
	if err != nil {
		return err
	}
	existing := findWorkspacePolicyRow(list)
	if disabled {
		if existing != nil {
			return nil
		}
		entity := &types.TenantSandboxConfigEntity{
			ID:          uuid.New().String(),
			TenantID:    tenantID,
			Name:        types.SandboxWorkspacePolicyConfigName,
			Description: "",
			SandboxType: string(sandbox.SandboxTypeDisabled),
			Config:      &types.TenantSandboxConfig{SandboxType: string(sandbox.SandboxTypeDisabled)},
		}
		return s.repo.Create(ctx, entity)
	}
	if existing == nil {
		return nil
	}
	return s.repo.SoftDelete(ctx, tenantID, existing.ID)
}

// Create stores a new config.
func (s *TenantSandboxConfigService) Create(
	ctx context.Context, tenantID uint64, in CreateSandboxConfigInput,
) (*types.TenantSandboxConfigEntity, error) {
	if err := validateNamedSandboxBackend(in.Config); err != nil {
		return nil, err
	}
	merged, err := SanitizeSandboxConfig(in.Config, nil)
	if err != nil {
		return nil, err
	}
	name := strings.TrimSpace(in.Name)
	if name == "" {
		return nil, ErrSandboxConfigNameRequired
	}
	entity := &types.TenantSandboxConfigEntity{
		ID:          uuid.New().String(),
		TenantID:    tenantID,
		Name:        name,
		Description: in.Description,
		Config:      merged,
	}
	if merged != nil {
		entity.SandboxType = merged.SandboxType
	}
	if err := s.repo.Create(ctx, entity); err != nil {
		return nil, err
	}
	return entity, nil
}

// List returns the workspace's user-facing configs (policy row excluded).
func (s *TenantSandboxConfigService) List(
	ctx context.Context, tenantID uint64,
) ([]*types.TenantSandboxConfigEntity, error) {
	list, err := s.repo.ListByTenant(ctx, tenantID)
	if err != nil {
		return nil, err
	}
	return filterPublicSandboxConfigs(list), nil
}

// Get returns one config, or nil when absent.
func (s *TenantSandboxConfigService) Get(
	ctx context.Context, tenantID uint64, id string,
) (*types.TenantSandboxConfigEntity, error) {
	entity, err := s.repo.GetByID(ctx, tenantID, id)
	if err != nil || entity == nil {
		return nil, err
	}
	if types.IsSandboxWorkspacePolicyRow(entity) {
		return nil, nil
	}
	return entity, nil
}

// QueryTemplates reads the provider's template catalog and optionally installs
// the standard WeKnora image when it is absent. This is intentionally driven by
// workspace credentials instead of deployment environment variables.
func (s *TenantSandboxConfigService) QueryTemplates(
	ctx context.Context, tenantID uint64, in SandboxTemplateQueryInput,
) (*SandboxTemplateCatalog, error) {
	if in.ReplaceStandard && strings.TrimSpace(in.ConfigID) == "" {
		return nil, apperrors.NewBadRequestError(
			"config_id is required to rebuild the standard template")
	}
	var existing *types.TenantSandboxConfig
	if strings.TrimSpace(in.ConfigID) != "" {
		entity, err := s.repo.GetByID(ctx, tenantID, in.ConfigID)
		if err != nil {
			return nil, err
		}
		if entity == nil || types.IsSandboxWorkspacePolicyRow(entity) {
			return nil, apperrors.NewNotFoundError("sandbox config not found")
		}
		existing = entity.Config
	}
	merged := types.MergeSandboxConfigForUpdate(in.Config, existing)
	if merged == nil {
		merged = types.MergeSandboxConfigForUpdate(existing, nil)
	}
	if merged == nil {
		return nil, apperrors.NewBadRequestError("sandbox config is required")
	}
	if in.ReplaceStandard {
		if err := s.refuseClusterSkillTemplateReplace(ctx, tenantID, merged); err != nil {
			return nil, err
		}
	}

	// Catalog access needs the control-plane connection but does not need a
	// spawn template yet. A private placeholder lets the same effective-config
	// validation protect every other required field without weakening runtime
	// validation.
	switch merged.SandboxType {
	case string(sandbox.SandboxTypeCube):
		if merged.Cube == nil {
			merged.Cube = &types.CubeSandboxConfig{}
		}
		if strings.TrimSpace(merged.Cube.TemplateID) == "" {
			merged.Cube.TemplateID = "__catalog__"
		}
	case string(sandbox.SandboxTypeE2B):
		if merged.E2B == nil {
			merged.E2B = &types.E2BSandboxConfig{}
		}
		if strings.TrimSpace(merged.E2B.TemplateID) == "" {
			merged.E2B.TemplateID = "__catalog__"
		}
	case string(sandbox.SandboxTypeDocker):
		if err := sandbox.EnsureDockerBackendAllowed(sandbox.SandboxTypeDocker); err != nil {
			return nil, err
		}
		// Docker's template is an image, and the catalog step is where the
		// admin picks one. The standard image stands in until they do, so the
		// effective-config validation below has something to accept.
		if merged.Docker == nil {
			merged.Docker = &types.DockerSandboxConfig{}
		}
		if strings.TrimSpace(merged.Docker.Image) == "" {
			merged.Docker.Image = sandbox.DefaultDockerImage
		}
	default:
		return nil, apperrors.NewBadRequestError(
			"sandbox template catalog only supports cube, e2b and docker backends")
	}

	for _, endpoint := range sandboxConfigEndpoints(merged) {
		if err := sandbox.ValidateOutboundURLWithPolicy(endpoint, sandbox.OutboundURLPolicy{
			AllowPrivate: merged.AllowPrivateEndpoints,
		}); err != nil {
			return nil, err
		}
	}
	effective, err := sandbox.ResolveEffectiveConfig(merged, sandbox.DefaultConfig())
	if err != nil {
		return nil, err
	}
	client, err := s.newClient(effective)
	if err != nil {
		return nil, err
	}
	catalog, ok := any(client).(sandbox.RemoteTemplateCatalog)
	if !ok {
		return nil, fmt.Errorf("sandbox: provider %q does not expose templates", effective.Type)
	}
	templates, err := catalog.ListTemplates(ctx)
	if err != nil {
		return nil, err
	}
	result := &SandboxTemplateCatalog{Templates: deduplicateSandboxTemplates(templates)}
	usable := pickStandardTemplate(result.Templates)
	if usable != nil {
		result.StandardTemplateID = usable.ID
	}
	oldStandardIDs := standardTemplateIDs(result.Templates)
	// Listing is read-only. Creating or replacing the WeKnora template is an
	// explicit settings-page action: auto-ensure on every refresh made DNS
	// and image changes impossible to apply, and left admins unsure whether
	// they had asked for a build.
	wantStandard := in.ReplaceStandard || (in.EnsureStandard && usable == nil)
	if wantStandard {
		op := "ensure"
		if in.ReplaceStandard {
			op = "replace"
		}
		key := op + ":" + ensureTemplateKey(tenantID, sandbox.IdentityOf(merged))
		ensured, ensureErr, _ := s.ensureTemplate.Do(key, func() (any, error) {
			if in.ReplaceStandard {
				return catalog.ReplaceStandardTemplate(ctx)
			}
			return catalog.EnsureStandardTemplate(ctx)
		})
		if ensureErr != nil {
			return nil, ensureErr
		}
		standard, ok := ensured.(*sandbox.RemoteTemplate)
		if !ok || standard == nil {
			return nil, fmt.Errorf("sandbox: provider %q returned no standard template", effective.Type)
		}
		result.Templates = deduplicateSandboxTemplates(append(result.Templates, *standard))
		if sandbox.IsTemplateReady(standard.Status) {
			result.Provisioned = true
			if in.ReplaceStandard {
				persistErr := s.persistSpawnTemplateID(ctx, tenantID, merged, standard.ID, oldStandardIDs)
				if persistErr != nil {
					logger.Warnf(ctx, "[sandbox] persist rebuilt template id: %v; keeping previous templates",
						persistErr)
				} else {
					result.StandardTemplateID = standard.ID
					result.Templates = hideSupersededStandardTemplates(result.Templates, standard.ID)
					s.deleteSupersededStandardTemplates(ctx, catalog, standard.ID)
				}
			} else {
				result.StandardTemplateID = standard.ID
			}
		} else if !sandbox.IsTemplateBuildFailed(standard.Status) {
			result.Provisioned = true
			if result.StandardTemplateID == "" {
				result.StandardTemplateID = standard.ID
			}
		}
	}
	sort.SliceStable(result.Templates, func(i, j int) bool {
		if result.Templates[i].Standard != result.Templates[j].Standard {
			return result.Templates[i].Standard
		}
		return strings.ToLower(result.Templates[i].Name) < strings.ToLower(result.Templates[j].Name)
	})
	return result, nil
}

func hideSupersededStandardTemplates(items []sandbox.RemoteTemplate, keepID string) []sandbox.RemoteTemplate {
	keepID = strings.TrimSpace(keepID)
	kept := make([]sandbox.RemoteTemplate, 0, len(items))
	for _, item := range items {
		if item.Standard && strings.TrimSpace(item.ID) != keepID {
			continue
		}
		kept = append(kept, item)
	}
	return kept
}

func standardTemplateIDs(items []sandbox.RemoteTemplate) []string {
	out := make([]string, 0, len(items))
	for _, item := range items {
		if item.Standard {
			if id := strings.TrimSpace(item.ID); id != "" {
				out = append(out, id)
			}
		}
	}
	return out
}

func (s *TenantSandboxConfigService) refuseClusterSkillTemplateReplace(
	ctx context.Context, tenantID uint64, merged *types.TenantSandboxConfig,
) error {
	identity := sandbox.IdentityOf(merged)
	list, err := s.repo.ListByTenant(ctx, tenantID)
	if err != nil {
		return err
	}
	for _, entity := range list {
		if entity == nil || types.IsSandboxWorkspacePolicyRow(entity) {
			continue
		}
		if sandbox.IdentityOf(entity.Config) != identity {
			continue
		}
		if configHasSkillSnapshot(entity.Config) ||
			s.configHasInFlightSkill(ctx, entity.TenantID, entity.ID) {
			return ErrSkillSnapshotBlocksTemplateChange
		}
	}
	return nil
}

func (s *TenantSandboxConfigService) configHasInFlightSkill(
	ctx context.Context, tenantID uint64, configID string,
) bool {
	if s == nil || s.skills == nil || strings.TrimSpace(configID) == "" {
		return false
	}
	rows, err := s.skills.ListSkillsByConfig(ctx, tenantID, configID)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] cannot read skills of config %s while judging a retarget: %v",
			configID, err)
		return true
	}
	for _, row := range rows {
		if row == nil {
			continue
		}
		switch row.Status {
		case types.SkillStatusInstalling, types.SkillStatusRemoving:
			return true
		}
	}
	return false
}

func (s *TenantSandboxConfigService) persistSpawnTemplateID(
	ctx context.Context,
	tenantID uint64,
	merged *types.TenantSandboxConfig,
	newID string,
	oldIDs []string,
) error {
	newID = strings.TrimSpace(newID)
	if newID == "" || s == nil || s.repo == nil {
		return nil
	}
	old := make(map[string]struct{}, len(oldIDs))
	for _, id := range oldIDs {
		if id = strings.TrimSpace(id); id != "" {
			old[id] = struct{}{}
		}
	}
	identity := sandbox.IdentityOf(merged)
	list, err := s.repo.ListByTenant(ctx, tenantID)
	if err != nil {
		return fmt.Errorf("list configs: %w", err)
	}
	var persistErr error
	for _, entity := range list {
		if entity == nil || entity.Config == nil || types.IsSandboxWorkspacePolicyRow(entity) {
			continue
		}
		if sandbox.IdentityOf(entity.Config) != identity {
			continue
		}
		current := spawnTemplateID(entity.Config)
		if current == newID {
			continue
		}
		if current != "" {
			if _, pointedAtOld := old[current]; !pointedAtOld {
				continue
			}
		}
		previous := current
		setSpawnTemplateID(entity.Config, newID)
		if err := s.repo.Update(ctx, entity); err != nil {
			setSpawnTemplateID(entity.Config, previous)
			logger.Warnf(ctx, "[sandbox] persist rebuilt template id on config %s: %v", entity.ID, err)
			persistErr = err
			continue
		}
		logger.Infof(ctx, "[sandbox] config %s spawn template is now %s after rebuild", entity.ID, newID)
	}
	return persistErr
}

func (s *TenantSandboxConfigService) deleteSupersededStandardTemplates(
	ctx context.Context, catalog sandbox.RemoteTemplateCatalog, keepID string,
) {
	if catalog == nil || strings.TrimSpace(keepID) == "" {
		return
	}
	if err := catalog.DeleteSupersededStandardTemplates(ctx, keepID); err != nil {
		logger.Warnf(ctx, "[sandbox] delete superseded standard templates failed: %v", err)
	}
}

func setSpawnTemplateID(cfg *types.TenantSandboxConfig, id string) {
	if cfg == nil {
		return
	}
	switch sandbox.SandboxType(cfg.SandboxType) {
	case sandbox.SandboxTypeCube:
		if cfg.Cube != nil {
			cfg.Cube.TemplateID = id
		}
	case sandbox.SandboxTypeE2B:
		if cfg.E2B != nil {
			cfg.E2B.TemplateID = id
		}
	}
}

// pickStandardTemplate returns the WeKnora template the UI should preselect, or
// nil when the cluster has none that could ever spawn a sandbox. A failed build
// is skipped so the caller can reprovision instead of offering it.
func pickStandardTemplate(items []sandbox.RemoteTemplate) *sandbox.RemoteTemplate {
	var best *sandbox.RemoteTemplate
	for i := range items {
		if !items[i].Standard || sandbox.IsTemplateBuildFailed(items[i].Status) {
			continue
		}
		if best == nil || templateStatusRank(items[i].Status) > templateStatusRank(best.Status) {
			best = &items[i]
		}
	}
	return best
}

// ensureTemplateKey names one cluster as seen by one tenant. The identity
// carries an API key, so it is hashed rather than formatted: this string is
// only ever compared, and it should not be able to surface a credential in a
// panic trace or a heap dump.
func ensureTemplateKey(tenantID uint64, identity sandbox.SandboxIdentity) string {
	sum := sha256.Sum256([]byte(fmt.Sprintf("%d|%#v", tenantID, identity)))
	return hex.EncodeToString(sum[:])
}

func deduplicateSandboxTemplates(items []sandbox.RemoteTemplate) []sandbox.RemoteTemplate {
	if len(items) < 2 {
		return items
	}
	result := make([]sandbox.RemoteTemplate, 0, len(items))
	indexByID := make(map[string]int, len(items))
	for _, item := range items {
		id := strings.TrimSpace(item.ID)
		if id == "" {
			result = append(result, item)
			continue
		}
		idx, exists := indexByID[id]
		if !exists {
			indexByID[id] = len(result)
			result = append(result, item)
			continue
		}
		current := &result[idx]
		current.Standard = current.Standard || item.Standard
		if templateStatusRank(item.Status) > templateStatusRank(current.Status) {
			current.Status = item.Status
			current.Version = item.Version
			current.UpdatedAt = item.UpdatedAt
			current.Error = item.Error
		}
		if strings.TrimSpace(current.Name) == "" ||
			(strings.EqualFold(current.Name, sandbox.StandardTemplateName) && strings.Contains(item.Name, "/")) {
			current.Name = item.Name
		}
		if current.Image == "" {
			current.Image = item.Image
		}
	}
	return result
}

func templateStatusRank(status string) int {
	switch strings.ToLower(strings.TrimSpace(status)) {
	case "ready", "available", "complete", "completed", "success", "succeeded":
		return 3
	case "building", "waiting", "pending", "queued", "processing", "running":
		return 2
	case "failed", "error", "cancelled", "canceled":
		return 1
	default:
		return 0
	}
}

// Inventory answers what changing or deleting this config would disturb.
//
// An unreachable provider is reported through Unverifiable rather than as an
// error: the management page still has to render the card, and the agent names
// it warns about come from our own database.
func (s *TenantSandboxConfigService) Inventory(
	ctx context.Context, tenantID uint64, id string,
) (SandboxInventory, error) {
	entity, err := s.repo.GetByID(ctx, tenantID, id)
	if err != nil {
		return SandboxInventory{}, err
	}
	if entity == nil {
		return SandboxInventory{}, apperrors.NewNotFoundError("sandbox config not found")
	}
	summaries, err := s.listSandboxes(ctx, entity.Config, tenantID, id)
	inv := s.inventoryFromSummaries(ctx, tenantID, id, summaries)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] inventory of config %s is unverifiable: %v", id, err)
		inv.Unverifiable = true
	}
	return inv, nil
}

// listSandboxes enumerates what the config currently owns. A nil client means
// the backend holds no remote resources at all, which is a verified empty.
func (s *TenantSandboxConfigService) listSandboxes(
	ctx context.Context,
	cfg *types.TenantSandboxConfig,
	tenantID uint64,
	id string,
) ([]sandbox.RemoteSandboxSummary, error) {
	client, err := s.clientFor(cfg)
	if err != nil {
		return nil, err
	}
	if client == nil {
		return nil, nil
	}
	return sandbox.ListConfigSandboxes(ctx, client, tenantID, id)
}

// Update applies an edit. Identity edits are cordoned before inventory and
// swept afterwards using the old client so credentials are never overwritten
// while they still own provider resources.
func (s *TenantSandboxConfigService) Update(
	ctx context.Context, tenantID uint64, id string, in UpdateSandboxConfigInput,
) (*types.TenantSandboxConfigEntity, error) {
	entity, err := s.repo.GetByID(ctx, tenantID, id)
	if err != nil || entity == nil {
		return nil, err
	}
	if types.IsSandboxWorkspacePolicyRow(entity) {
		return nil, apperrors.NewBadRequestError("workspace policy cannot be edited here")
	}
	if in.Config != nil && strings.TrimSpace(in.Config.SandboxType) != "" {
		if err := validateNamedSandboxBackend(in.Config); err != nil {
			return nil, err
		}
	}
	merged, err := SanitizeSandboxConfig(in.Config, entity.Config)
	if err != nil {
		return nil, err
	}
	if err := validateNamedSandboxBackend(merged); err != nil {
		return nil, err
	}
	if skillSnapshotBlocksConnectionChange(entity.Config, merged) ||
		(skillRetargetWouldChange(entity.Config, merged) && s.configHasInFlightSkill(ctx, tenantID, id)) {
		return nil, ErrSkillSnapshotBlocksTemplateChange
	}
	if !SandboxIdentityChanged(entity.Config, merged) {
		return s.writeConfig(ctx, entity, in, merged)
	}

	if err := s.repo.SetCordon(ctx, tenantID, id, s.now()); err != nil {
		return nil, err
	}
	defer s.clearCordonAfterRequest(ctx, tenantID, id)

	// When old credentials no longer reach the provider we cannot enumerate
	// sandboxes to refuse the edit — but blocking the save traps the admin on
	// a key they are trying to fix. Proceed and skip the post-write sweep;
	// sandboxes we cannot see may become orphans and need provider-side cleanup.
	oldClient, err := s.clientFor(entity.Config)
	if err != nil {
		logger.Warnf(ctx,
			"[sandbox] config %s: old credentials unusable for inventory: %v; proceeding",
			id, err)
		oldClient = nil
	}
	if oldClient != nil {
		summaries, listErr := sandbox.ListConfigSandboxes(ctx, oldClient, tenantID, id)
		if listErr != nil {
			logger.Warnf(ctx,
				"[sandbox] config %s: cannot verify sandbox inventory with old credentials: %v; proceeding",
				id, listErr)
			oldClient = nil
		} else if len(summaries) > 0 {
			inv := s.inventoryFromSummaries(ctx, tenantID, id, summaries)
			return nil, &SandboxesStillLiveError{Inventory: inv}
		}
	}

	updated, err := s.writeConfig(ctx, entity, in, merged)
	if err != nil {
		return nil, err
	}

	if oldClient != nil {
		s.sweepAfterWrite(ctx, oldClient, tenantID, id)
	}
	return updated, nil
}

// Delete refuses while the config still owns sandboxes, and while skill
// snapshots on its ledger cannot be destroyed. Agent references are permanent
// state and are surfaced as warnings by callers, not blockers.
//
// force covers two cases we cannot complete from here: the provider could not
// be reached to list sandboxes, and a skill snapshot could not be destroyed.
// Without it a config whose endpoint's DNS record disappeared, or whose
// snapshot API is down, could never be removed — unlike an edit, deletion has
// no "create a second config" way out. It does NOT override sandboxes we can
// actually see; those still have to be dealt with through their sessions,
// otherwise the forced deletion would be precisely the permanent leak this
// whole flow prevents.
func (s *TenantSandboxConfigService) Delete(
	ctx context.Context, tenantID uint64, id string, force bool,
) error {
	entity, err := s.repo.GetByID(ctx, tenantID, id)
	if err != nil {
		return err
	}
	// Reporting success for a config that is not there would let the UI drop a
	// card the workspace still has, so absence is an explicit 404.
	if entity == nil {
		return apperrors.NewNotFoundError("sandbox config not found")
	}
	if types.IsSandboxWorkspacePolicyRow(entity) {
		return apperrors.NewBadRequestError("workspace policy cannot be deleted here")
	}
	summaries, listErr := s.listSandboxes(ctx, entity.Config, tenantID, id)
	if listErr != nil {
		if !force {
			return fmt.Errorf("%w: %v", ErrSandboxInventoryUnverifiable, listErr)
		}
		logger.Warnf(ctx,
			"[sandbox] force-deleting config %s without verifying its sandboxes: %v",
			id, listErr)
	}
	if len(summaries) > 0 {
		inv := s.inventoryFromSummaries(ctx, tenantID, id, summaries)
		return &SandboxesStillLiveError{Inventory: inv}
	}
	if err := s.releaseSkillSnapshots(ctx, entity, tenantID, id, force); err != nil {
		return err
	}
	return s.repo.SoftDelete(ctx, tenantID, id)
}

func snapshotReleaserFrom(client sandbox.ConfigSandboxClient) (sandboxSnapshotReleaser, bool) {
	if client == nil {
		return nil, false
	}
	releaser, ok := client.(sandboxSnapshotReleaser)
	return releaser, ok
}

// pendingNeedProvider reports whether releasing these rows has to talk to the
// provider at all. A planned name counts: it is how an abandoned build — one
// whose commit outlived the process that started it — is still addressable.
func pendingNeedProvider(rows []*types.TenantSkillSnapshotEntity) bool {
	for _, row := range rows {
		if row == nil {
			continue
		}
		if strings.TrimSpace(row.SnapshotID) != "" ||
			strings.TrimSpace(row.PlannedName) != "" {
			return true
		}
	}
	return false
}

// resolveAbandonedBuildIDs fills in the provider ID of a build that died
// between its commit and the ledger write.
//
// This is the last chance to reclaim one: PruneSupersededSnapshots walks live
// configs, so once this config's row is gone the snapshot is unreachable by
// anything. A name that resolves to nothing is left alone — the usual reason
// is that the commit never happened, and there is no resource to release.
func resolveAbandonedBuildIDs(
	ctx context.Context,
	client sandbox.ConfigSandboxClient,
	pending []*types.TenantSkillSnapshotEntity,
) {
	unresolved := make([]*types.TenantSkillSnapshotEntity, 0, len(pending))
	for _, row := range pending {
		if row != nil && strings.TrimSpace(row.SnapshotID) == "" &&
			strings.TrimSpace(row.PlannedName) != "" {
			unresolved = append(unresolved, row)
		}
	}
	if len(unresolved) == 0 {
		return
	}
	lister, ok := client.(skillSnapshotLister)
	if !ok {
		return
	}
	listed, err := lister.ListSnapshots(ctx, "")
	if err != nil {
		logger.Warnf(ctx, "[sandbox] list snapshots to release abandoned builds failed: %v", err)
		return
	}
	prefix := skillSnapshotNamePrefix(unresolved[0].TenantID, unresolved[0].SandboxConfigID)
	listed = snapshotsNotFromOtherConfig(listed, prefix)
	for _, row := range unresolved {
		id := matchSnapshotByName(listed, row.PlannedName)
		if id == "" {
			continue
		}
		logger.Infof(ctx, "[sandbox] abandoned build %s resolved to snapshot %s for release",
			row.PlannedName, id)
		row.SnapshotID = id
	}
}

func pendingSkillSnapshots(rows []*types.TenantSkillSnapshotEntity) []*types.TenantSkillSnapshotEntity {
	var out []*types.TenantSkillSnapshotEntity
	for _, row := range rows {
		if row == nil || row.State == types.SkillSnapshotStateDeleted {
			continue
		}
		out = append(out, row)
	}
	return out
}

func skillSnapshotName(row *types.TenantSkillSnapshotEntity) string {
	if row == nil {
		return ""
	}
	if name := strings.TrimSpace(row.SnapshotID); name != "" {
		return name
	}
	return row.ID
}

func (s *TenantSandboxConfigService) releaseSkillSnapshots(
	ctx context.Context,
	entity *types.TenantSandboxConfigEntity,
	tenantID uint64,
	configID string,
	force bool,
) error {
	if s.skills == nil {
		return nil
	}

	rows, err := s.skills.ListSnapshotsByConfig(ctx, tenantID, configID)
	if err != nil {
		if !force {
			return fmt.Errorf("%w: list snapshots: %v", ErrSkillSnapshotReleaseFailed, err)
		}
		logger.Warnf(ctx,
			"[sandbox] force-deleting config %s without listing skill snapshots: %v",
			configID, err)
	}

	pending := pendingSkillSnapshots(rows)
	remaining := s.destroyPendingSnapshots(ctx, entity, pending)
	if len(remaining) > 0 && !force {
		return &SkillSnapshotReleaseFailedError{Remaining: remaining}
	}
	if len(remaining) > 0 {
		logger.Warnf(ctx,
			"[sandbox] force-deleting config %s with unreleased snapshots %s",
			configID, strings.Join(remaining, ", "))
	}

	s.cleanupSkillMetadata(ctx, tenantID, configID)
	return nil
}

func (s *TenantSandboxConfigService) destroyPendingSnapshots(
	ctx context.Context,
	entity *types.TenantSandboxConfigEntity,
	pending []*types.TenantSkillSnapshotEntity,
) []string {
	if len(pending) == 0 {
		return nil
	}

	releaser := sandboxSnapshotReleaser(nil)
	if pendingNeedProvider(pending) {
		client := s.snapshotClientFor(ctx, entity)
		resolveAbandonedBuildIDs(ctx, client, pending)
		if found, ok := snapshotReleaserFrom(client); ok {
			releaser = found
		} else if client != nil {
			logger.Warnf(ctx, "[sandbox] config %s provider does not support snapshot delete",
				entityID(entity))
		}
	}
	var remaining []string
	for _, row := range pending {
		if err := deleteProviderSnapshot(ctx, releaser, row); err != nil {
			remaining = append(remaining, skillSnapshotName(row))
			continue
		}
		if err := s.skills.MarkSnapshotState(
			ctx, row.TenantID, row.ID, types.SkillSnapshotStateDeleted, row.SnapshotID,
		); err != nil {
			logger.Warnf(ctx, "[sandbox] mark snapshot %s deleted failed: %v", row.ID, err)
			remaining = append(remaining, skillSnapshotName(row))
		}
	}
	return remaining
}

// snapshotClientFor builds the provider client config deletion needs to
// release this config's snapshots. It returns the client rather than just the
// delete capability because an abandoned build has to be looked up by name
// first, which is a different capability on the same client.
func (s *TenantSandboxConfigService) snapshotClientFor(
	ctx context.Context, entity *types.TenantSandboxConfigEntity,
) sandbox.ConfigSandboxClient {
	cfg := (*types.TenantSandboxConfig)(nil)
	if entity != nil {
		cfg = entity.Config
	}
	client, err := s.clientFor(cfg)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] cannot build snapshot client for config %s: %v",
			entityID(entity), err)
		return nil
	}
	return client
}

func entityID(entity *types.TenantSandboxConfigEntity) string {
	if entity == nil {
		return ""
	}
	return entity.ID
}

func deleteProviderSnapshot(
	ctx context.Context, releaser sandboxSnapshotReleaser, row *types.TenantSkillSnapshotEntity,
) error {
	if row == nil {
		return nil
	}
	snapshotID := strings.TrimSpace(row.SnapshotID)
	if snapshotID == "" {
		return nil
	}
	if releaser == nil {
		return fmt.Errorf("snapshot delete is unavailable")
	}
	err := releaser.DeleteSnapshot(ctx, snapshotID)
	if err == nil || sandbox.IsRemoteNotFound(err) {
		return nil
	}
	return err
}

func (s *TenantSandboxConfigService) cleanupSkillMetadata(
	ctx context.Context, tenantID uint64, configID string,
) {
	skills, err := s.skills.ListSkillsByConfig(ctx, tenantID, configID)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] list skills for config %s cleanup failed: %v", configID, err)
	}
	var pinned []string
	seen := map[string]struct{}{}
	for _, skill := range skills {
		if skill == nil {
			continue
		}
		if ref := strings.TrimSpace(skill.BundleRef); ref != "" {
			if _, dup := seen[ref]; !dup {
				seen[ref] = struct{}{}
				pinned = append(pinned, ref)
			}
		}
		if err := s.skills.DeleteSkill(ctx, tenantID, configID, skill.ID); err != nil {
			logger.Warnf(ctx, "[sandbox] delete skill %s on config %s failed: %v",
				skill.ID, configID, err)
		}
	}
	// After the rows are gone, never before: each ref is named by the very row
	// being deleted, so asking first would always find a reader.
	s.releasePinnedBundles(ctx, tenantID, pinned)
	if err := s.skills.DeleteSnapshotRowsByConfig(ctx, tenantID, configID); err != nil {
		logger.Warnf(ctx, "[sandbox] delete snapshot ledger for config %s failed: %v",
			configID, err)
	}
	// DeleteSkill only takes the values filed under a skill; the config-wide
	// ones have no skill to hang off and would outlive the config.
	if err := s.skills.DeleteUserEnvVarsByConfig(ctx, tenantID, configID); err != nil {
		logger.Warnf(ctx, "[sandbox] delete member env vars for config %s failed: %v",
			configID, err)
	}
}

// releasePinnedBundles drops the archives whose last reader was an install row
// this config deletion just removed.
//
// An install row names an object only in one case: a re-register replaced the
// definition's copy while this sandbox kept running the image built from the
// old one, so the row pinned those bytes. Deleting the config drops that
// reader and no code path will ever look for them again.
//
// The pin is not exclusive, which is why this cannot just delete what the rows
// named. Sibling configs installed from the same archive pin the same object,
// and a definition may have been rolled back onto it, so the bytes go only once
// no catalog and no surviving install names them. A list that cannot be read
// keeps the archive: a leaked one costs storage, a deleted one costs some other
// sandbox its files.
func (s *TenantSandboxConfigService) releasePinnedBundles(
	ctx context.Context, tenantID uint64, refs []string,
) {
	if len(refs) == 0 || s.files == nil || s.skills == nil {
		return
	}
	catalogs, err := s.skills.ListCatalogsByTenant(ctx, tenantID)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] list catalogs before releasing skill archives failed: %v", err)
		return
	}
	installs, err := s.skills.ListSkillsByTenant(ctx, tenantID)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] list installs before releasing skill archives failed: %v", err)
		return
	}
	held := make(map[string]struct{}, len(catalogs)+len(installs))
	for _, cat := range catalogs {
		if cat != nil {
			held[strings.TrimSpace(cat.BundleRef)] = struct{}{}
		}
	}
	for _, row := range installs {
		if row != nil {
			held[strings.TrimSpace(row.BundleRef)] = struct{}{}
		}
	}

	var fs interfaces.FileService
	for _, ref := range refs {
		if _, still := held[ref]; still {
			continue
		}
		if fs == nil {
			resolved, _, resolveErr := s.files.ResolveFileService(ctx, &types.Tenant{ID: tenantID}, "", "", "")
			if resolveErr != nil || resolved == nil {
				logger.Warnf(ctx, "[sandbox] resolve file service to release skill archives failed: %v",
					resolveErr)
				return
			}
			fs = resolved
		}
		if err := fs.DeleteFile(ctx, ref); err != nil {
			logger.Warnf(ctx, "[sandbox] delete skill archive %s failed: %v", ref, err)
		}
	}
}

func (s *TenantSandboxConfigService) clientFor(
	cfg *types.TenantSandboxConfig,
) (sandbox.ConfigSandboxClient, error) {
	// The baseline only supplies the deployment's execution timeout; every
	// provider field comes from cfg, so a nil globalCfg cannot change which
	// backend this client talks to.
	base := s.globalCfg
	if base == nil {
		base = sandbox.DefaultConfig()
	}
	effective, err := sandbox.ResolveEffectiveConfig(cfg, base)
	if err != nil {
		return nil, err
	}
	switch effective.Type {
	case sandbox.SandboxTypeCube, sandbox.SandboxTypeE2B, sandbox.SandboxTypeDocker:
		return s.newClient(effective)
	default:
		return nil, nil
	}
}

func (s *TenantSandboxConfigService) writeConfig(
	ctx context.Context,
	entity *types.TenantSandboxConfigEntity,
	in UpdateSandboxConfigInput,
	merged *types.TenantSandboxConfig,
) (*types.TenantSandboxConfigEntity, error) {
	if name := strings.TrimSpace(in.Name); name != "" {
		entity.Name = name
	}
	entity.Description = in.Description
	entity.Config = merged
	if merged != nil {
		entity.SandboxType = merged.SandboxType
	}
	if err := s.repo.Update(ctx, entity); err != nil {
		return nil, err
	}
	return entity, nil
}

func (s *TenantSandboxConfigService) inventoryFromSummaries(
	ctx context.Context,
	tenantID uint64,
	id string,
	summaries []sandbox.RemoteSandboxSummary,
) SandboxInventory {
	inv := SandboxInventory{SandboxCount: len(summaries)}
	for _, summary := range summaries {
		if sessionID := summary.Metadata[sandbox.MetadataSessionIDKey()]; sessionID != "" {
			inv.SessionIDs = append(inv.SessionIDs, sessionID)
		}
	}
	if s.agents == nil {
		return inv
	}
	names, err := s.agents.ListNamesBySandboxConfigID(ctx, tenantID, id)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] list agents for config %s: %v", id, err)
		return inv
	}
	inv.AgentNames = names
	return inv
}

func (s *TenantSandboxConfigService) clearCordonAfterRequest(
	ctx context.Context,
	tenantID uint64,
	id string,
) {
	cleanupCtx, cancel := context.WithTimeout(
		context.WithoutCancel(ctx), sandboxConfigCleanupTimeout)
	defer cancel()
	if err := s.repo.ClearCordon(cleanupCtx, tenantID, id); err != nil {
		logger.Warnf(ctx, "[sandbox] clear cordon on config %s: %v", id, err)
	}
}

func (s *TenantSandboxConfigService) sweepAfterWrite(
	ctx context.Context,
	oldClient sandbox.ConfigSandboxClient,
	tenantID uint64,
	id string,
) {
	cleanupCtx, cancel := context.WithTimeout(
		context.WithoutCancel(ctx), sandboxConfigCleanupTimeout)
	defer cancel()
	deleted, err := sandbox.ReleaseConfigSandboxes(cleanupCtx, oldClient, tenantID, id)
	if err != nil {
		logger.Warnf(ctx, "[sandbox] post-write sweep of config %s failed: %v", id, err)
		return
	}
	if deleted > 0 {
		logger.Infof(ctx,
			"[sandbox] swept %d sandbox(es) created during the cordon window on config %s",
			deleted, id)
	}
}

// sandboxConfigEndpoints returns every non-empty tenant-supplied URL.
func sandboxConfigEndpoints(cfg *types.TenantSandboxConfig) []string {
	if cfg == nil {
		return nil
	}
	var endpoints []string
	if cfg.Cube != nil {
		for _, raw := range []string{cfg.Cube.APIURL, cfg.Cube.ProxyURL} {
			if raw != "" {
				endpoints = append(endpoints, raw)
			}
		}
	}
	if cfg.E2B != nil && cfg.E2B.APIURL != "" {
		endpoints = append(endpoints, cfg.E2B.APIURL)
	}
	return endpoints
}

// sandboxConfigHasSecrets reports whether cfg carries any value that must be
// encrypted at rest.
func sandboxConfigHasSecrets(cfg *types.TenantSandboxConfig) bool {
	if cfg == nil {
		return false
	}
	if cfg.Cube != nil && cfg.Cube.APIKey != "" {
		return true
	}
	if cfg.E2B != nil && cfg.E2B.APIKey != "" {
		return true
	}
	for _, value := range cfg.EnvVars {
		if value != "" {
			return true
		}
	}
	if cfg.Network != nil {
		for _, rule := range cfg.Network.CubeRules {
			for _, inject := range rule.Inject {
				if inject.Secret != "" {
					return true
				}
			}
		}
		for _, rule := range cfg.Network.E2BHostRules {
			for _, value := range rule.Headers {
				if value != "" {
					return true
				}
			}
		}
	}
	return false
}