WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

internal/sandbox/remote_errors.go

@@ -0,0 +1,283 @@
+// Package sandbox: provider-neutral error contract for RemoteSandboxClient.
+//
+// Adapters normalize provider-native errors (HTTP status codes, SDK error
+// types, gRPC codes) into RemoteError with a stable Kind so SessionBoundManager
+// can make lifecycle decisions without knowing which backend is in use.
+//
+// Rule of thumb for adapters:
+//
+//   - NotFound / Terminal   → binding may be replaced
+//   - Timeout / Unavailable / Capacity / Conflict / Authentication → PRESERVE binding
+//   - InvalidRequest / Unsupported → surface to caller; do not touch binding
+//   - Internal → last resort; preserve binding (default-preserve on ambiguity)
+//
+// The original cause is always retained via errors.Unwrap so callers can
+// still inspect provider-specific detail when logging.
+package sandbox
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+// RemoteErrorKind is the stable, provider-neutral classification of a
+// RemoteSandboxClient failure.
+type RemoteErrorKind string
+
+const (
+	// RemoteErrorKindNotFound: sandbox / file / path does not exist.
+	RemoteErrorKindNotFound RemoteErrorKind = "not_found"
+
+	// RemoteErrorKindTerminal: sandbox is permanently gone (killed,
+	// crashed, terminated). Semantically equivalent to NotFound for
+	// lifecycle purposes but distinct for observability.
+	RemoteErrorKindTerminal RemoteErrorKind = "terminal"
+
+	// RemoteErrorKindAuthentication: bad credentials, expired token, etc.
+	RemoteErrorKindAuthentication RemoteErrorKind = "authentication"
+
+	// RemoteErrorKindInvalidRequest: caller-side error (bad template ID,
+	// bad path, oversized payload). Not retryable.
+	RemoteErrorKindInvalidRequest RemoteErrorKind = "invalid_request"
+
+	// RemoteErrorKindUnsupported: provider does not implement the requested
+	// capability (e.g. metadata, "never" timeout).
+	RemoteErrorKindUnsupported RemoteErrorKind = "unsupported"
+
+	// RemoteErrorKindConflict: concurrent modification (e.g. sandbox state
+	// changed under us, template being built).
+	RemoteErrorKindConflict RemoteErrorKind = "conflict"
+
+	// RemoteErrorKindCapacity: quota / rate-limit / out-of-capacity.
+	RemoteErrorKindCapacity RemoteErrorKind = "capacity"
+
+	// RemoteErrorKindTimeout: request exceeded its deadline before the
+	// provider responded. Distinct from execution timeout, which is a
+	// normal RemoteExecResult with Killed=true.
+	RemoteErrorKindTimeout RemoteErrorKind = "timeout"
+
+	// RemoteErrorKindUnavailable: transient provider outage (5xx, network
+	// error, control-plane unreachable).
+	RemoteErrorKindUnavailable RemoteErrorKind = "unavailable"
+
+	// RemoteErrorKindInternal: catch-all for unclassified failures.
+	// Adapters SHOULD narrow this to a more specific kind when possible.
+	RemoteErrorKindInternal RemoteErrorKind = "internal"
+)
+
+// RemoteError is the wire-agnostic error type returned by every
+// RemoteSandboxClient method.
+type RemoteError struct {
+	// Kind classifies the failure for coordinator decisions.
+	Kind RemoteErrorKind
+
+	// Provider identifies which backend produced the error, for logging.
+	Provider RemoteProvider
+
+	// Op names the RemoteSandboxClient operation (e.g. "Create", "Exec").
+	Op string
+
+	// Message is a human-readable summary. Provider-specific status codes
+	// or SDK error strings belong here.
+	Message string
+
+	// StatusCode is the provider's HTTP status when the failure came from an
+	// HTTP response, or 0 when it did not. Kind deliberately collapses statuses
+	// that call for the same lifecycle decision, so diagnostics that must tell
+	// those statuses apart read this instead of re-parsing Message.
+	StatusCode int
+
+	// Cause is the original provider-side error, retained for
+	// errors.Unwrap so callers can still errors.Is / errors.As it.
+	Cause error
+}
+
+// Error implements the error interface.
+func (e *RemoteError) Error() string {
+	if e == nil {
+		return "<nil remote error>"
+	}
+	prov := string(e.Provider)
+	if prov == "" {
+		prov = "remote"
+	}
+	if e.Message == "" && e.Cause != nil {
+		return fmt.Sprintf("%s %s: %s: %v", prov, e.Op, e.Kind, e.Cause)
+	}
+	if e.Cause != nil {
+		return fmt.Sprintf("%s %s: %s: %s: %v", prov, e.Op, e.Kind, e.Message, e.Cause)
+	}
+	return fmt.Sprintf("%s %s: %s: %s", prov, e.Op, e.Kind, e.Message)
+}
+
+// Unwrap exposes the wrapped provider-native error for errors.Is / errors.As.
+func (e *RemoteError) Unwrap() error {
+	if e == nil {
+		return nil
+	}
+	return e.Cause
+}
+
+// NewRemoteError builds a RemoteError. Convenience for adapters.
+func NewRemoteError(provider RemoteProvider, op string, kind RemoteErrorKind, message string, cause error) *RemoteError {
+	return &RemoteError{
+		Kind:     kind,
+		Provider: provider,
+		Op:       op,
+		Message:  message,
+		Cause:    cause,
+	}
+}
+
+// RemoteErrorDiagnostics formats err for logs. When err wraps a RemoteError
+// it includes kind, op, HTTP status, and message without re-parsing text.
+func RemoteErrorDiagnostics(err error) string {
+	if err == nil {
+		return ""
+	}
+	var re *RemoteError
+	if !errors.As(err, &re) {
+		return err.Error()
+	}
+	parts := []string{string(re.Kind)}
+	if re.Op != "" {
+		parts = append(parts, "op="+re.Op)
+	}
+	if re.StatusCode != 0 {
+		parts = append(parts, fmt.Sprintf("http=%d", re.StatusCode))
+	}
+	if re.Message != "" {
+		parts = append(parts, re.Message)
+	}
+	return strings.Join(parts, " ")
+}
+
+// remoteKind extracts the Kind from err, or "" when err is not a
+// *RemoteError. Wraps errors.As so callers can pass any error value.
+func remoteKind(err error) RemoteErrorKind {
+	if err == nil {
+		return ""
+	}
+	var re *RemoteError
+	if errors.As(err, &re) {
+		return re.Kind
+	}
+	return ""
+}
+
+// IsRemoteNotFound reports whether err classifies as sandbox / path not found.
+// Both NotFound and Terminal are treated as "gone" for coordinator decisions.
+func IsRemoteNotFound(err error) bool {
+	switch remoteKind(err) {
+	case RemoteErrorKindNotFound, RemoteErrorKindTerminal:
+		return true
+	default:
+		return false
+	}
+}
+
+// IsRemoteInvalidRequest reports whether err classifies as a caller-side
+// error (bad template, oversized payload, malformed path, etc.).
+func IsRemoteInvalidRequest(err error) bool {
+	return remoteKind(err) == RemoteErrorKindInvalidRequest
+}
+
+// IsRemoteConflict reports whether the provider refused the call because the
+// resource is busy (concurrent mutation, snapshot still referenced, …).
+func IsRemoteConflict(err error) bool {
+	return remoteKind(err) == RemoteErrorKindConflict
+}
+
+// snapshotDeleteKind reclassifies a snapshot/template delete that failed
+// because sandboxes still reference it. E2B returns that as HTTP 400
+// invalid_request; it is Conflict: the caller did nothing wrong and should
+// retry after those sandboxes (typically paused) go away.
+func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind {
+	if op != "DeleteSnapshot" && op != "DeleteTemplate" {
+		return kind
+	}
+	if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) {
+		return RemoteErrorKindConflict
+	}
+	return kind
+}
+
+func snapshotInUseBySandboxes(message string) bool {
+	msg := strings.ToLower(message)
+	if strings.Contains(msg, "paused sandbox") {
+		return true
+	}
+	if strings.Contains(msg, "sandboxes using") {
+		return true
+	}
+	return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it")
+}
+
+// IsRemoteDirAlreadyExists reports whether MakeDir failed because the
+// directory is already present. Cube's envd MakeDir is not mkdir -p: a
+// directory created by a previous call (or by a shell `mkdir -p` in
+// resetSkillDir) comes back as an internal error instead of success, and
+// seeding SKILL.md would otherwise abort on a directory it is supposed to
+// write into.
+func IsRemoteDirAlreadyExists(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	if strings.Contains(msg, "directory already exists") {
+		return true
+	}
+	return remoteKind(err) == RemoteErrorKindConflict && strings.Contains(msg, "already exists")
+}
+
+// ignoreExistingDir turns an exist-ok MakeDir failure into success so callers
+// can treat MakeDir as mkdir -p.
+func ignoreExistingDir(err error) error {
+	if IsRemoteDirAlreadyExists(err) {
+		return nil
+	}
+	return err
+}
+
+// CanReplaceRemoteBinding reports whether the error proves that the bound
+// remote sandbox is permanently gone. This is intentionally allow-list based:
+// unknown and newly introduced errors preserve bindings by default.
+func CanReplaceRemoteBinding(err error) bool {
+	switch remoteKind(err) {
+	case RemoteErrorKindNotFound, RemoteErrorKindTerminal:
+		return true
+	default:
+		return false
+	}
+}
+
+// httpErrorKind maps an HTTP status code to a RemoteErrorKind. It is shared
+// by all remote sandbox adapter backends (Cube, E2B).
+func httpErrorKind(op string, status int) RemoteErrorKind {
+	switch status {
+	case http.StatusBadRequest, http.StatusUnprocessableEntity:
+		return RemoteErrorKindInvalidRequest
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return RemoteErrorKindAuthentication
+	case http.StatusNotFound:
+		if op == "Create" {
+			return RemoteErrorKindInvalidRequest
+		}
+		return RemoteErrorKindNotFound
+	case http.StatusRequestTimeout, http.StatusGatewayTimeout:
+		return RemoteErrorKindTimeout
+	case http.StatusConflict:
+		return RemoteErrorKindConflict
+	case http.StatusGone:
+		return RemoteErrorKindTerminal
+	case http.StatusTooManyRequests, http.StatusInsufficientStorage:
+		return RemoteErrorKindCapacity
+	default:
+		if status >= 500 {
+			return RemoteErrorKindUnavailable
+		}
+		return RemoteErrorKindInternal
+	}
+}