fix: store digital-human upload text in files, slim jsonb for list
App / Format, vet, test, and build (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s

Avoid pulling multi-MB result payloads on uploads?view=summary over remote DB, which caused 30s client timeouts and HTTP 500.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
yi
2026-09-23 19:20:13 +08:00
co-authored by Cursor
parent f7fc992f57
commit 9fb2c1321a
6 changed files with 255 additions and 17 deletions
@@ -161,18 +161,27 @@ func NewResumableUploadService(db *gorm.DB, bases interfaces.KnowledgeBaseServic
}
return accesspolicy.WithSnapshot(ctx, snapshot), nil
}
var uploads *resumable.Service
ingest := func(ctx context.Context, row *resumable.Upload, path string) (int64, types.JSON, bool, error) {
if row.Kind == "digital-human" {
source, selection, e := humans.readResumableMaterial(ctx, row.Name, path, row.Size)
if e != nil {
return 0, nil, false, e
}
result, e := json.Marshal(uploadResult{Source: &source, Selection: selection})
if e = uploads.SaveMaterial(row.ID, source); e != nil {
return 0, nil, false, e
}
// jsonb keeps only small metadata; full text lives in {id}.material.
result, e := json.Marshal(uploadResult{
Selection: selection,
Source: &types.DigitalHumanSource{Name: source.Name},
})
return row.Size, result, true, e
}
return ingestResumableKnowledge(ctx, knowledge, row, path)
}
uploads, e := resumable.NewService(db, os.Getenv("RESUMABLE_UPLOAD_DIR"), authorize, workerContext, ingest)
var e error
uploads, e = resumable.NewService(db, os.Getenv("RESUMABLE_UPLOAD_DIR"), authorize, workerContext, ingest)
if e != nil {
return nil, e
}
@@ -192,19 +201,40 @@ func NewResumableUploadService(db *gorm.DB, bases interfaces.KnowledgeBaseServic
return nil, digitalHumanInputError("材料尚未解析完成、已过期或不属于此数字人")
}
var result uploadResult
if json.Unmarshal(row.Result, &result) != nil || result.Source == nil {
if json.Unmarshal(row.Result, &result) != nil {
return nil, digitalHumanInputError("材料解析结果无效")
}
if result.Selection != "full" {
return nil, digitalHumanInputError("材料“" + row.Name + "”是旧版原文节选,请重新上传以完整蒸馏")
}
sources = append(sources, *result.Source)
source, e := resolveDigitalHumanUploadSource(ctx, humans, uploads, row, result)
if e != nil {
return nil, e
}
sources = append(sources, source)
}
return sources, nil
}
return uploads, nil
}
// resolveDigitalHumanUploadSource prefers the file-backed material, then legacy
// jsonb text, then re-parses the raw upload and backfills .material.
func resolveDigitalHumanUploadSource(ctx context.Context, humans *DigitalHumanService, uploads *resumable.Service, row *resumable.Upload, result uploadResult) (types.DigitalHumanSource, error) {
if source, e := uploads.LoadMaterial(row.ID); e == nil {
return source, nil
}
if result.Source != nil && strings.TrimSpace(result.Source.Text) != "" {
return *result.Source, nil
}
source, _, e := humans.readResumableMaterial(ctx, row.Name, uploads.FilePath(row.ID), row.Size)
if e != nil {
return types.DigitalHumanSource{}, digitalHumanInputError("材料解析结果无效")
}
_ = uploads.SaveMaterial(row.ID, source)
return source, nil
}
func textUpload(name string) bool {
switch strings.ToLower(filepath.Ext(name)) {
case ".txt", ".md", ".markdown", ".srt", ".vtt", ".csv", ".json", ".html", ".htm":