fix: store digital-human upload text in files, slim jsonb for list
App / Format, vet, test, and build (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Avoid pulling multi-MB result payloads on uploads?view=summary over remote DB, which caused 30s client timeouts and HTTP 500. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -161,18 +161,27 @@ func NewResumableUploadService(db *gorm.DB, bases interfaces.KnowledgeBaseServic
|
||||
}
|
||||
return accesspolicy.WithSnapshot(ctx, snapshot), nil
|
||||
}
|
||||
var uploads *resumable.Service
|
||||
ingest := func(ctx context.Context, row *resumable.Upload, path string) (int64, types.JSON, bool, error) {
|
||||
if row.Kind == "digital-human" {
|
||||
source, selection, e := humans.readResumableMaterial(ctx, row.Name, path, row.Size)
|
||||
if e != nil {
|
||||
return 0, nil, false, e
|
||||
}
|
||||
result, e := json.Marshal(uploadResult{Source: &source, Selection: selection})
|
||||
if e = uploads.SaveMaterial(row.ID, source); e != nil {
|
||||
return 0, nil, false, e
|
||||
}
|
||||
// jsonb keeps only small metadata; full text lives in {id}.material.
|
||||
result, e := json.Marshal(uploadResult{
|
||||
Selection: selection,
|
||||
Source: &types.DigitalHumanSource{Name: source.Name},
|
||||
})
|
||||
return row.Size, result, true, e
|
||||
}
|
||||
return ingestResumableKnowledge(ctx, knowledge, row, path)
|
||||
}
|
||||
uploads, e := resumable.NewService(db, os.Getenv("RESUMABLE_UPLOAD_DIR"), authorize, workerContext, ingest)
|
||||
var e error
|
||||
uploads, e = resumable.NewService(db, os.Getenv("RESUMABLE_UPLOAD_DIR"), authorize, workerContext, ingest)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
@@ -192,19 +201,40 @@ func NewResumableUploadService(db *gorm.DB, bases interfaces.KnowledgeBaseServic
|
||||
return nil, digitalHumanInputError("材料尚未解析完成、已过期或不属于此数字人")
|
||||
}
|
||||
var result uploadResult
|
||||
if json.Unmarshal(row.Result, &result) != nil || result.Source == nil {
|
||||
if json.Unmarshal(row.Result, &result) != nil {
|
||||
return nil, digitalHumanInputError("材料解析结果无效")
|
||||
}
|
||||
if result.Selection != "full" {
|
||||
return nil, digitalHumanInputError("材料“" + row.Name + "”是旧版原文节选,请重新上传以完整蒸馏")
|
||||
}
|
||||
sources = append(sources, *result.Source)
|
||||
source, e := resolveDigitalHumanUploadSource(ctx, humans, uploads, row, result)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
sources = append(sources, source)
|
||||
}
|
||||
return sources, nil
|
||||
}
|
||||
return uploads, nil
|
||||
}
|
||||
|
||||
// resolveDigitalHumanUploadSource prefers the file-backed material, then legacy
|
||||
// jsonb text, then re-parses the raw upload and backfills .material.
|
||||
func resolveDigitalHumanUploadSource(ctx context.Context, humans *DigitalHumanService, uploads *resumable.Service, row *resumable.Upload, result uploadResult) (types.DigitalHumanSource, error) {
|
||||
if source, e := uploads.LoadMaterial(row.ID); e == nil {
|
||||
return source, nil
|
||||
}
|
||||
if result.Source != nil && strings.TrimSpace(result.Source.Text) != "" {
|
||||
return *result.Source, nil
|
||||
}
|
||||
source, _, e := humans.readResumableMaterial(ctx, row.Name, uploads.FilePath(row.ID), row.Size)
|
||||
if e != nil {
|
||||
return types.DigitalHumanSource{}, digitalHumanInputError("材料解析结果无效")
|
||||
}
|
||||
_ = uploads.SaveMaterial(row.ID, source)
|
||||
return source, nil
|
||||
}
|
||||
|
||||
func textUpload(name string) bool {
|
||||
switch strings.ToLower(filepath.Ext(name)) {
|
||||
case ".txt", ".md", ".markdown", ".srt", ".vtt", ".csv", ".json", ".html", ".htm":
|
||||
|
||||
Reference in New Issue
Block a user