WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
Anydoc / anydoc parser engine (push) Canceled after 0s
App / Format, vet, test, and build (push) Canceled after 0s
CLI / test (macos-latest / go 1.26) (push) Canceled after 0s
CLI / test (ubuntu-latest / go 1.26) (push) Canceled after 0s
CLI / test (windows-latest / go 1.26) (push) Canceled after 0s
Build and Push Docker Image / build-ui (push) Canceled after 0s
Build and Push Docker Image / build-docreader (push) Canceled after 0s
Build and Push Docker Image / build-sandbox (push) Canceled after 0s
Build and Push Docker Image / build-app (amd64, linux/amd64, ubuntu-latest) (push) Canceled after 0s
Build and Push Docker Image / build-app (arm64, linux/arm64, ubuntu-24.04-arm) (push) Canceled after 0s
DocReader / Python tests and Go client tests (push) Canceled after 0s
Build & Publish (dsh-weknora) / Typecheck, test, pack (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (pinned) (push) Canceled after 0s
Build & Publish (dsh-weknora) / End-to-end inside dsh (latest, advisory) (push) Canceled after 0s
Frontend / Test, type-check, and build (push) Canceled after 0s
Go Lint Cache / warm lint cache (.) (push) Canceled after 0s
Go Lint Cache / warm lint cache (cli) (push) Canceled after 0s
Go Lint Cache / warm lint cache (client) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.10) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.11) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.12) (push) Canceled after 0s
Build & Publish (mcp-server) / Test (Python 3.13) (push) Canceled after 0s
Build and Push Docker Image / merge (push) Canceled after 0s
Build & Publish (dsh-weknora) / Publish to npm (push) Canceled after 0s
Build & Publish (mcp-server) / Build sdist + wheel (push) Canceled after 0s
Build & Publish (mcp-server) / Publish to PyPI (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// backendScopedFileService makes the storage instance part of every newly
|
||||
// persisted path while delegating actual I/O to the existing provider driver.
|
||||
type backendScopedFileService struct {
|
||||
backendID string
|
||||
inner interfaces.FileService
|
||||
}
|
||||
|
||||
func NewBackendScopedFileService(backendID string, inner interfaces.FileService) interfaces.FileService {
|
||||
return &backendScopedFileService{backendID: backendID, inner: inner}
|
||||
}
|
||||
|
||||
func (s *backendScopedFileService) unwrap(path string) (string, error) {
|
||||
id, inner, ok := types.ParseStorageBackendPath(path)
|
||||
if !ok {
|
||||
return path, nil
|
||||
}
|
||||
if id != s.backendID {
|
||||
return "", fmt.Errorf("storage backend mismatch")
|
||||
}
|
||||
return inner, nil
|
||||
}
|
||||
|
||||
func (s *backendScopedFileService) wrap(path string) string {
|
||||
return types.BuildStorageBackendPath(s.backendID, path)
|
||||
}
|
||||
func (s *backendScopedFileService) CheckConnectivity(ctx context.Context) error {
|
||||
return s.inner.CheckConnectivity(ctx)
|
||||
}
|
||||
func (s *backendScopedFileService) SaveFile(ctx context.Context, f *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
|
||||
p, err := s.inner.SaveFile(ctx, f, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.wrap(p), nil
|
||||
}
|
||||
func (s *backendScopedFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, name string, temp bool) (string, error) {
|
||||
p, err := s.inner.SaveBytes(ctx, data, tenantID, name, temp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.wrap(p), nil
|
||||
}
|
||||
func (s *backendScopedFileService) GetFile(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.inner.GetFile(ctx, p)
|
||||
}
|
||||
func (s *backendScopedFileService) GetFileURL(ctx context.Context, path string) (string, error) {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := s.inner.GetFileURL(ctx, p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
scoped := s.wrap(p)
|
||||
if result == p {
|
||||
return scoped, nil
|
||||
}
|
||||
// Local storage may return an app-level presigned URL. Re-sign it with
|
||||
// the scoped path so the proxy resolves the exact local instance instead
|
||||
// of falling back to another backend of the same provider.
|
||||
if u, parseErr := url.Parse(result); parseErr == nil && strings.HasSuffix(u.Path, "/api/v1/files/presigned") && u.Query().Get("file_path") == p {
|
||||
basePath := strings.TrimSuffix(u.Path, "/api/v1/files/presigned")
|
||||
baseURL := u.Scheme + "://" + u.Host + basePath
|
||||
ttl := time.Duration(0)
|
||||
if expires, convErr := strconv.ParseInt(u.Query().Get("expires"), 10, 64); convErr == nil {
|
||||
ttl = time.Until(time.Unix(expires, 0))
|
||||
}
|
||||
if signed, signErr := secutils.SignFileURL(baseURL, scoped, secutils.ParseTenantIDFromStoragePath(scoped), ttl); signErr == nil {
|
||||
return signed, nil
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (s *backendScopedFileService) DeleteFile(ctx context.Context, path string) error {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.inner.DeleteFile(ctx, p)
|
||||
}
|
||||
func (s *backendScopedFileService) CopyFile(ctx context.Context, path string, tenantID uint64, knowledgeID string) (string, error) {
|
||||
p, err := s.unwrap(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := s.inner.CopyFile(ctx, p, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.wrap(result), nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBackendScopedLocalURLRetainsBackendID(t *testing.T) {
|
||||
t.Setenv("SYSTEM_AES_KEY", "0123456789abcdef0123456789abcdef")
|
||||
inner := NewLocalFileService(t.TempDir(), "https://weknora.example.com/base")
|
||||
svc := NewBackendScopedFileService("backend-local-a", inner)
|
||||
|
||||
path, err := svc.SaveBytes(context.Background(), []byte("hello"), 7, "exports/image.txt", false)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, path, "storage://backend-local-a/local://")
|
||||
|
||||
signed, err := svc.GetFileURL(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
u, err := url.Parse(signed)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, path, u.Query().Get("file_path"))
|
||||
|
||||
reader, err := svc.GetFile(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello", string(data))
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/tencentyun/cos-go-sdk-v5"
|
||||
)
|
||||
|
||||
// cosFileService implements the FileService interface for Tencent Cloud COS
|
||||
type cosFileService struct {
|
||||
client *cos.Client
|
||||
bucketURL string
|
||||
cosPathPrefix string
|
||||
tempClient *cos.Client
|
||||
tempBucketURL string
|
||||
bucketName string
|
||||
region string
|
||||
}
|
||||
|
||||
const cosScheme = "cos://"
|
||||
|
||||
func newCOSHTTPClient(secretID, secretKey string) *http.Client {
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client := utils.NewSSRFSafeHTTPClient(httpConfig)
|
||||
// COS GetCredential expects AuthorizationTransport to remain the outermost
|
||||
// transport. Put the per-request SSRF guard immediately underneath it while
|
||||
// retaining the safe client's redirect policy.
|
||||
client.Transport = &cos.AuthorizationTransport{
|
||||
SecretID: secretID,
|
||||
SecretKey: secretKey,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
},
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
// newCosClient creates a bare cosFileService with just the SDK client initialised.
|
||||
// Shared by NewCosFileService* constructors and CheckCosConnectivity.
|
||||
func newCosClient(bucketName, region, secretID, secretKey string) (*cosFileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
logger.Infof(context.Background(), "newCosClient: bucketURL: %s", bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
}
|
||||
client := cos.NewClient(&cos.BaseURL{BucketURL: u}, newCOSHTTPClient(secretID, secretKey))
|
||||
return &cosFileService{client: client, bucketURL: bucketURL, bucketName: bucketName, region: region}, nil
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
|
||||
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
svc, err := newCosClient(bucketName, region, secretId, secretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
svc.cosPathPrefix = cosPathPrefix
|
||||
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region
|
||||
}
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", tempBucketName, tempRegion)
|
||||
tempU, err := url.Parse(tempBucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
|
||||
}
|
||||
svc.tempClient = cos.NewClient(&cos.BaseURL{BucketURL: tempU}, newCOSHTTPClient(secretId, secretKey))
|
||||
svc.tempBucketURL = tempBucketURL
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies COS is reachable by performing a HEAD request on the bucket.
|
||||
func (s *cosFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := s.client.Bucket.Head(checkCtx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckCosConnectivity tests COS connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckCosConnectivity(ctx context.Context, bucketName, region, secretID, secretKey string) error {
|
||||
svc, err := newCosClient(bucketName, region, secretID, secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// SaveFile saves a file to COS storage
|
||||
// It generates a unique name for the file and organizes it by tenant and knowledge ID
|
||||
func (s *cosFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
_, err = s.client.Object.Put(ctx, objectName, src, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to COS: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("cos://%s/%s/%s", s.bucketName, s.region, objectName), nil
|
||||
}
|
||||
|
||||
// GetFile retrieves a file from COS storage by its path URL
|
||||
func (s *cosFileService) GetFile(ctx context.Context, filePathUrl string) (io.ReadCloser, error) {
|
||||
objectName, err := s.parseCosObjectName(filePathUrl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
resp, err := s.client.Object.Get(ctx, objectName, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from COS: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// DeleteFile removes a file from COS storage
|
||||
func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
objectName, err := s.parseCosObjectName(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
_, err = s.client.Object.Delete(ctx, objectName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseCosObjectName extracts the object name from:
|
||||
// - provider scheme: cos://{bucket}/{region}/{objectKey}
|
||||
// - legacy URL: https://bucket.cos.region.myqcloud.com/{objectKey}
|
||||
func (s *cosFileService) parseCosObjectName(filePath string) (string, error) {
|
||||
for _, other := range []string{"local://", "minio://", "s3://", "tos://", "oss://", "ks3://", "obs://"} {
|
||||
if strings.HasPrefix(filePath, other) {
|
||||
return "", fmt.Errorf("cos file service cannot resolve %s path", strings.Split(other, "://")[0])
|
||||
}
|
||||
}
|
||||
// Provider scheme format: cos://{bucket}/{region}/{objectKey}
|
||||
if strings.HasPrefix(filePath, cosScheme) {
|
||||
rest := strings.TrimPrefix(filePath, cosScheme)
|
||||
parts := strings.SplitN(rest, "/", 3)
|
||||
if len(parts) == 3 {
|
||||
return parts[2], nil
|
||||
}
|
||||
return rest, nil
|
||||
}
|
||||
// Legacy format: https://bucket.cos.region.myqcloud.com/{objectKey}
|
||||
return strings.TrimPrefix(filePath, s.bucketURL), nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing COS object to a new knowledge-owned object using a
|
||||
// server-side Object.Copy (no data leaves COS). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath is not a cos:// path.
|
||||
func (s *cosFileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
srcObjectKey, err := s.parseCosObjectName(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cos copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
if err := utils.SafeObjectKey(srcObjectKey); err != nil {
|
||||
return "", fmt.Errorf("invalid source path: %w", err)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
destKey := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
// sourceURL is the host + object key WITHOUT a scheme, per the COS SDK contract.
|
||||
sourceURL := fmt.Sprintf("%s.cos.%s.myqcloud.com/%s", s.bucketName, s.region, srcObjectKey)
|
||||
_, _, err = s.client.Object.Copy(ctx, destKey, sourceURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in COS: %w", err)
|
||||
}
|
||||
|
||||
newPath := fmt.Sprintf("cos://%s/%s/%s", s.bucketName, s.region, destKey)
|
||||
logger.Infof(ctx, "Copied COS object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to COS
|
||||
// If temp is true and temp bucket is configured, saves to temp bucket (with lifecycle auto-expiration)
|
||||
// Otherwise saves to main bucket
|
||||
func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
// 如果请求写入临时桶且临时桶已配置
|
||||
if temp && s.tempClient != nil {
|
||||
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to COS temp bucket: %w", err)
|
||||
}
|
||||
// Temp bucket still uses legacy URL format for backward compat (auto-expiring)
|
||||
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// 写入主桶
|
||||
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
|
||||
_, err = s.client.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to COS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("cos://%s/%s/%s", s.bucketName, s.region, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// 判断文件属于哪个桶
|
||||
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
|
||||
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.tempClient.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.tempClient.GetCredential().SecretID, s.tempClient.GetCredential().SecretKey, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL for temp bucket: %w", err)
|
||||
}
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
objectName, err := s.parseCosObjectName(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.client.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.client.GetCredential().SecretID, s.client.GetCredential().SecretKey, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseCosObjectName_RejectsLocalScheme(t *testing.T) {
|
||||
svc := &cosFileService{bucketURL: "https://b.cos.ap-shanghai.myqcloud.com/"}
|
||||
_, err := svc.parseCosObjectName("local://10000/exports/img.png")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "local")
|
||||
}
|
||||
|
||||
func TestParseCosObjectName_CosScheme(t *testing.T) {
|
||||
svc := &cosFileService{bucketURL: "https://b.cos.ap-shanghai.myqcloud.com/"}
|
||||
key, err := svc.parseCosObjectName("cos://bucket/ap-shanghai/weknora/10000/exports/a.png")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "weknora/10000/exports/a.png", key)
|
||||
}
|
||||
|
||||
func TestParseCosObjectName_RejectsMinioScheme(t *testing.T) {
|
||||
svc := &cosFileService{bucketURL: "https://b.cos.ap-shanghai.myqcloud.com/"}
|
||||
_, err := svc.parseCosObjectName("minio://wizard-test/10000/exports/img.png")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "minio")
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// DummyFileService is a no-op implementation of the FileService interface
|
||||
// used for testing or when file storage is not required
|
||||
type DummyFileService struct{}
|
||||
|
||||
// CheckConnectivity always succeeds for the dummy service.
|
||||
func (s *DummyFileService) CheckConnectivity(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewDummyFileService creates a new instance of DummyFileService
|
||||
func NewDummyFileService() interfaces.FileService {
|
||||
return &DummyFileService{}
|
||||
}
|
||||
|
||||
// SaveFile pretends to save a file but just returns a random UUID
|
||||
// This is useful for testing without actual file operations
|
||||
func (s *DummyFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
return fmt.Sprintf("dummy://%d/%s", tenantID, uuid.New().String()), nil
|
||||
}
|
||||
|
||||
// GetFile always returns an error as dummy service doesn't store files
|
||||
func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
// DeleteFile is a no-op operation that always succeeds
|
||||
func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes pretends to save bytes but just returns a random UUID
|
||||
func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
return fmt.Sprintf("dummy://%d/%s", tenantID, uuid.New().String()), nil
|
||||
}
|
||||
|
||||
// CopyFile is a no-op for the dummy service: it logs a warning and returns the
|
||||
// source path unchanged (the shared reference is intentional in this stub).
|
||||
func (s *DummyFileService) CopyFile(ctx context.Context, srcPath string, tenantID uint64, knowledgeID string) (string, error) {
|
||||
logger.Warnf(ctx, "[dummy] CopyFile no-op: returning source path %q unchanged (no real copy performed)", srcPath)
|
||||
return srcPath, nil
|
||||
}
|
||||
|
||||
// GetFileURL returns the file path as URL (dummy implementation)
|
||||
func (s *DummyFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
return filePath, nil
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStorageClientsRejectUnsafeEndpointAtConstruction(t *testing.T) {
|
||||
const endpoint = "http://169.254.169.254/latest/meta-data"
|
||||
tests := map[string]func() error{
|
||||
"minio": func() error {
|
||||
_, err := newMinioClient(endpoint, "ak", "sk", "bucket", false)
|
||||
return err
|
||||
},
|
||||
"s3": func() error {
|
||||
_, err := newS3Client(endpoint, "ak", "sk", "bucket", "region", "", true)
|
||||
return err
|
||||
},
|
||||
"obs": func() error {
|
||||
return CheckObsConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket")
|
||||
},
|
||||
"oss": func() error {
|
||||
_, err := newOSSClient(endpoint, "region", "ak", "sk")
|
||||
return err
|
||||
},
|
||||
"tos": func() error {
|
||||
return CheckTosConnectivity(context.Background(), endpoint, "region", "ak", "sk", "bucket")
|
||||
},
|
||||
"ks3": func() error {
|
||||
_, err := newKS3Client(endpoint, "region", "ak", "sk")
|
||||
return err
|
||||
},
|
||||
}
|
||||
for name, run := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := run(); err == nil {
|
||||
t.Fatal("expected unsafe endpoint to be rejected")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package file
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrCrossBackendCopy is returned by CopyFile implementations when the source
|
||||
// path belongs to a different storage provider than the destination service.
|
||||
// PR1 only supports same-backend (server-side) copies; cross-backend streaming
|
||||
// copy is intentionally not implemented yet.
|
||||
var ErrCrossBackendCopy = errors.New("file: cross-backend copy not supported")
|
||||
@@ -0,0 +1,184 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// NewFileServiceFromStorageConfig builds a provider-specific FileService from tenant storage config.
|
||||
// provider can be empty; in that case it falls back to sec.DefaultProvider.
|
||||
// Returns the resolved provider name together with the service.
|
||||
func NewFileServiceFromStorageConfig(
|
||||
provider string,
|
||||
sec *types.StorageEngineConfig,
|
||||
localBaseDir string,
|
||||
) (interfaces.FileService, string, error) {
|
||||
p := strings.ToLower(strings.TrimSpace(provider))
|
||||
if p == "" && sec != nil {
|
||||
p = strings.ToLower(strings.TrimSpace(sec.DefaultProvider))
|
||||
}
|
||||
if p == "" {
|
||||
return nil, "", fmt.Errorf("empty provider")
|
||||
}
|
||||
|
||||
if localBaseDir == "" {
|
||||
localBaseDir = strings.TrimSpace(os.Getenv("LOCAL_STORAGE_BASE_DIR"))
|
||||
}
|
||||
if localBaseDir == "" {
|
||||
localBaseDir = "/data/files"
|
||||
}
|
||||
|
||||
switch p {
|
||||
case "local":
|
||||
baseDir := localBaseDir
|
||||
if sec != nil && sec.Local != nil {
|
||||
rawPrefix := strings.TrimSpace(sec.Local.PathPrefix)
|
||||
prefix := strings.Trim(rawPrefix, "/\\")
|
||||
if prefix != "" {
|
||||
candidate := filepath.Join(baseDir, prefix)
|
||||
if safeBaseDir, err := secutils.SafePathUnderBase(baseDir, candidate); err == nil {
|
||||
baseDir = safeBaseDir
|
||||
}
|
||||
}
|
||||
}
|
||||
externalURL := strings.TrimSpace(os.Getenv("APP_EXTERNAL_URL"))
|
||||
return NewLocalFileService(baseDir, externalURL), p, nil
|
||||
|
||||
case "minio":
|
||||
if sec == nil || sec.MinIO == nil {
|
||||
return nil, p, fmt.Errorf("missing minio config")
|
||||
}
|
||||
var endpoint, accessKeyID, secretAccessKey string
|
||||
if sec.MinIO.Mode == "remote" {
|
||||
endpoint = strings.TrimSpace(sec.MinIO.Endpoint)
|
||||
accessKeyID = strings.TrimSpace(sec.MinIO.AccessKeyID)
|
||||
secretAccessKey = strings.TrimSpace(sec.MinIO.SecretAccessKey)
|
||||
} else {
|
||||
endpoint = strings.TrimSpace(os.Getenv("MINIO_ENDPOINT"))
|
||||
accessKeyID = strings.TrimSpace(os.Getenv("MINIO_ACCESS_KEY_ID"))
|
||||
secretAccessKey = strings.TrimSpace(os.Getenv("MINIO_SECRET_ACCESS_KEY"))
|
||||
}
|
||||
bucketName := strings.TrimSpace(sec.MinIO.BucketName)
|
||||
if bucketName == "" {
|
||||
bucketName = strings.TrimSpace(os.Getenv("MINIO_BUCKET_NAME"))
|
||||
}
|
||||
if endpoint == "" || accessKeyID == "" || secretAccessKey == "" || bucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete minio config")
|
||||
}
|
||||
svc, err := NewMinioFileService(endpoint, accessKeyID, secretAccessKey, bucketName, sec.MinIO.UseSSL)
|
||||
return svc, p, err
|
||||
|
||||
case "cos":
|
||||
if sec == nil || sec.COS == nil || sec.COS.SecretID == "" || sec.COS.SecretKey == "" || sec.COS.BucketName == "" || sec.COS.Region == "" {
|
||||
return nil, p, fmt.Errorf("incomplete cos config")
|
||||
}
|
||||
pathPrefix := strings.TrimSpace(sec.COS.PathPrefix)
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora"
|
||||
}
|
||||
svc, err := NewCosFileServiceWithTempBucket(sec.COS.BucketName, sec.COS.Region, sec.COS.SecretID, sec.COS.SecretKey, pathPrefix, sec.COS.TempBucketName, sec.COS.TempRegion)
|
||||
return svc, p, err
|
||||
|
||||
case "tos":
|
||||
if sec == nil || sec.TOS == nil || sec.TOS.Endpoint == "" || sec.TOS.Region == "" || sec.TOS.AccessKey == "" || sec.TOS.SecretKey == "" || sec.TOS.BucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete tos config")
|
||||
}
|
||||
svc, err := NewTosFileServiceWithTempBucket(sec.TOS.Endpoint, sec.TOS.Region, sec.TOS.AccessKey, sec.TOS.SecretKey, sec.TOS.BucketName, sec.TOS.PathPrefix, sec.TOS.TempBucketName, sec.TOS.TempRegion)
|
||||
return svc, p, err
|
||||
case "s3":
|
||||
if sec == nil || sec.S3 == nil || sec.S3.Region == "" || sec.S3.BucketName == "" || (sec.S3.AccessKey == "") != (sec.S3.SecretKey == "") {
|
||||
return nil, p, fmt.Errorf("incomplete s3 config")
|
||||
}
|
||||
pathPrefix := strings.TrimSpace(sec.S3.PathPrefix)
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora/"
|
||||
}
|
||||
svc, err := NewS3FileServiceWithOptions(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix, sec.S3.ForcePathStyle)
|
||||
return svc, p, err
|
||||
|
||||
case "obs":
|
||||
obsEndpoint, obsRegion, obsAccessKey := "", "", ""
|
||||
obsSecretKey, obsBucketName, obsPathPrefix := "", "", ""
|
||||
if sec != nil && sec.OBS != nil {
|
||||
obsEndpoint = strings.TrimSpace(sec.OBS.Endpoint)
|
||||
obsRegion = strings.TrimSpace(sec.OBS.Region)
|
||||
obsAccessKey = strings.TrimSpace(sec.OBS.AccessKey)
|
||||
obsSecretKey = strings.TrimSpace(sec.OBS.SecretKey)
|
||||
obsBucketName = strings.TrimSpace(sec.OBS.BucketName)
|
||||
obsPathPrefix = strings.TrimSpace(sec.OBS.PathPrefix)
|
||||
}
|
||||
if obsEndpoint == "" {
|
||||
obsEndpoint = strings.TrimSpace(os.Getenv("OBS_ENDPOINT"))
|
||||
}
|
||||
if obsRegion == "" {
|
||||
obsRegion = strings.TrimSpace(os.Getenv("OBS_REGION"))
|
||||
}
|
||||
if obsAccessKey == "" {
|
||||
obsAccessKey = strings.TrimSpace(os.Getenv("OBS_ACCESS_KEY"))
|
||||
}
|
||||
if obsSecretKey == "" {
|
||||
obsSecretKey = strings.TrimSpace(os.Getenv("OBS_SECRET_KEY"))
|
||||
}
|
||||
if obsBucketName == "" {
|
||||
obsBucketName = strings.TrimSpace(os.Getenv("OBS_BUCKET_NAME"))
|
||||
}
|
||||
if obsPathPrefix == "" {
|
||||
obsPathPrefix = strings.TrimSpace(os.Getenv("OBS_PATH_PREFIX"))
|
||||
}
|
||||
if obsPathPrefix == "" {
|
||||
obsPathPrefix = "weknora/"
|
||||
}
|
||||
if obsEndpoint == "" || obsAccessKey == "" || obsSecretKey == "" || obsBucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete obs config")
|
||||
}
|
||||
if obsRegion == "" {
|
||||
obsRegion = "cn-north-4"
|
||||
}
|
||||
svc, err := NewObsFileService(obsEndpoint, obsRegion, obsAccessKey, obsSecretKey, obsBucketName, obsPathPrefix)
|
||||
return svc, p, err
|
||||
|
||||
case "oss":
|
||||
if sec == nil || sec.OSS == nil || sec.OSS.Endpoint == "" || sec.OSS.Region == "" || sec.OSS.AccessKey == "" || sec.OSS.SecretKey == "" || sec.OSS.BucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete oss config")
|
||||
}
|
||||
pathPrefix := strings.TrimSpace(sec.OSS.PathPrefix)
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora/"
|
||||
}
|
||||
var svc interfaces.FileService
|
||||
var err error
|
||||
if sec.OSS.UseTempBucket && sec.OSS.TempBucketName != "" {
|
||||
svc, err = NewOssFileServiceWithTempBucket(
|
||||
sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey,
|
||||
sec.OSS.BucketName, pathPrefix,
|
||||
sec.OSS.TempBucketName, sec.OSS.TempRegion,
|
||||
)
|
||||
} else {
|
||||
svc, err = NewOssFileService(
|
||||
sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey,
|
||||
sec.OSS.BucketName, pathPrefix,
|
||||
)
|
||||
}
|
||||
return svc, p, err
|
||||
|
||||
case "ks3":
|
||||
if sec == nil || sec.KS3 == nil || sec.KS3.Endpoint == "" || sec.KS3.Region == "" || sec.KS3.AccessKey == "" || sec.KS3.SecretKey == "" || sec.KS3.BucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete ks3 config")
|
||||
}
|
||||
pathPrefix := strings.TrimSpace(sec.KS3.PathPrefix)
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora/"
|
||||
}
|
||||
svc, err := NewKS3FileService(sec.KS3.Endpoint, sec.KS3.Region, sec.KS3.AccessKey, sec.KS3.SecretKey, sec.KS3.BucketName, pathPrefix)
|
||||
return svc, p, err
|
||||
|
||||
default:
|
||||
return nil, p, fmt.Errorf("unsupported provider %q", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
ks3aws "github.com/ks3sdklib/aws-sdk-go/aws"
|
||||
"github.com/ks3sdklib/aws-sdk-go/aws/credentials"
|
||||
ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
const ks3Scheme = "ks3://"
|
||||
|
||||
// ks3FileService implements FileService for Kingsoft Cloud KS3.
|
||||
// KS3 uses V2 signing by default and virtual-hosted style addressing,
|
||||
// so it cannot be handled by the generic S3 provider without workarounds.
|
||||
type ks3FileService struct {
|
||||
client *ks3s3.S3
|
||||
bucketName string
|
||||
pathPrefix string
|
||||
}
|
||||
|
||||
// NewKS3FileService creates a KS3 file service and ensures the bucket exists.
|
||||
func NewKS3FileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
|
||||
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pathPrefix = strings.Trim(pathPrefix, "/")
|
||||
|
||||
svc := &ks3FileService{
|
||||
client: client,
|
||||
bucketName: bucketName,
|
||||
pathPrefix: pathPrefix,
|
||||
}
|
||||
|
||||
if err := ensureKS3Bucket(client, bucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe KS3 endpoint: %w", err)
|
||||
}
|
||||
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
client := ks3s3.New(&ks3aws.Config{
|
||||
Credentials: creds,
|
||||
Region: region,
|
||||
Endpoint: endpoint,
|
||||
DisableSSL: false,
|
||||
S3ForcePathStyle: false, // KS3 uses virtual-hosted style
|
||||
SignerVersion: "V2", // KS3 recommends V2 signing
|
||||
MaxRetries: 3,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
})
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
|
||||
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// Bucket doesn't exist, try to create it
|
||||
_, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
if createErr != nil {
|
||||
return fmt.Errorf("failed to create KS3 bucket %q: %w", bucketName, createErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckKS3Connectivity tests KS3 connectivity using the provided credentials.
|
||||
func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.HeadBucket(&ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(bucketName),
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-done:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func joinKS3Key(parts ...string) string {
|
||||
filtered := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
p = strings.Trim(p, "/")
|
||||
if p != "" {
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
}
|
||||
return strings.Join(filtered, "/")
|
||||
}
|
||||
|
||||
func parseKS3FilePath(filePath string) (bucket, objectKey string, err error) {
|
||||
if !strings.HasPrefix(filePath, ks3Scheme) {
|
||||
return "", "", fmt.Errorf("invalid KS3 file path: %s", filePath)
|
||||
}
|
||||
rest := strings.TrimPrefix(filePath, ks3Scheme)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("invalid KS3 file path: %s", filePath)
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext)
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
Body: src,
|
||||
ContentType: ks3aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to KS3: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, objectKey), nil
|
||||
}
|
||||
|
||||
func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext)
|
||||
|
||||
_, err = s.client.PutObject(&ks3s3.PutObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: ks3aws.String(utils.GetContentTypeByExt(ext)),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to KS3: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, objectKey), nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing KS3 object to a new knowledge-owned object using a
|
||||
// server-side CopyObject (no data leaves KS3). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath is not a ks3:// path.
|
||||
func (s *ks3FileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
srcBucket, srcKey, err := parseKS3FilePath(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ks3 copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
if err := utils.SafeObjectKey(srcKey); err != nil {
|
||||
return "", fmt.Errorf("invalid source path: %w", err)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
destKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext)
|
||||
|
||||
_, err = s.client.CopyObject(&ks3s3.CopyObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(destKey),
|
||||
SourceBucket: ks3aws.String(srcBucket),
|
||||
SourceKey: ks3aws.String(srcKey),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in KS3: %w", err)
|
||||
}
|
||||
|
||||
newPath := fmt.Sprintf("%s%s/%s", ks3Scheme, s.bucketName, destKey)
|
||||
logger.Infof(ctx, "Copied KS3 object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
_, objectKey, err := parseKS3FilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectKey); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
resp, err := s.client.GetObject(&ks3s3.GetObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from KS3: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
_, objectKey, err := parseKS3FilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectKey); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file from KS3: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ks3FileService) CheckConnectivity(ctx context.Context) error {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-done:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
_, objectKey, err := parseKS3FilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectKey); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
url, err := s.client.GeneratePresignedUrl(&ks3s3.GeneratePresignedUrlInput{
|
||||
Bucket: ks3aws.String(s.bucketName),
|
||||
Key: ks3aws.String(objectKey),
|
||||
HTTPMethod: ks3s3.HTTPMethod("GET"),
|
||||
Expires: int64((24 * time.Hour).Seconds()),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate KS3 presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return url, nil
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// localFileService implements the FileService interface for local file system storage
|
||||
type localFileService struct {
|
||||
baseDir string // Base directory for file storage
|
||||
externalURL string // External URL base for presigned URL generation (empty = return local:// paths)
|
||||
}
|
||||
|
||||
const localScheme = "local://"
|
||||
|
||||
// CheckConnectivity verifies the local storage directory exists and is accessible.
|
||||
func (s *localFileService) CheckConnectivity(ctx context.Context) error {
|
||||
info, err := os.Stat(s.baseDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage directory not accessible: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("storage path is not a directory: %s", s.baseDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewLocalFileService creates a new local file service instance.
|
||||
// externalURL is the externally-reachable base URL (e.g. "https://weknora.example.com");
|
||||
// when set, GetFileURL returns presigned HTTP URLs instead of local:// paths.
|
||||
func NewLocalFileService(baseDir, externalURL string) interfaces.FileService {
|
||||
return &localFileService{
|
||||
baseDir: baseDir,
|
||||
externalURL: strings.TrimRight(externalURL, "/"),
|
||||
}
|
||||
}
|
||||
|
||||
// SaveFile stores an uploaded file to the local file system
|
||||
// The file is stored in a directory structure: baseDir/tenantID/knowledgeID/filename
|
||||
// Returns the full file path or an error if saving fails
|
||||
func (s *localFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
logger.Info(ctx, "Starting to save file locally")
|
||||
logger.Infof(ctx, "File information: name=%s, size=%d, tenant ID=%d, knowledge ID=%s",
|
||||
file.Filename, file.Size, tenantID, knowledgeID)
|
||||
|
||||
// Create storage directory with tenant and knowledge ID
|
||||
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), knowledgeID)
|
||||
if _, err := secutils.SafePathUnderBase(s.baseDir, dir); err != nil {
|
||||
logger.Errorf(ctx, "Path traversal denied for SaveFile dir: %v", err)
|
||||
return "", fmt.Errorf("invalid path: %w", err)
|
||||
}
|
||||
logger.Infof(ctx, "Creating directory: %s", dir)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create directory: %v", err)
|
||||
return "", fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Generate unique filename using timestamp
|
||||
ext := filepath.Ext(file.Filename)
|
||||
filename := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext)
|
||||
filePath := filepath.Join(dir, filename)
|
||||
logger.Infof(ctx, "Generated file path: %s", filePath)
|
||||
|
||||
// Open source file for reading
|
||||
logger.Info(ctx, "Opening source file")
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to open source file: %v", err)
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Create destination file for writing
|
||||
logger.Info(ctx, "Creating destination file")
|
||||
dst, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to create destination file: %v", err)
|
||||
return "", fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
// Copy content from source to destination
|
||||
logger.Info(ctx, "Copying file content")
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
logger.Errorf(ctx, "Failed to copy file content: %v", err)
|
||||
return "", fmt.Errorf("failed to save file: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "File saved successfully: %s", filePath)
|
||||
// Return provider:// path format: local://{relative_path}
|
||||
relPath, _ := filepath.Rel(s.baseDir, filePath)
|
||||
return localScheme + filepath.ToSlash(relPath), nil
|
||||
}
|
||||
|
||||
// GetFile retrieves a file from the local file system by its path
|
||||
// Returns a ReadCloser for reading the file content
|
||||
// Supports both provider scheme: local://{relative_path} and legacy absolute paths.
|
||||
// 路径必须在 baseDir 下,防止路径遍历(如 ../../)
|
||||
func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
logger.Infof(ctx, "Getting file: %s", filePath)
|
||||
|
||||
candidate := s.normalizePathForBase(filePath)
|
||||
resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Path traversal denied for GetFile: %v", err)
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
file, err := os.Open(resolved)
|
||||
if err != nil {
|
||||
// baseDir/resolved are logged so a storage base-dir mismatch (e.g.
|
||||
// writer and reader started with different LOCAL_STORAGE_BASE_DIR)
|
||||
// is immediately visible instead of just "no such file or directory".
|
||||
logger.Errorf(ctx, "Failed to open file: baseDir=%s resolvedPath=%s err=%v", s.baseDir, resolved, err)
|
||||
return nil, fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
logger.Info(ctx, "File opened successfully")
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// DeleteFile removes a file from the local file system
|
||||
// Returns an error if deletion fails
|
||||
// 路径必须在 baseDir 下,防止路径遍历(如 ../../)
|
||||
func (s *localFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
logger.Infof(ctx, "Deleting file: %s", filePath)
|
||||
|
||||
candidate := s.normalizePathForBase(filePath)
|
||||
resolved, err := secutils.SafePathUnderBase(s.baseDir, candidate)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Path traversal denied for DeleteFile: %v", err)
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
err = os.Remove(resolved)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete file: %v", err)
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
|
||||
logger.Info(ctx, "File deleted successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing local object to a new knowledge-owned object.
|
||||
// The destination uses the same layout as SaveFile (baseDir/{tenantID}/{knowledgeID}/{unique}{ext}),
|
||||
// and the copy is a real byte-for-byte copy (no hardlink) so deleting the source
|
||||
// never affects it. Returns ErrCrossBackendCopy when srcPath is not a local path.
|
||||
func (s *localFileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
// Only local paths are accepted. A provider scheme other than local://
|
||||
// (e.g. s3://, minio://) means a cross-backend copy, which this service
|
||||
// does not support. Legacy bare/absolute paths have no scheme and pass.
|
||||
if i := strings.Index(srcPath, "://"); i >= 0 && srcPath[:i+3] != localScheme {
|
||||
return "", fmt.Errorf("local file service cannot copy %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
|
||||
// Validate and resolve the source path under baseDir (same guard as GetFile).
|
||||
srcCandidate := s.normalizePathForBase(srcPath)
|
||||
srcResolved, err := secutils.SafePathUnderBase(s.baseDir, srcCandidate)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Path traversal denied for CopyFile src: %v", err)
|
||||
return "", fmt.Errorf("invalid source path: %w", err)
|
||||
}
|
||||
|
||||
// Build destination path with the knowledge-owned layout.
|
||||
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), knowledgeID)
|
||||
if _, err := secutils.SafePathUnderBase(s.baseDir, dir); err != nil {
|
||||
logger.Errorf(ctx, "Path traversal denied for CopyFile dir: %v", err)
|
||||
return "", fmt.Errorf("invalid path: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
filename := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext)
|
||||
dstPath := filepath.Join(dir, filename)
|
||||
|
||||
src, err := os.Open(srcResolved)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open source file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
if err := copyLocalFileContents(dstPath, src); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
relPath, _ := filepath.Rel(s.baseDir, dstPath)
|
||||
newPath := localScheme + filepath.ToSlash(relPath)
|
||||
logger.Infof(ctx, "Copied local file %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
func copyLocalFileContents(dstPath string, src io.Reader) (err error) {
|
||||
dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create destination file: %w", err)
|
||||
}
|
||||
// A failed copy returns no object path, so its caller cannot reclaim a
|
||||
// partial file. Keep cleanup here, including errors flushed on close.
|
||||
defer func() {
|
||||
if closeErr := dst.Close(); closeErr != nil {
|
||||
err = errors.Join(err, fmt.Errorf("failed to close copied file: %w", closeErr))
|
||||
}
|
||||
if err != nil {
|
||||
if removeErr := os.Remove(dstPath); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||||
err = errors.Join(err, fmt.Errorf("failed to remove partial copy: %w", removeErr))
|
||||
}
|
||||
}
|
||||
}()
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
return fmt.Errorf("failed to copy file content: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to a file and returns the file path
|
||||
// temp parameter is ignored for local storage (no auto-expiration support)
|
||||
// fileName 仅允许安全文件名,禁止路径遍历(如 ../../)
|
||||
func (s *localFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
logger.Infof(ctx, "Saving bytes data: fileName=%s, size=%d, tenantID=%d, temp=%v", fileName, len(data), tenantID, temp)
|
||||
|
||||
safeName, err := secutils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Invalid fileName for SaveBytes: %v", err)
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
|
||||
// Create storage directory with tenant ID
|
||||
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), "exports")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create directory: %v", err)
|
||||
return "", fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Generate unique filename using timestamp
|
||||
ext := filepath.Ext(safeName)
|
||||
baseName := safeName[:len(safeName)-len(ext)]
|
||||
uniqueFileName := fmt.Sprintf("%s_%d%s", baseName, time.Now().UnixNano(), ext)
|
||||
filePath := filepath.Join(dir, uniqueFileName)
|
||||
|
||||
// Write data to file
|
||||
if err := os.WriteFile(filePath, data, 0o644); err != nil {
|
||||
logger.Errorf(ctx, "Failed to write file: %v", err)
|
||||
return "", fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Bytes data saved successfully: %s", filePath)
|
||||
relPath, _ := filepath.Rel(s.baseDir, filePath)
|
||||
return localScheme + filepath.ToSlash(relPath), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a download URL for the file.
|
||||
// When externalURL is configured, returns a presigned HTTP URL suitable for external access.
|
||||
// Otherwise returns the local://... path for backward compatibility.
|
||||
func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// Normalize to provider:// format.
|
||||
normalized := filePath
|
||||
if !strings.HasPrefix(filePath, localScheme) {
|
||||
relPath, err := filepath.Rel(s.baseDir, filePath)
|
||||
if err != nil {
|
||||
normalized = filePath
|
||||
} else {
|
||||
normalized = localScheme + filepath.ToSlash(relPath)
|
||||
}
|
||||
}
|
||||
|
||||
// If external URL is configured, generate a presigned HTTP URL.
|
||||
if s.externalURL != "" {
|
||||
// Tenant ID is parsed from the storage path, which encodes the
|
||||
// resource owner's tenant (not the caller's). The verifier on
|
||||
// /api/v1/files/presigned uses this ID to look up the owning
|
||||
// tenant's StorageEngineConfig — using the caller's tenant would
|
||||
// break cross-tenant shared resources (e.g. shared KB images).
|
||||
tenantID := secutils.ParseTenantIDFromStoragePath(normalized)
|
||||
presignedURL, err := secutils.SignFileURL(s.externalURL, normalized, tenantID, 0)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to generate presigned URL for %s: %v, returning local:// path", normalized, err)
|
||||
return normalized, nil
|
||||
}
|
||||
return presignedURL, nil
|
||||
}
|
||||
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// normalizePathForBase keeps backward compatibility for legacy file paths:
|
||||
// - provider scheme: "local://tenant/.." → baseDir/tenant/..
|
||||
// - absolute path: "/data/files/tenant/.."
|
||||
// - path under base dir: "tenant/.."
|
||||
// - legacy relative with base prefix: "data/files/tenant/.."
|
||||
func (s *localFileService) normalizePathForBase(filePath string) string {
|
||||
// Handle provider:// format: local://{relPath}
|
||||
if strings.HasPrefix(filePath, localScheme) {
|
||||
relPath := strings.TrimPrefix(filePath, localScheme)
|
||||
return filepath.Join(s.baseDir, filepath.FromSlash(relPath))
|
||||
}
|
||||
|
||||
clean := filepath.Clean(strings.TrimSpace(filePath))
|
||||
if clean == "." || clean == "" {
|
||||
return clean
|
||||
}
|
||||
if filepath.IsAbs(clean) {
|
||||
return clean
|
||||
}
|
||||
|
||||
// Strip duplicated base prefix in legacy relative paths, e.g. "data/files/..."
|
||||
baseClean := filepath.Clean(s.baseDir)
|
||||
baseNoSlash := strings.Trim(baseClean, string(filepath.Separator))
|
||||
cleanNoDot := strings.TrimPrefix(clean, "."+string(filepath.Separator))
|
||||
if strings.HasPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator)) {
|
||||
cleanNoDot = strings.TrimPrefix(cleanNoDot, baseNoSlash+string(filepath.Separator))
|
||||
}
|
||||
return filepath.Join(baseClean, cleanNoDot)
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// seedLocalObject writes a file under baseDir at the SaveBytes-style layout
|
||||
// (tenant/exports) and returns its local:// path.
|
||||
func seedLocalObject(t *testing.T, baseDir string, tenantID uint64, name string, data []byte) string {
|
||||
t.Helper()
|
||||
dir := filepath.Join(baseDir, "0", "src-knowledge")
|
||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
||||
full := filepath.Join(dir, name)
|
||||
require.NoError(t, os.WriteFile(full, data, 0o644))
|
||||
rel, err := filepath.Rel(baseDir, full)
|
||||
require.NoError(t, err)
|
||||
return localScheme + filepath.ToSlash(rel)
|
||||
}
|
||||
|
||||
func readLocal(t *testing.T, svc interface {
|
||||
GetFile(context.Context, string) (io.ReadCloser, error)
|
||||
}, path string) []byte {
|
||||
t.Helper()
|
||||
rc, err := svc.GetFile(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
defer rc.Close()
|
||||
b, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
// TestLocalCopyFile_IndependentCopy verifies that CopyFile creates a real,
|
||||
// independent copy: it exists at a new knowledge-owned path AND survives
|
||||
// deletion of the source object (the C1/C2 regression this PR fixes).
|
||||
func TestLocalCopyFile_IndependentCopy(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
svc := NewLocalFileService(base, "")
|
||||
|
||||
content := []byte("hello deep copy")
|
||||
srcPath := seedLocalObject(t, base, 0, "doc.txt", content)
|
||||
|
||||
newPath, err := svc.CopyFile(context.Background(), srcPath, 42, "dst-knowledge")
|
||||
require.NoError(t, err)
|
||||
|
||||
// New path must be knowledge-owned: local://42/dst-knowledge/<unique>.txt
|
||||
require.True(t, strings.HasPrefix(newPath, localScheme+"42/dst-knowledge/"),
|
||||
"unexpected dst path: %s", newPath)
|
||||
assert.Equal(t, ".txt", filepath.Ext(newPath))
|
||||
|
||||
// Copy is readable and byte-identical.
|
||||
assert.Equal(t, content, readLocal(t, svc, newPath))
|
||||
|
||||
// Delete the source — the copy must remain intact.
|
||||
require.NoError(t, svc.DeleteFile(context.Background(), srcPath))
|
||||
assert.Equal(t, content, readLocal(t, svc, newPath),
|
||||
"copy should survive deletion of source")
|
||||
}
|
||||
|
||||
// TestLocalCopyFile_CrossBackend verifies that handing a non-local provider
|
||||
// scheme to the local service is rejected with ErrCrossBackendCopy.
|
||||
func TestLocalCopyFile_CrossBackend(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
svc := NewLocalFileService(base, "")
|
||||
|
||||
_, err := svc.CopyFile(context.Background(), "s3://bucket/10/exports/a.png", 7, "k")
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Is(err, ErrCrossBackendCopy),
|
||||
"expected ErrCrossBackendCopy, got %v", err)
|
||||
}
|
||||
|
||||
// TestLocalCopyFile_TraversalRejected verifies that the same path guard used by
|
||||
// GetFile/DeleteFile rejects a traversal source path.
|
||||
func TestLocalCopyFile_TraversalRejected(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
svc := NewLocalFileService(base, "")
|
||||
|
||||
_, err := svc.CopyFile(context.Background(), localScheme+"../../etc/passwd", 7, "k")
|
||||
require.Error(t, err)
|
||||
assert.False(t, errors.Is(err, ErrCrossBackendCopy),
|
||||
"traversal should be a path error, not cross-backend")
|
||||
}
|
||||
|
||||
type failedLocalCopyReader struct{ err error }
|
||||
|
||||
func (r failedLocalCopyReader) Read([]byte) (int, error) { return 0, r.err }
|
||||
|
||||
func TestLocalCopyFile_ReadFailureRemovesUnreturnedFile(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
svc := NewLocalFileService(base, "")
|
||||
source := filepath.Join(base, "source-directory")
|
||||
require.NoError(t, os.Mkdir(source, 0o755))
|
||||
path, err := svc.CopyFile(context.Background(), localScheme+"source-directory", 7, "target")
|
||||
require.ErrorContains(t, err, "failed to copy file content")
|
||||
require.Empty(t, path)
|
||||
entries, err := os.ReadDir(filepath.Join(base, "7", "target"))
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, entries)
|
||||
info, err := os.Stat(source)
|
||||
require.NoError(t, err)
|
||||
require.True(t, info.IsDir())
|
||||
}
|
||||
|
||||
func TestLocalCopyFile_PartialWriteFailureRemovesFile(t *testing.T) {
|
||||
target := filepath.Join(t.TempDir(), "partial.txt")
|
||||
cause := errors.New("source read interrupted")
|
||||
source := io.MultiReader(strings.NewReader("bytes written before failure"), failedLocalCopyReader{err: cause})
|
||||
err := copyLocalFileContents(target, source)
|
||||
require.ErrorIs(t, err, cause)
|
||||
_, err = os.Stat(target)
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestLocalCopyFile_ExistingDestinationRemainsUntouched(t *testing.T) {
|
||||
target := filepath.Join(t.TempDir(), "existing.txt")
|
||||
require.NoError(t, os.WriteFile(target, []byte("original"), 0o600))
|
||||
err := copyLocalFileContents(target, strings.NewReader("replacement"))
|
||||
require.ErrorIs(t, err, os.ErrExist)
|
||||
content, err := os.ReadFile(target)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "original", string(content))
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// extractTenantIDFromPresignedURL pulls the tenant_id query parameter from a
|
||||
// signed URL. Returns "" when the URL is not parseable as a presigned URL.
|
||||
func extractTenantIDFromPresignedURL(t *testing.T, presigned string) string {
|
||||
t.Helper()
|
||||
u, err := url.Parse(presigned)
|
||||
require.NoError(t, err)
|
||||
return u.Query().Get("tenant_id")
|
||||
}
|
||||
|
||||
// TestLocalGetFileURL_TenantIDFromPath verifies that tenant ID is extracted
|
||||
// from the storage path — which encodes the resource owner, so cross-tenant
|
||||
// shared resources resolve to the correct owning tenant's storage config.
|
||||
func TestLocalGetFileURL_TenantIDFromPath(t *testing.T) {
|
||||
t.Setenv("SYSTEM_AES_KEY", "weknora-test-aes-key-32bytes!!!")
|
||||
|
||||
svc := NewLocalFileService("/data/files", "https://weknora.example.com")
|
||||
|
||||
got, err := svc.GetFileURL(context.Background(), "local://7/abc/img.png")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "7", extractTenantIDFromPresignedURL(t, got))
|
||||
}
|
||||
|
||||
// TestLocalGetFileURL_NoExternalURL verifies backward compatibility: without
|
||||
// APP_EXTERNAL_URL, GetFileURL still returns the local:// path unchanged.
|
||||
func TestLocalGetFileURL_NoExternalURL(t *testing.T) {
|
||||
svc := NewLocalFileService("/data/files", "")
|
||||
|
||||
got, err := svc.GetFileURL(context.Background(), "local://1/abc/img.png")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "local://1/abc/img.png", got)
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// minioFileService MinIO file service implementation
|
||||
type minioFileService struct {
|
||||
client *minio.Client
|
||||
bucketName string
|
||||
}
|
||||
|
||||
// newMinioClient creates a bare minioFileService with just the SDK client initialised.
|
||||
// Shared by NewMinioFileService (which also ensures the bucket exists) and
|
||||
// CheckMinioConnectivity (read-only probe).
|
||||
func newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (*minioFileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe MinIO endpoint: %w", err)
|
||||
}
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
Transport: &utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
|
||||
}
|
||||
return &minioFileService{client: client, bucketName: bucketName}, nil
|
||||
}
|
||||
|
||||
// NewMinioFileService creates a MinIO file service.
|
||||
// It verifies that the bucket exists and creates it if missing.
|
||||
func NewMinioFileService(endpoint,
|
||||
accessKeyID, secretAccessKey, bucketName string, useSSL bool,
|
||||
) (interfaces.FileService, error) {
|
||||
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
exists, err := svc.client.BucketExists(context.Background(), bucketName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err = svc.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies MinIO is reachable and, if a bucket is configured,
|
||||
// that the bucket exists. This is a read-only probe — it never creates a bucket.
|
||||
func (s *minioFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if s.bucketName != "" {
|
||||
exists, err := s.client.BucketExists(checkCtx, s.bucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, err := s.client.ListBuckets(checkCtx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckMinioConnectivity tests MinIO connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckMinioConnectivity(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) error {
|
||||
svc, err := newMinioClient(endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// parseMinioFilePath extracts the object name from a provider scheme: minio://{bucket}/{objectKey}
|
||||
func (s *minioFileService) parseMinioFilePath(filePath string) (string, error) {
|
||||
// Provider scheme format: minio://{bucket}/{objectKey}
|
||||
const prefix = "minio://"
|
||||
if !strings.HasPrefix(filePath, prefix) {
|
||||
return "", fmt.Errorf("invalid MinIO file path: %s", filePath)
|
||||
}
|
||||
rest := strings.TrimPrefix(filePath, prefix)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", fmt.Errorf("invalid MinIO file path: %s", filePath)
|
||||
}
|
||||
if parts[0] != s.bucketName {
|
||||
return "", fmt.Errorf("bucket mismatch in path: got %s, want %s", parts[0], s.bucketName)
|
||||
}
|
||||
if err := utils.SafeObjectKey(parts[1]); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
return parts[1], nil
|
||||
}
|
||||
|
||||
// SaveFile saves a file to MinIO
|
||||
func (s *minioFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
// Generate object name
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := fmt.Sprintf("%d/%s/%s%s", tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
// Open file
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Upload file to MinIO
|
||||
_, err = s.client.PutObject(ctx, s.bucketName, objectName, src, file.Size, minio.PutObjectOptions{
|
||||
ContentType: file.Header.Get("Content-Type"),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to MinIO: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFile gets a file from MinIO
|
||||
func (s *minioFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
objectName, err := s.parseMinioFilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
obj, err := s.client.GetObject(ctx, s.bucketName, objectName, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from MinIO: %w", err)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// DeleteFile deletes a file
|
||||
func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
objectName, err := s.parseMinioFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.client.RemoveObject(ctx, s.bucketName, objectName, minio.RemoveObjectOptions{
|
||||
GovernanceBypass: true,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing MinIO object to a new knowledge-owned object using a
|
||||
// server-side CopyObject (no data leaves MinIO). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath is not a minio:// path.
|
||||
func (s *minioFileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
srcKey, err := s.parseMinioFilePath(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
destKey := fmt.Sprintf("%d/%s/%s%s", tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
_, err = s.client.CopyObject(ctx,
|
||||
minio.CopyDestOptions{Bucket: s.bucketName, Object: destKey},
|
||||
minio.CopySrcOptions{Bucket: s.bucketName, Object: srcKey},
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in MinIO: %w", err)
|
||||
}
|
||||
|
||||
newPath := fmt.Sprintf("minio://%s/%s", s.bucketName, destKey)
|
||||
logger.Infof(ctx, "Copied MinIO object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to MinIO and returns the file path
|
||||
// temp parameter is ignored for MinIO (no auto-expiration support in this implementation)
|
||||
func (s *minioFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
objectName := fmt.Sprintf("%d/exports/%s%s", tenantID, uuid.New().String(), ext)
|
||||
|
||||
// Upload bytes to MinIO
|
||||
reader := bytes.NewReader(data)
|
||||
_, err = s.client.PutObject(ctx, s.bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{
|
||||
ContentType: utils.GetContentTypeByExt(ext),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to MinIO: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *minioFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
objectName, err := s.parseMinioFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
presignedURL, err := s.client.PresignedGetObject(ctx, s.bucketName, objectName, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type obsFileService struct {
|
||||
client *s3.Client
|
||||
bucketName string
|
||||
endpoint string
|
||||
region string
|
||||
pathPrefix string
|
||||
proxyDomain string
|
||||
}
|
||||
|
||||
type obsEndpointResolver struct {
|
||||
url string
|
||||
}
|
||||
|
||||
func (r *obsEndpointResolver) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: r.url,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewObsFileService(
|
||||
endpoint, region, accessKeyID, secretAccessKey, bucketName string,
|
||||
pathPrefix string,
|
||||
) (interfaces.FileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe OBS endpoint: %w", err)
|
||||
}
|
||||
|
||||
client := s3.New(s3.Options{
|
||||
Region: region,
|
||||
EndpointResolver: &obsEndpointResolver{url: endpoint},
|
||||
Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""),
|
||||
UsePathStyle: true,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
})
|
||||
|
||||
_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
_, createErr := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if createErr != nil {
|
||||
fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
|
||||
}
|
||||
}
|
||||
|
||||
proxyDomain := strings.TrimSuffix(os.Getenv("OBS_PROXY_DOMAIN"), "/")
|
||||
|
||||
return &obsFileService{
|
||||
client: client,
|
||||
bucketName: bucketName,
|
||||
endpoint: endpoint,
|
||||
region: region,
|
||||
pathPrefix: strings.Trim(pathPrefix, "/"),
|
||||
proxyDomain: proxyDomain,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return fmt.Errorf("unsafe OBS endpoint: %w", err)
|
||||
}
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client := s3.New(s3.Options{
|
||||
Region: region,
|
||||
EndpointResolver: &obsEndpointResolver{url: endpoint},
|
||||
Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
UsePathStyle: true,
|
||||
HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
})
|
||||
|
||||
_, err := client.HeadBucket(checkCtx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("OBS connectivity check failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := s.client.HeadBucket(checkCtx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("OBS connectivity check failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) parseObsFilePath(filePath string) (string, error) {
|
||||
prefix := s.getPrifix()
|
||||
|
||||
if strings.HasPrefix(filePath, prefix) {
|
||||
rest := strings.TrimPrefix(filePath, prefix)
|
||||
// With proxy domain: path is {prefix}/{objectKey} (no bucket name)
|
||||
if s.proxyDomain != "" {
|
||||
rest = strings.TrimPrefix(rest, "/")
|
||||
if rest != "" {
|
||||
return rest, nil
|
||||
}
|
||||
return "", fmt.Errorf("invalid OBS file path: %s", filePath)
|
||||
}
|
||||
// Without proxy domain: path is {prefix}/{bucketName}/{objectKey}
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) == 2 && parts[0] == s.bucketName && parts[1] != "" {
|
||||
return parts[1], nil
|
||||
}
|
||||
return "", fmt.Errorf("invalid OBS file path: %s", filePath)
|
||||
}
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) getPrifix() string {
|
||||
if s.proxyDomain != "" {
|
||||
return s.proxyDomain + "/"
|
||||
}
|
||||
return "obs://"
|
||||
}
|
||||
|
||||
func (s *obsFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
|
||||
var objectKey string
|
||||
if s.pathPrefix != "" {
|
||||
objectKey = fmt.Sprintf("%s/%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
} else {
|
||||
objectKey = fmt.Sprintf("%d/%s/%s%s", tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: src,
|
||||
ContentLength: aws.Int64(file.Size),
|
||||
ContentType: aws.String(contentType),
|
||||
// ACL: "private",
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to OBS: %w", err)
|
||||
}
|
||||
prefix := s.getPrifix()
|
||||
if s.proxyDomain != "" {
|
||||
return fmt.Sprintf("%s%s", prefix, objectKey), nil
|
||||
}
|
||||
return fmt.Sprintf("%s%s/%s", prefix, s.bucketName, objectKey), nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
objectKey, err := s.parseObsFilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from OBS: %w", err)
|
||||
}
|
||||
|
||||
return output.Body, nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
objectKey, err := s.parseObsFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file from OBS: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
if strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") {
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
objectKey, err := s.parseObsFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if s.proxyDomain != "" {
|
||||
return s.proxyDomain + "/" + strings.TrimPrefix(objectKey, "/"), nil
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s/%s", s.endpoint, s.bucketName, strings.TrimPrefix(objectKey, "/")), nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing OBS object to a new knowledge-owned object using a
|
||||
// server-side CopyObject (OBS is S3-compatible). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath does not belong
|
||||
// to this OBS service.
|
||||
func (s *obsFileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
// Reject paths that do not use this service's prefix (proxy domain or obs://).
|
||||
// parseObsFilePath falls back to returning the raw input for unknown prefixes,
|
||||
// so guard explicitly here to detect cross-backend sources.
|
||||
if !strings.HasPrefix(srcPath, s.getPrifix()) {
|
||||
return "", fmt.Errorf("obs copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
srcKey, err := s.parseObsFilePath(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("obs copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
var destKey string
|
||||
if s.pathPrefix != "" {
|
||||
destKey = fmt.Sprintf("%s/%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
} else {
|
||||
destKey = fmt.Sprintf("%d/%s/%s%s", tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
}
|
||||
|
||||
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
|
||||
// (url.PathEscape would turn them into %2F and break the bucket/key split).
|
||||
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
CopySource: aws.String(s.bucketName + "/" + srcKey),
|
||||
Key: aws.String(destKey),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in OBS: %w", err)
|
||||
}
|
||||
|
||||
prefix := s.getPrifix()
|
||||
var newPath string
|
||||
if s.proxyDomain != "" {
|
||||
newPath = fmt.Sprintf("%s%s", prefix, destKey)
|
||||
} else {
|
||||
newPath = fmt.Sprintf("%s%s/%s", prefix, s.bucketName, destKey)
|
||||
}
|
||||
logger.Infof(ctx, "Copied OBS object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
func (s *obsFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
ext := filepath.Ext(fileName)
|
||||
|
||||
var objectKey string
|
||||
if temp {
|
||||
if s.pathPrefix != "" {
|
||||
objectKey = fmt.Sprintf("%s/temp/%d/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
|
||||
} else {
|
||||
objectKey = fmt.Sprintf("temp/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
}
|
||||
} else {
|
||||
if s.pathPrefix != "" {
|
||||
objectKey = fmt.Sprintf("%s/%d/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
|
||||
} else {
|
||||
objectKey = fmt.Sprintf("%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
}
|
||||
}
|
||||
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(string(data)),
|
||||
ContentType: aws.String("application/octet-stream"),
|
||||
ACL: "public-read",
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to OBS: %w", err)
|
||||
}
|
||||
|
||||
prefix := s.getPrifix()
|
||||
if s.proxyDomain != "" {
|
||||
return fmt.Sprintf("%s%s", prefix, objectKey), nil
|
||||
}
|
||||
return fmt.Sprintf("%s%s/%s", prefix, s.bucketName, objectKey), nil
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// ossFileService implements the FileService interface for Aliyun OSS
|
||||
// using the official Aliyun OSS SDK v2 (github.com/aliyun/alibabacloud-oss-go-sdk-v2).
|
||||
type ossFileService struct {
|
||||
client *oss.Client
|
||||
tempClient *oss.Client
|
||||
pathPrefix string
|
||||
bucketName string
|
||||
tempBucketName string
|
||||
}
|
||||
|
||||
const ossScheme = "oss://"
|
||||
|
||||
// newOSSClient creates an OSS client using the official Aliyun SDK v2.
|
||||
func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe OSS endpoint: %w", err)
|
||||
}
|
||||
creds := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
|
||||
|
||||
cfg := oss.LoadDefaultConfig().
|
||||
WithCredentialsProvider(creds).
|
||||
WithRegion(region).
|
||||
WithEndpoint(endpoint).
|
||||
WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()))
|
||||
|
||||
return oss.NewClient(cfg), nil
|
||||
}
|
||||
|
||||
// ossEnsureBucket checks if the bucket exists and creates it if missing.
|
||||
func ossEnsureBucket(client *oss.Client, bucketName string) error {
|
||||
exists, err := client.IsBucketExist(context.Background(), bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check OSS bucket: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = client.PutBucket(context.Background(), &oss.PutBucketRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
var svcErr *oss.ServiceError
|
||||
if errors.As(err, &svcErr) && svcErr.StatusCode == http.StatusConflict {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to create OSS bucket: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewOssFileService creates an Aliyun OSS file service.
|
||||
// It verifies that the bucket exists and creates it if missing.
|
||||
func NewOssFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
|
||||
return NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewOssFileServiceWithTempBucket creates an Aliyun OSS file service with optional temp bucket.
|
||||
func NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
client, err := newOSSClient(endpoint, region, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ossEnsureBucket(client, bucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tempClient *oss.Client
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region
|
||||
}
|
||||
tempClient, err = newOSSClient(endpoint, tempRegion, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize OSS temp client: %w", err)
|
||||
}
|
||||
if err := ossEnsureBucket(tempClient, tempBucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize pathPrefix: ensure it ends with '/' if not empty
|
||||
if pathPrefix != "" && !strings.HasSuffix(pathPrefix, "/") {
|
||||
pathPrefix += "/"
|
||||
}
|
||||
|
||||
return &ossFileService{
|
||||
client: client,
|
||||
tempClient: tempClient,
|
||||
pathPrefix: pathPrefix,
|
||||
bucketName: bucketName,
|
||||
tempBucketName: tempBucketName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CheckOssConnectivity tests OSS connectivity using the provided credentials.
|
||||
func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
client, err := newOSSClient(endpoint, region, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
exists, err := client.IsBucketExist(ctx, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check OSS bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist or is not accessible", bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseOssFilePath extracts bucket and object key from: oss://{bucket}/{objectKey}
|
||||
func parseOssFilePath(filePath string) (bucketName string, objectKey string, err error) {
|
||||
if !strings.HasPrefix(filePath, ossScheme) {
|
||||
return "", "", fmt.Errorf("invalid OSS file path: %s", filePath)
|
||||
}
|
||||
|
||||
rest := strings.TrimPrefix(filePath, ossScheme)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("invalid OSS file path: %s", filePath)
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies OSS is reachable and the main bucket exists.
|
||||
func (s *ossFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
exists, err := s.client.IsBucketExist(checkCtx, s.bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check OSS bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveFile saves a file to OSS using the Uploader manager for large files.
|
||||
func (s *ossFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
// Use Uploader for files > 10MB (auto multipart with concurrent uploads)
|
||||
const multipartThreshold = 10 * 1024 * 1024
|
||||
if file.Size > multipartThreshold {
|
||||
uploader := s.client.NewUploader(func(uo *oss.UploaderOptions) {
|
||||
uo.PartSize = 10 * 1024 * 1024 // 10MB per part
|
||||
uo.ParallelNum = 3 // 3 concurrent uploads
|
||||
})
|
||||
|
||||
_, err = uploader.UploadFrom(ctx,
|
||||
&oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(s.bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
ContentType: oss.Ptr(contentType),
|
||||
},
|
||||
src,
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to OSS (multipart): %w", err)
|
||||
}
|
||||
} else {
|
||||
_, err = s.client.PutObject(ctx, &oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(s.bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
Body: src,
|
||||
ContentType: oss.Ptr(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to OSS: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to OSS.
|
||||
// If temp is true and temp bucket is configured, saves to temp bucket.
|
||||
// Otherwise saves to main bucket.
|
||||
func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
|
||||
targetBucket := s.bucketName
|
||||
client := s.client
|
||||
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
|
||||
|
||||
if temp && s.tempClient != nil {
|
||||
targetBucket = s.tempBucketName
|
||||
client = s.tempClient
|
||||
objectName = fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
}
|
||||
|
||||
_, err = client.PutObject(ctx, &oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(targetBucket),
|
||||
Key: oss.Ptr(objectName),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: oss.Ptr(utils.GetContentTypeByExt(ext)),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to OSS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("oss://%s/%s", targetBucket, objectName), nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing OSS object to a new knowledge-owned object using a
|
||||
// server-side CopyObject (no data leaves OSS). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath is not an oss:// path.
|
||||
func (s *ossFileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
srcBucket, srcKey, err := parseOssFilePath(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("oss copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
if err := utils.SafeObjectKey(srcKey); err != nil {
|
||||
return "", fmt.Errorf("invalid source path: %w", err)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
destKey := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
_, err = s.client.CopyObject(ctx, &oss.CopyObjectRequest{
|
||||
Bucket: oss.Ptr(s.bucketName),
|
||||
Key: oss.Ptr(destKey),
|
||||
SourceBucket: oss.Ptr(srcBucket),
|
||||
SourceKey: oss.Ptr(srcKey),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in OSS: %w", err)
|
||||
}
|
||||
|
||||
newPath := fmt.Sprintf("oss://%s/%s", s.bucketName, destKey)
|
||||
logger.Infof(ctx, "Copied OSS object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
// GetFile retrieves a file from OSS by its path.
|
||||
func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
bucketName, objectName, err := parseOssFilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
var client *oss.Client
|
||||
if bucketName == s.tempBucketName && s.tempClient != nil {
|
||||
client = s.tempClient
|
||||
} else {
|
||||
client = s.client
|
||||
}
|
||||
|
||||
resp, err := client.GetObject(ctx, &oss.GetObjectRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from OSS: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// DeleteFile removes a file from OSS.
|
||||
func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
bucketName, objectName, err := parseOssFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
var client *oss.Client
|
||||
if bucketName == s.tempBucketName && s.tempClient != nil {
|
||||
client = s.tempClient
|
||||
} else {
|
||||
client = s.client
|
||||
}
|
||||
|
||||
_, err = client.DeleteObject(ctx, &oss.DeleteObjectRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file from OSS: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file.
|
||||
func (s *ossFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
bucketName, objectName, err := parseOssFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
// Determine which client to use
|
||||
var client *oss.Client
|
||||
if bucketName == s.tempBucketName && s.tempClient != nil {
|
||||
client = s.tempClient
|
||||
} else {
|
||||
client = s.client
|
||||
}
|
||||
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
result, err := client.Presign(ctx, &oss.GetObjectRequest{
|
||||
Bucket: oss.Ptr(bucketName),
|
||||
Key: oss.Ptr(objectName),
|
||||
}, oss.PresignExpires(24*time.Hour))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate OSS presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return result.URL, nil
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseOssFilePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantBucket string
|
||||
wantKey string
|
||||
wantErr bool
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "valid path with nested key",
|
||||
input: "oss://my-bucket/123/exports/abc123.csv",
|
||||
wantBucket: "my-bucket",
|
||||
wantKey: "123/exports/abc123.csv",
|
||||
},
|
||||
{
|
||||
name: "valid path with simple key",
|
||||
input: "oss://test-bucket/key",
|
||||
wantBucket: "test-bucket",
|
||||
wantKey: "key",
|
||||
},
|
||||
{
|
||||
name: "valid path with deep nesting",
|
||||
input: "oss://bucket/prefix/tenant/exports/uuid.png",
|
||||
wantBucket: "bucket",
|
||||
wantKey: "prefix/tenant/exports/uuid.png",
|
||||
},
|
||||
{
|
||||
name: "invalid scheme",
|
||||
input: "s3://bucket/key",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "bucket only no key",
|
||||
input: "oss://bucket/",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "scheme only",
|
||||
input: "oss://",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "no slash after bucket",
|
||||
input: "oss://bucket",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "empty bucket name",
|
||||
input: "oss:///some-key",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
bucket, key, err := parseOssFilePath(tt.input)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("parseOssFilePath(%q) expected error, got bucket=%q key=%q", tt.input, bucket, key)
|
||||
}
|
||||
if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) {
|
||||
t.Errorf("parseOssFilePath(%q) error = %v, want containing %q", tt.input, err, tt.errContains)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseOssFilePath(%q) unexpected error: %v", tt.input, err)
|
||||
return
|
||||
}
|
||||
if bucket != tt.wantBucket {
|
||||
t.Errorf("parseOssFilePath(%q) bucket = %q, want %q", tt.input, bucket, tt.wantBucket)
|
||||
}
|
||||
if key != tt.wantKey {
|
||||
t.Errorf("parseOssFilePath(%q) key = %q, want %q", tt.input, key, tt.wantKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOSSClient(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
region string
|
||||
accessKey string
|
||||
secretKey string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid parameters create client",
|
||||
endpoint: "https://oss-cn-hangzhou.aliyuncs.com",
|
||||
region: "cn-hangzhou",
|
||||
accessKey: "test-access-key",
|
||||
secretKey: "test-secret-key",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "custom endpoint",
|
||||
endpoint: "https://example.com",
|
||||
region: "cn-shanghai",
|
||||
accessKey: "ak",
|
||||
secretKey: "sk",
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client, err := newOSSClient(tt.endpoint, tt.region, tt.accessKey, tt.secretKey)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Error("expected error but got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("newOSSClient() unexpected error: %v", err)
|
||||
return
|
||||
}
|
||||
if client == nil {
|
||||
t.Error("expected non-nil client")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOSSClientRejectsUnsafeEndpoint(t *testing.T) {
|
||||
if _, err := newOSSClient("http://127.0.0.1:9000", "cn-hangzhou", "ak", "sk"); err == nil {
|
||||
t.Fatal("expected loopback OSS endpoint to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Should fail with an invalid/unreachable endpoint
|
||||
err := CheckOssConnectivity(ctx,
|
||||
"https://invalid-oss-endpoint-that-does-not-exist.local",
|
||||
"cn-hangzhou",
|
||||
"invalid-access-key",
|
||||
"invalid-secret-key",
|
||||
"nonexistent-bucket",
|
||||
)
|
||||
|
||||
if err == nil {
|
||||
t.Error("CheckOssConnectivity with invalid endpoint should return an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOssEnsureBucket_NonExistent(t *testing.T) {
|
||||
client, err := newOSSClient(
|
||||
"https://oss-cn-hangzhou.aliyuncs.com",
|
||||
"cn-hangzhou",
|
||||
"test-invalid-key",
|
||||
"test-invalid-secret",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newOSSClient() error: %v", err)
|
||||
}
|
||||
|
||||
// Bucket that definitely doesn't exist - should return error
|
||||
err = ossEnsureBucket(client, "this-bucket-definitely-does-not-exist-12345")
|
||||
if err == nil {
|
||||
t.Error("ossEnsureBucket with non-existent bucket should return an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOssEnsureBucket_CreateFails(t *testing.T) {
|
||||
client, err := newOSSClient(
|
||||
"https://oss-cn-hangzhou.aliyuncs.com",
|
||||
"cn-hangzhou",
|
||||
"test-invalid-key",
|
||||
"test-invalid-secret",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newOSSClient() error: %v", err)
|
||||
}
|
||||
|
||||
// Use a bucket that does not exist so IsBucketExist returns false and the
|
||||
// create path is exercised; with invalid credentials PutBucket then fails.
|
||||
// A common name like "test-bucket" already exists globally on OSS, which
|
||||
// would short-circuit at IsBucketExist and make this assertion flaky.
|
||||
err = ossEnsureBucket(client, "weknora-nonexistent-bucket-create-fails-12345")
|
||||
if err == nil {
|
||||
t.Error("ossEnsureBucket with invalid credentials should return an error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// resourceCatalogFileService keeps provider drivers physical-path-only while
|
||||
// exposing stable resource:// references to the application layer.
|
||||
type resourceCatalogFileService struct {
|
||||
inner interfaces.FileService
|
||||
catalog interfaces.ResourceCatalog
|
||||
externalURL string
|
||||
}
|
||||
|
||||
// NewResourceCatalogFileService decorates a physical FileService with stable
|
||||
// resource registration and resolution.
|
||||
func NewResourceCatalogFileService(
|
||||
inner interfaces.FileService,
|
||||
catalog interfaces.ResourceCatalog,
|
||||
) interfaces.FileService {
|
||||
if inner == nil || catalog == nil {
|
||||
return inner
|
||||
}
|
||||
return &resourceCatalogFileService{
|
||||
inner: inner,
|
||||
catalog: catalog,
|
||||
externalURL: strings.TrimRight(strings.TrimSpace(os.Getenv("APP_EXTERNAL_URL")), "/"),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) CheckConnectivity(ctx context.Context) error {
|
||||
return s.inner.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
func resourceKind(name string) (string, string) {
|
||||
mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name)))
|
||||
kind := "file"
|
||||
switch {
|
||||
case strings.HasPrefix(mimeType, "image/"):
|
||||
kind = "image"
|
||||
case strings.HasPrefix(mimeType, "audio/"):
|
||||
kind = "audio"
|
||||
case strings.HasPrefix(mimeType, "video/"):
|
||||
kind = "video"
|
||||
}
|
||||
return kind, mimeType
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) register(
|
||||
ctx context.Context,
|
||||
physical string,
|
||||
tenantID uint64,
|
||||
name string,
|
||||
size int64,
|
||||
temporary bool,
|
||||
contentHash string,
|
||||
) (string, error) {
|
||||
kind, mimeType := resourceKind(name)
|
||||
ref, err := s.catalog.Register(ctx, tenantID, physical, interfaces.ResourceRegistration{
|
||||
Kind: kind,
|
||||
MimeType: mimeType,
|
||||
OriginalName: filepath.Base(name),
|
||||
Size: size,
|
||||
ContentHash: contentHash,
|
||||
Temporary: temporary,
|
||||
})
|
||||
if err != nil {
|
||||
_ = s.inner.DeleteFile(ctx, physical)
|
||||
return "", fmt.Errorf("register stored resource: %w", err)
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) SaveFile(
|
||||
ctx context.Context,
|
||||
file *multipart.FileHeader,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
) (string, error) {
|
||||
physical, err := s.inner.SaveFile(ctx, file, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ref, err := s.register(ctx, physical, tenantID, file.Filename, file.Size, false, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if knowledgeID != "" {
|
||||
if err := s.catalog.Bind(ctx, ref, types.ResourceOwnerKnowledge, knowledgeID, types.ResourceRelationSourceFile); err != nil {
|
||||
_ = s.DeleteFile(ctx, ref)
|
||||
return "", fmt.Errorf("bind stored resource: %w", err)
|
||||
}
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) SaveBytes(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
tenantID uint64,
|
||||
fileName string,
|
||||
temp bool,
|
||||
) (string, error) {
|
||||
physical, err := s.inner.SaveBytes(ctx, data, tenantID, fileName, temp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return s.register(ctx, physical, tenantID, fileName, int64(len(data)), temp, hex.EncodeToString(sum[:]))
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) resolve(ctx context.Context, value string) (string, bool, error) {
|
||||
physical, resource, err := s.catalog.ResolvePath(ctx, value)
|
||||
return physical, resource != nil, err
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
physical, _, err := s.resolve(ctx, filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.inner.GetFile(ctx, physical)
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
physical, isResource, err := s.resolve(ctx, filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if isResource && s.externalURL != "" {
|
||||
token, grantErr := s.catalog.CreateAccessGrant(ctx, filePath, 2*time.Hour)
|
||||
if grantErr != nil {
|
||||
return "", grantErr
|
||||
}
|
||||
return s.externalURL + "/r/" + token, nil
|
||||
}
|
||||
return s.inner.GetFileURL(ctx, physical)
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
physical, isResource, err := s.resolve(ctx, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.inner.DeleteFile(ctx, physical); err != nil {
|
||||
return err
|
||||
}
|
||||
if isResource {
|
||||
return s.catalog.MarkDeleted(ctx, filePath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *resourceCatalogFileService) CopyFile(
|
||||
ctx context.Context,
|
||||
filePath string,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
) (string, error) {
|
||||
physical, _, err := s.resolve(ctx, filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
copied, err := s.inner.CopyFile(ctx, physical, tenantID, knowledgeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ref, err := s.register(ctx, copied, tenantID, filepath.Base(physical), 0, false, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if knowledgeID != "" {
|
||||
if err := s.catalog.Bind(ctx, ref, types.ResourceOwnerKnowledge, knowledgeID, types.ResourceRelationSourceFile); err != nil {
|
||||
_ = s.DeleteFile(ctx, ref)
|
||||
return "", fmt.Errorf("bind copied resource: %w", err)
|
||||
}
|
||||
}
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
var _ interfaces.FileService = (*resourceCatalogFileService)(nil)
|
||||
@@ -0,0 +1,111 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type catalogStub struct {
|
||||
resource *types.StoredResource
|
||||
ref string
|
||||
}
|
||||
|
||||
func (c *catalogStub) Register(
|
||||
_ context.Context,
|
||||
tenantID uint64,
|
||||
physicalPath string,
|
||||
meta interfaces.ResourceRegistration,
|
||||
) (string, error) {
|
||||
c.resource = &types.StoredResource{
|
||||
ID: "resource-1",
|
||||
Handle: "AbCdEfGhIjKlMnOpQrStUv",
|
||||
TenantID: tenantID,
|
||||
PhysicalPath: physicalPath,
|
||||
OriginalName: meta.OriginalName,
|
||||
}
|
||||
c.ref = types.BuildResourcePath(c.resource.Handle)
|
||||
return c.ref, nil
|
||||
}
|
||||
|
||||
func (c *catalogStub) Resolve(_ context.Context, _ string) (*types.StoredResource, error) {
|
||||
return c.resource, nil
|
||||
}
|
||||
|
||||
func (c *catalogStub) ResolvePath(_ context.Context, value string) (string, *types.StoredResource, error) {
|
||||
if value == c.ref {
|
||||
return c.resource.PhysicalPath, c.resource, nil
|
||||
}
|
||||
return value, nil, nil
|
||||
}
|
||||
func (c *catalogStub) Bind(context.Context, string, string, string, string) error { return nil }
|
||||
func (c *catalogStub) MarkDeleted(context.Context, string) error { return nil }
|
||||
|
||||
func (c *catalogStub) Release(context.Context, string, string, string) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (c *catalogStub) CreateAccessGrant(context.Context, string, time.Duration) (string, error) {
|
||||
return "GrantTokenAbCdEfGhIjKl", nil
|
||||
}
|
||||
|
||||
func (c *catalogStub) ResolveAccessGrant(context.Context, string) (*types.StoredResource, error) {
|
||||
return c.resource, nil
|
||||
}
|
||||
|
||||
type physicalFileStub struct {
|
||||
savedPath string
|
||||
readPath string
|
||||
}
|
||||
|
||||
func (s *physicalFileStub) CheckConnectivity(context.Context) error { return nil }
|
||||
func (s *physicalFileStub) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (s *physicalFileStub) SaveBytes(context.Context, []byte, uint64, string, bool) (string, error) {
|
||||
return s.savedPath, nil
|
||||
}
|
||||
|
||||
func (s *physicalFileStub) GetFile(_ context.Context, path string) (io.ReadCloser, error) {
|
||||
s.readPath = path
|
||||
return io.NopCloser(strings.NewReader("body")), nil
|
||||
}
|
||||
func (s *physicalFileStub) GetFileURL(context.Context, string) (string, error) { return "", nil }
|
||||
func (s *physicalFileStub) DeleteFile(context.Context, string) error { return nil }
|
||||
func (s *physicalFileStub) CopyFile(context.Context, string, uint64, string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func TestResourceCatalogFileServiceReturnsReferenceAndResolvesReads(t *testing.T) {
|
||||
inner := &physicalFileStub{savedPath: "local://7/exports/a.png"}
|
||||
catalog := &catalogStub{}
|
||||
svc := NewResourceCatalogFileService(inner, catalog)
|
||||
|
||||
ref, err := svc.SaveBytes(context.Background(), []byte("image"), 7, "a.png", false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "resource://AbCdEfGhIjKlMnOpQrStUv", ref)
|
||||
reader, err := svc.GetFile(context.Background(), ref)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, reader.Close())
|
||||
require.Equal(t, inner.savedPath, inner.readPath)
|
||||
}
|
||||
|
||||
func TestResourceCatalogFileServiceReturnsShortExternalGrantURL(t *testing.T) {
|
||||
t.Setenv("APP_EXTERNAL_URL", "https://weknora.example.com/")
|
||||
inner := &physicalFileStub{savedPath: "local://7/exports/a.png"}
|
||||
catalog := &catalogStub{}
|
||||
svc := NewResourceCatalogFileService(inner, catalog)
|
||||
|
||||
ref, err := svc.SaveBytes(context.Background(), []byte("image"), 7, "a.png", false)
|
||||
require.NoError(t, err)
|
||||
externalURL, err := svc.GetFileURL(context.Background(), ref)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "https://weknora.example.com/r/GrantTokenAbCdEfGhIjKl", externalURL)
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// s3FileService AWS S3 file service implementation
|
||||
type s3FileService struct {
|
||||
client *s3.Client
|
||||
bucketName string
|
||||
pathPrefix string
|
||||
}
|
||||
|
||||
// newS3Client creates a bare s3FileService with just the SDK client initialised.
|
||||
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe S3 endpoint: %w", err)
|
||||
}
|
||||
var cfg aws.Config
|
||||
var err error
|
||||
|
||||
// With no explicit AK/SK, keep the AWS default credential chain intact. This
|
||||
// supports IAM roles for EC2/ECS/EKS (IRSA), web identity, shared config, and
|
||||
// environment credentials without persisting long-lived keys in WeKnora.
|
||||
loadOptions := []func(*config.LoadOptions) error{config.WithRegion(region)}
|
||||
if accessKey != "" || secretKey != "" {
|
||||
if accessKey == "" || secretKey == "" {
|
||||
return nil, fmt.Errorf("S3 access key and secret key must be provided together")
|
||||
}
|
||||
loadOptions = append(loadOptions, config.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
))
|
||||
}
|
||||
cfg, err = config.LoadDefaultConfig(context.Background(), loadOptions...)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
// Create S3 client with custom endpoint if provided.
|
||||
// For S3-compatible services (non-AWS), use path-style addressing
|
||||
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
|
||||
httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())
|
||||
var client *s3.Client
|
||||
if endpoint != "" {
|
||||
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
|
||||
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = usePathStyle
|
||||
if !strings.Contains(endpoint, "amazonaws.com") {
|
||||
// S3-compatible services commonly reject the SDK's default
|
||||
// trailing checksum negotiation. Only relax this for explicit
|
||||
// non-AWS endpoints; standard AWS S3 keeps its default behavior.
|
||||
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
||||
}
|
||||
o.HTTPClient = httpClient
|
||||
})
|
||||
} else {
|
||||
// Standard AWS S3
|
||||
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.HTTPClient = httpClient
|
||||
})
|
||||
}
|
||||
|
||||
// Normalize pathPrefix: ensure it ends with '/' if not empty
|
||||
if pathPrefix != "" && !strings.HasSuffix(pathPrefix, "/") {
|
||||
pathPrefix += "/"
|
||||
}
|
||||
|
||||
return &s3FileService{
|
||||
client: client,
|
||||
bucketName: bucketName,
|
||||
pathPrefix: pathPrefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewS3FileService creates an AWS S3 file service.
|
||||
// It verifies that the bucket exists and creates it if missing.
|
||||
func NewS3FileService(endpoint,
|
||||
accessKey, secretKey, bucketName, region, pathPrefix string,
|
||||
) (interfaces.FileService, error) {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if bucket exists
|
||||
exists, err := svc.bucketExists(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
if err = svc.createBucket(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// NewS3FileServiceWithOptions is the instance-aware S3 constructor. Existing
|
||||
// callers keep the historical endpoint-based path-style inference.
|
||||
func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (interfaces.FileService, error) {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix, forcePathStyle)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exists, err := svc.bucketExists(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err = svc.createBucket(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// bucketExists checks if the bucket exists
|
||||
func (s *s3FileService) bucketExists(ctx context.Context) (bool, error) {
|
||||
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
// Check if the error is a NotFound error
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// createBucket creates a new bucket
|
||||
func (s *s3FileService) createBucket(ctx context.Context) error {
|
||||
_, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies S3 is reachable and, if a bucket is configured,
|
||||
// that the bucket exists. This is a read-only probe — it never creates a bucket.
|
||||
func (s *s3FileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if s.bucketName != "" {
|
||||
exists, err := s.bucketExists(checkCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List buckets to verify connectivity
|
||||
_, err := s.client.ListBuckets(checkCtx, &s3.ListBucketsInput{})
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckS3Connectivity tests S3 connectivity using the provided credentials.
|
||||
// It creates a temporary service instance internally and delegates to CheckConnectivity.
|
||||
func CheckS3Connectivity(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string) error {
|
||||
return CheckS3ConnectivityWithOptions(ctx, endpoint, accessKey, secretKey, bucketName, region, false)
|
||||
}
|
||||
|
||||
func CheckS3ConnectivityWithOptions(ctx context.Context, endpoint, accessKey, secretKey, bucketName, region string, forcePathStyle bool) error {
|
||||
svc, err := newS3Client(endpoint, accessKey, secretKey, bucketName, region, "", forcePathStyle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return svc.CheckConnectivity(ctx)
|
||||
}
|
||||
|
||||
// parseS3FilePath extracts the object name from a provider scheme: s3://{bucket}/{objectKey}
|
||||
func (s *s3FileService) parseS3FilePath(filePath string) (string, error) {
|
||||
// Provider scheme format: s3://{bucket}/{objectKey}
|
||||
const prefix = "s3://"
|
||||
if !strings.HasPrefix(filePath, prefix) {
|
||||
return "", fmt.Errorf("invalid S3 file path: %s", filePath)
|
||||
}
|
||||
rest := strings.TrimPrefix(filePath, prefix)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", fmt.Errorf("invalid S3 file path: %s", filePath)
|
||||
}
|
||||
if parts[0] != s.bucketName {
|
||||
return "", fmt.Errorf("bucket mismatch in path: got %s, want %s", parts[0], s.bucketName)
|
||||
}
|
||||
if err := utils.SafeObjectKey(parts[1]); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
return parts[1], nil
|
||||
}
|
||||
|
||||
// SaveFile saves a file to S3
|
||||
func (s *s3FileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
// Generate object name
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
// Open file
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Determine content type
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
// Upload file to S3
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
Body: src,
|
||||
ContentLength: aws.Int64(file.Size),
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to S3: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("s3://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFile gets a file from S3
|
||||
func (s *s3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
objectName, err := s.parseS3FilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from S3: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// DeleteFile deletes a file
|
||||
func (s *s3FileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
objectName, err := s.parseS3FilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing S3 object to a new knowledge-owned object using a
|
||||
// server-side CopyObject (no data leaves S3). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath is not an s3:// path.
|
||||
func (s *s3FileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
srcKey, err := s.parseS3FilePath(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("s3 copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
destKey := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
|
||||
// (url.PathEscape would turn them into %2F and break the bucket/key split).
|
||||
// srcKey is already validated by parseS3FilePath -> SafeObjectKey.
|
||||
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
CopySource: aws.String(s.bucketName + "/" + srcKey),
|
||||
Key: aws.String(destKey),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in S3: %w", err)
|
||||
}
|
||||
|
||||
newPath := fmt.Sprintf("s3://%s/%s", s.bucketName, destKey)
|
||||
logger.Infof(ctx, "Copied S3 object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to S3 and returns the file path
|
||||
// temp parameter is ignored for S3 (no auto-expiration support in this implementation)
|
||||
func (s *s3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
|
||||
|
||||
// Upload bytes to S3
|
||||
reader := bytes.NewReader(data)
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
Body: reader,
|
||||
ContentLength: aws.Int64(int64(len(data))),
|
||||
ContentType: aws.String(utils.GetContentTypeByExt(ext)),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to S3: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("s3://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *s3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
objectName, err := s.parseS3FilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Create presign client
|
||||
presignClient := s3.NewPresignClient(s.client)
|
||||
|
||||
// Generate presigned URL
|
||||
presignedReq, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
}, s3.WithPresignExpires(24*time.Hour))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedReq.URL, nil
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
)
|
||||
|
||||
func TestNewS3Client_Credentials(t *testing.T) {
|
||||
t.Run("static credentials remain supported", func(t *testing.T) {
|
||||
svc, err := newS3Client("", "static-ak", "static-sk", "bucket", "us-east-1", "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("newS3Client() error = %v", err)
|
||||
}
|
||||
got, err := svc.client.Options().Credentials.Retrieve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Retrieve() error = %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "static-ak" || got.SecretAccessKey != "static-sk" {
|
||||
t.Fatalf("unexpected credentials: access key %q", got.AccessKeyID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty keys use the AWS default credential chain", func(t *testing.T) {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "role-ak")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "role-sk")
|
||||
svc, err := newS3Client("", "", "", "bucket", "us-east-1", "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("newS3Client() error = %v", err)
|
||||
}
|
||||
got, err := svc.client.Options().Credentials.Retrieve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Retrieve() error = %v", err)
|
||||
}
|
||||
if got.AccessKeyID != "role-ak" || got.SecretAccessKey != "role-sk" {
|
||||
t.Fatalf("default credential chain returned access key %q", got.AccessKeyID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("partial static credentials are rejected", func(t *testing.T) {
|
||||
_, err := newS3Client("", "only-ak", "", "bucket", "us-east-1", "", false)
|
||||
if err == nil {
|
||||
t.Fatal("newS3Client() expected an error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
wantPathStyle bool
|
||||
}{
|
||||
{
|
||||
name: "S3-compatible service uses path-style",
|
||||
endpoint: "https://storage.internal:9000",
|
||||
wantPathStyle: true,
|
||||
},
|
||||
{
|
||||
name: "MinIO endpoint uses path-style",
|
||||
endpoint: "http://minio.local:9000",
|
||||
wantPathStyle: true,
|
||||
},
|
||||
{
|
||||
name: "AWS S3 regional endpoint uses virtual-hosted",
|
||||
endpoint: "https://s3.us-east-1.amazonaws.com",
|
||||
wantPathStyle: false,
|
||||
},
|
||||
{
|
||||
name: "AWS China endpoint uses virtual-hosted",
|
||||
endpoint: "https://s3.cn-north-1.amazonaws.com.cn",
|
||||
wantPathStyle: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Verify the path-style detection logic matches our implementation
|
||||
usePathStyle := !strings.Contains(tt.endpoint, "amazonaws.com")
|
||||
if usePathStyle != tt.wantPathStyle {
|
||||
t.Errorf("endpoint %q: usePathStyle = %v, want %v", tt.endpoint, usePathStyle, tt.wantPathStyle)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3Client_EmptyEndpoint(t *testing.T) {
|
||||
// Empty endpoint should not trigger path-style (standard AWS S3)
|
||||
endpoint := ""
|
||||
if endpoint != "" {
|
||||
t.Fatal("expected empty endpoint to skip custom configuration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewS3Client_RequestChecksumCalculation(t *testing.T) {
|
||||
const (
|
||||
compatibleEndpoint = "https://s3-compatible.example.com"
|
||||
awsEndpoint = "https://s3.us-east-1.amazonaws.com"
|
||||
)
|
||||
|
||||
// Keep the test independent of the developer's AWS profile while checking
|
||||
// that a standard AWS endpoint still honors the SDK-resolved default.
|
||||
t.Setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_supported")
|
||||
t.Setenv("SSRF_WHITELIST", "s3-compatible.example.com,s3.us-east-1.amazonaws.com")
|
||||
utils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(utils.ResetSSRFWhitelistForTest)
|
||||
|
||||
standard, err := newS3Client("", "ak", "sk", "bucket", "us-east-1", "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("newS3Client() with empty endpoint error = %v", err)
|
||||
}
|
||||
wantStandard := standard.client.Options().RequestChecksumCalculation
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
want aws.RequestChecksumCalculation
|
||||
}{
|
||||
{
|
||||
name: "empty endpoint preserves standard AWS setting",
|
||||
endpoint: "",
|
||||
want: wantStandard,
|
||||
},
|
||||
{
|
||||
name: "AWS endpoint preserves standard AWS setting",
|
||||
endpoint: awsEndpoint,
|
||||
want: wantStandard,
|
||||
},
|
||||
{
|
||||
name: "S3-compatible endpoint calculates checksum only when required",
|
||||
endpoint: compatibleEndpoint,
|
||||
want: aws.RequestChecksumCalculationWhenRequired,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
svc, err := newS3Client(tt.endpoint, "ak", "sk", "bucket", "us-east-1", "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("newS3Client() error = %v", err)
|
||||
}
|
||||
got := svc.client.Options().RequestChecksumCalculation
|
||||
if got != tt.want {
|
||||
t.Errorf("RequestChecksumCalculation = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseS3FilePath(t *testing.T) {
|
||||
svc := &s3FileService{bucketName: "test-bucket"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid s3 path",
|
||||
input: "s3://test-bucket/123/exports/abc.png",
|
||||
want: "123/exports/abc.png",
|
||||
},
|
||||
{
|
||||
name: "wrong bucket",
|
||||
input: "s3://other-bucket/key",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "not s3 scheme",
|
||||
input: "minio://test-bucket/key",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing object key",
|
||||
input: "s3://test-bucket/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := svc.parseS3FilePath(tt.input)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("parseS3FilePath(%q) expected error, got %q", tt.input, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseS3FilePath(%q) unexpected error: %v", tt.input, err)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("parseS3FilePath(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
|
||||
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
|
||||
)
|
||||
|
||||
// tosFileService implements the FileService interface for Volcengine TOS.
|
||||
type tosFileService struct {
|
||||
client *tos.ClientV2
|
||||
pathPrefix string
|
||||
bucketName string
|
||||
tempBucketName string
|
||||
}
|
||||
|
||||
const tosScheme = "tos://"
|
||||
|
||||
// NewTosFileService creates a TOS file service.
|
||||
func NewTosFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
|
||||
return NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewTosFileServiceWithTempBucket creates a TOS file service with optional temp bucket.
|
||||
func NewTosFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return nil, fmt.Errorf("unsafe TOS endpoint: %w", err)
|
||||
}
|
||||
httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
client, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(region),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize TOS client: %w", err)
|
||||
}
|
||||
|
||||
if err := ensureTOSBucket(client, bucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region
|
||||
}
|
||||
// Temporary bucket may belong to another region, so probe with a short-lived client.
|
||||
tempClient, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(tempRegion),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(httpConfig),
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize TOS temp client: %w", err)
|
||||
}
|
||||
if err := ensureTOSBucket(tempClient, tempBucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return &tosFileService{
|
||||
client: client,
|
||||
pathPrefix: strings.Trim(pathPrefix, "/"),
|
||||
bucketName: bucketName,
|
||||
tempBucketName: tempBucketName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies TOS is reachable by performing a HeadBucket request.
|
||||
func (s *tosFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := s.client.HeadBucket(checkCtx, &tos.HeadBucketInput{
|
||||
Bucket: s.bucketName,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckTosConnectivity tests TOS connectivity using the provided credentials.
|
||||
func CheckTosConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
|
||||
return fmt.Errorf("unsafe TOS endpoint: %w", err)
|
||||
}
|
||||
client, err := tos.NewClientV2(
|
||||
endpoint,
|
||||
tos.WithRegion(region),
|
||||
tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)),
|
||||
tos.WithHTTPTransport(&utils.SSRFValidatingRoundTripper{
|
||||
Base: utils.NewSSRFSafeTransport(utils.DefaultSSRFSafeHTTPClientConfig()),
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize TOS client: %w", err)
|
||||
}
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = client.HeadBucket(checkCtx, &tos.HeadBucketInput{
|
||||
Bucket: bucketName,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func ensureTOSBucket(client *tos.ClientV2, bucketName string) error {
|
||||
_, err := client.HeadBucket(context.Background(), &tos.HeadBucketInput{
|
||||
Bucket: bucketName,
|
||||
})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var serverErr *tos.TosServerError
|
||||
if errors.As(err, &serverErr) && serverErr.StatusCode == 404 {
|
||||
_, createErr := client.CreateBucketV2(context.Background(), &tos.CreateBucketV2Input{
|
||||
Bucket: bucketName,
|
||||
})
|
||||
if createErr == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.As(createErr, &serverErr) && serverErr.StatusCode == 409 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to create TOS bucket: %w", createErr)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to check TOS bucket: %w", err)
|
||||
}
|
||||
|
||||
func joinTOSObjectKey(parts ...string) string {
|
||||
filtered := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.Trim(part, "/")
|
||||
if part != "" {
|
||||
filtered = append(filtered, part)
|
||||
}
|
||||
}
|
||||
return strings.Join(filtered, "/")
|
||||
}
|
||||
|
||||
func parseTOSFilePath(filePath string) (bucketName string, objectKey string, err error) {
|
||||
if !strings.HasPrefix(filePath, tosScheme) {
|
||||
return "", "", fmt.Errorf("invalid TOS file path: %s", filePath)
|
||||
}
|
||||
|
||||
rest := strings.TrimPrefix(filePath, tosScheme)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("invalid TOS file path: %s", filePath)
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := joinTOSObjectKey(
|
||||
s.pathPrefix,
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
knowledgeID,
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = utils.GetContentTypeByExt(ext)
|
||||
}
|
||||
_, err = s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{
|
||||
PutObjectBasicInput: tos.PutObjectBasicInput{
|
||||
Bucket: s.bucketName,
|
||||
Key: objectName,
|
||||
ContentType: contentType,
|
||||
},
|
||||
Content: src,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to TOS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("tos://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
targetBucket := s.bucketName
|
||||
objectName := joinTOSObjectKey(
|
||||
s.pathPrefix,
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
"exports",
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
|
||||
if temp && s.tempBucketName != "" {
|
||||
targetBucket = s.tempBucketName
|
||||
objectName = joinTOSObjectKey(
|
||||
"exports",
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
}
|
||||
|
||||
_, err = s.client.PutObjectV2(ctx, &tos.PutObjectV2Input{
|
||||
PutObjectBasicInput: tos.PutObjectBasicInput{
|
||||
Bucket: targetBucket,
|
||||
Key: objectName,
|
||||
ContentType: utils.GetContentTypeByExt(ext),
|
||||
},
|
||||
Content: reader,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to TOS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("tos://%s/%s", targetBucket, objectName), nil
|
||||
}
|
||||
|
||||
// CopyFile copies an existing TOS object to a new knowledge-owned object using a
|
||||
// server-side CopyObject (no data leaves TOS). The destination uses the same
|
||||
// layout as SaveFile. Returns ErrCrossBackendCopy when srcPath is not a tos:// path.
|
||||
func (s *tosFileService) CopyFile(ctx context.Context,
|
||||
srcPath string, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
srcBucket, srcKey, err := parseTOSFilePath(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("tos copy rejected source %q: %w", srcPath, ErrCrossBackendCopy)
|
||||
}
|
||||
if err := utils.SafeObjectKey(srcKey); err != nil {
|
||||
return "", fmt.Errorf("invalid source path: %w", err)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(srcPath)
|
||||
destKey := joinTOSObjectKey(
|
||||
s.pathPrefix,
|
||||
fmt.Sprintf("%d", tenantID),
|
||||
knowledgeID,
|
||||
uuid.New().String()+ext,
|
||||
)
|
||||
|
||||
_, err = s.client.CopyObject(ctx, &tos.CopyObjectInput{
|
||||
Bucket: s.bucketName,
|
||||
Key: destKey,
|
||||
SrcBucket: srcBucket,
|
||||
SrcKey: srcKey,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file in TOS: %w", err)
|
||||
}
|
||||
|
||||
newPath := fmt.Sprintf("tos://%s/%s", s.bucketName, destKey)
|
||||
logger.Infof(ctx, "Copied TOS object %s to %s", srcPath, newPath)
|
||||
return newPath, nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
bucketName, objectName, err := parseTOSFilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
output, err := s.client.GetObjectV2(ctx, &tos.GetObjectV2Input{
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from TOS: %w", err)
|
||||
}
|
||||
return output.Content, nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
bucketName, objectName, err := parseTOSFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObjectV2(ctx, &tos.DeleteObjectV2Input{
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file from TOS: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *tosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
bucketName, objectName, err := parseTOSFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
output, err := s.client.PreSignedURL(&tos.PreSignedURLInput{
|
||||
HTTPMethod: enum.HttpMethodGet,
|
||||
Bucket: bucketName,
|
||||
Key: objectName,
|
||||
Expires: int64((24 * time.Hour).Seconds()),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate TOS presigned URL: %w", err)
|
||||
}
|
||||
return output.SignedUrl, nil
|
||||
}
|
||||
Reference in New Issue
Block a user