diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index 35ebe96cc5..d45c1f7c18 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -245,6 +245,7 @@ func mFile2Object(file MFile) *model.ObjThumbURL { Ctime: ctime, IsFolder: false, Size: file.Size, + HashInfo: utils.NewHashInfo(utils.MD5, file.MD5), }, Thumbnail: model.Thumbnail{ Thumbnail: file.ThumbnailURL, diff --git a/drivers/quark_open/types.go b/drivers/quark_open/types.go index 652b0a5a7a..c54cb6188f 100644 --- a/drivers/quark_open/types.go +++ b/drivers/quark_open/types.go @@ -2,6 +2,7 @@ package quark_open import ( "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "time" ) @@ -57,6 +58,7 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.UnixMilli(f.UpdatedAt), IsFolder: f.FileType == "0", Ctime: time.UnixMilli(f.CreatedAt), + HashInfo: utils.NewHashInfo(utils.SHA1, f.ContentHash), }, Thumbnail: model.Thumbnail{Thumbnail: f.ThumbnailURL}, } diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index b902aeef9b..32748ea48e 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -187,6 +187,14 @@ func InitialSettings() []model.SettingItem { {Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`}, + {Key: conf.SeedSiteURL, Value: "", Type: conf.TypeString, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Public base URL embedded in generated transfer seed sources when configured`}, + {Key: conf.SeedDefaultMatrix, Value: `{"md5":{"whole":true,"pieces":false},"sha1":{"whole":true,"pieces":false},"sha256":{"whole":true,"pieces":false}}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default right-click hash matrix for transfer seed generation`}, + {Key: conf.SeedFormatPolicies, Value: `{"oss":"off","torrent":"off","cas":"off"}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.SeedDefaultFormat, Value: "oss", Type: conf.TypeSelect, Options: "oss,torrent,cas", Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.SeedSingleDirectPreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.SeedCASDirectAccess, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC, Help: `When opening a single-file CAS seed, immediately rapid-upload it into the same folder and preview the restored file`}, + {Key: conf.SeedAutoGeneratePolicy, Value: "off", Type: conf.TypeSelect, Options: "off,on", Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Global upload sidecar policy; storage-specific inheritance can be layered without changing the safe default`}, + {Key: conf.SeedDefaultTrackers, Value: "", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default tracker list offered when generating torrent seeds (one tracker per line)`}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index 47e0b59ebf..a9ea494f19 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -49,4 +49,5 @@ func InitTaskManager() { op.RegisterSettingChangingCallback(func() { fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers))) }) + fs.SeedGenerateTaskManager = tache.NewManager[*fs.SeedGenerateTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //seed generation will not support persist } diff --git a/internal/conf/const.go b/internal/conf/const.go index cc8a51d416..d558da9b3b 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -60,6 +60,16 @@ const ( HandleHookRateLimit = "handle_hook_rate_limit" IgnoreSystemFiles = "ignore_system_files" + // transfer seeds + SeedSiteURL = "seed_site_url" + SeedDefaultMatrix = "seed_default_matrix" + SeedFormatPolicies = "seed_format_policies" + SeedDefaultFormat = "seed_default_format" + SeedSingleDirectPreview = "seed_single_direct_preview" + SeedCASDirectAccess = "seed_cas_direct_access" + SeedAutoGeneratePolicy = "seed_auto_generate_policy" + SeedDefaultTrackers = "seed_default_trackers" + // index SearchIndex = "search_index" AutoUpdateIndex = "auto_update_index" diff --git a/internal/fs/seed_generate.go b/internal/fs/seed_generate.go new file mode 100644 index 0000000000..ed5a628b1e --- /dev/null +++ b/internal/fs/seed_generate.go @@ -0,0 +1,599 @@ +package fs + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + stdpath "path" + "slices" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/OpenListTeam/tache" + "github.com/pkg/errors" +) + +// MaxSeedGenerateSyncSize bounds synchronous seed generation (1GB). Larger +// requests are turned into an asynchronous task. +const MaxSeedGenerateSyncSize = 1 * 1024 * 1024 * 1024 + +// SeedGenerateNeedsAsync reports whether the given files must be generated +// asynchronously because they exceed the synchronous size limit. It resolves +// each path and sums the file sizes, returning the first error encountered. +func SeedGenerateNeedsAsync(ctx context.Context, user *model.User, paths []string) (bool, error) { + var total int64 + for _, requestedPath := range paths { + fullPath, err := user.JoinPath(requestedPath) + if err != nil { + return false, err + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return false, err + } + obj, err := op.Get(ctx, storage, actualPath) + if err != nil { + return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath) + } + if obj.IsDir() { + return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath) + } + total += obj.GetSize() + if total > MaxSeedGenerateSyncSize { + return true, nil + } + } + return false, nil +} + +// SeedHashSelection controls whole-file and piece hash inclusion. +type SeedHashSelection struct { + Whole bool `json:"whole"` + Pieces bool `json:"pieces"` +} + +// SeedHashMatrix controls the optional hash metadata stored in a seed. +type SeedHashMatrix struct { + MD5 SeedHashSelection `json:"md5"` + SHA1 SeedHashSelection `json:"sha1"` + SHA256 SeedHashSelection `json:"sha256"` +} + +// SeedGenerateParams carries a fully-resolved seed generation request, free of +// any HTTP transport concerns so it can run synchronously or as a task. +type SeedGenerateParams struct { + Paths []string + Formats []string + Name string + Comment string + FileComments map[string]string + HashMatrix SeedHashMatrix + PieceSize int64 + Trackers []string + Channels []torrent.SeedChannel + OutputPath string + IncludeShare bool + IncludeDirectSource bool + ShareFiles []string + DirectFiles []string +} + +// SeedArtifact is one generated seed container. +type SeedArtifact struct { + Format string `json:"format"` + Name string `json:"name"` + FileName string `json:"file_name"` + SeedData string `json:"seed_data"` + Size int `json:"size"` + Path string `json:"path,omitempty"` +} + +// DeriveSeedName derives a sensible default seed name from the source paths: +// single selection uses the file name, multi selection uses the common base +// name (ignoring extensions) when all files share one, otherwise the folder name. +func DeriveSeedName(paths []string) string { + if len(paths) == 0 { + return "OpenList Seed" + } + if len(paths) == 1 { + return seedBaseName(paths[0]) + } + // Common base name ignoring extensions (e.g. a.docx + a.exe -> "a"). + commonBase := seedBaseName(paths[0]) + for _, p := range paths[1:] { + if base := seedBaseName(p); base != commonBase { + commonBase = "" + break + } + } + if commonBase != "" { + return commonBase + } + // Fall back to the common parent directory name. + dir := commonParentDir(paths) + if base := stdpath.Base(dir); base != "" && base != "/" && base != "." { + return base + } + return "OpenList Seed" +} + +// seedBaseName returns the file name without its extension. +func seedBaseName(p string) string { + base := stdpath.Base(p) + return strings.TrimSuffix(base, stdpath.Ext(base)) +} + +// commonParentDir returns the longest common parent directory of the given paths. +func commonParentDir(paths []string) string { + if len(paths) == 0 { + return "/" + } + parts := strings.Split(strings.Trim(stdpath.Dir(paths[0]), "/"), "/") + for _, p := range paths[1:] { + cur := strings.Split(strings.Trim(stdpath.Dir(p), "/"), "/") + n := 0 + for n < len(parts) && n < len(cur) && parts[n] == cur[n] { + n++ + } + parts = parts[:n] + } + if len(parts) == 0 { + return "/" + } + return "/" + strings.Join(parts, "/") +} + +// NormalizeSeedFormats validates and deduplicates a list of seed format names. +func NormalizeSeedFormats(rawFormats []string) ([]string, error) { + formats := append([]string(nil), rawFormats...) + if len(formats) == 1 && strings.TrimSpace(formats[0]) == "" { + formats[0] = setting.GetStr(conf.SeedDefaultFormat, "oss") + } + if len(formats) > 3 { + return nil, fmt.Errorf("at most three seed formats may be generated") + } + result := make([]string, 0, len(formats)) + seen := make(map[string]struct{}, len(formats)) + for _, rawFormat := range formats { + format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), ".")) + if format == "bt" { + format = "torrent" + } + if format != "oss" && format != "torrent" && format != "cas" { + return nil, fmt.Errorf("unsupported seed format %q", rawFormat) + } + if _, exists := seen[format]; exists { + continue + } + seen[format] = struct{}{} + result = append(result, format) + } + return result, nil +} + +func seedFormats(params SeedGenerateParams) ([]string, error) { + formats, err := NormalizeSeedFormats(params.Formats) + if err != nil { + return nil, err + } + if len(formats) == 0 { + formats = []string{setting.GetStr(conf.SeedDefaultFormat, "oss")} + } + return formats, nil +} + +func seedMatrixEmpty(matrix SeedHashMatrix) bool { + return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces +} + +func loadSeedDefaultMatrix() SeedHashMatrix { + var matrix SeedHashMatrix + if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" { + _ = json.Unmarshal([]byte(raw), &matrix) + } + return matrix +} + +func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix { + if seedMatrixEmpty(matrix) { + matrix = loadSeedDefaultMatrix() + } + if seedMatrixEmpty(matrix) { + matrix = SeedHashMatrix{ + MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true}, + SHA256: SeedHashSelection{Whole: true, Pieces: true}, + } + } + for _, format := range formats { + switch format { + case "torrent": + matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true} + case "cas": + matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true} + } + } + return matrix +} + +func canReuseListedHashes(hashInfo utils.HashInfo, matrix SeedHashMatrix) bool { + if matrix.MD5.Pieces || matrix.SHA1.Pieces || matrix.SHA256.Pieces { + return false + } + if matrix.MD5.Whole && hashInfo.GetHash(utils.MD5) == "" { + return false + } + if matrix.SHA1.Whole && hashInfo.GetHash(utils.SHA1) == "" { + return false + } + if matrix.SHA256.Whole && hashInfo.GetHash(utils.SHA256) == "" { + return false + } + return true +} + +func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) { + if !matrix.MD5.Whole { + file.Hashes.MD5 = "" + } + if !matrix.SHA1.Whole { + file.Hashes.SHA1 = "" + } + if !matrix.SHA256.Whole { + file.Hashes.SHA256 = "" + } + if file.Hashes.Pieces == nil { + return + } + if !matrix.MD5.Pieces { + file.Hashes.Pieces.MD5 = nil + } + if !matrix.SHA1.Pieces { + file.Hashes.Pieces.SHA1 = nil + } + if !matrix.SHA256.Pieces { + file.Hashes.Pieces.SHA256 = nil + } + if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 { + file.Hashes.Pieces = nil + } +} + +// EncodeGeneratedSeed serializes a seed in the requested container format. +func EncodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byte) ([]byte, error) { + if format != "torrent" { + return torrent.EncodeSeed(seed, format) + } + t := &torrent.Torrent{ + Info: torrent.TorrentInfo{Name: seed.Name, PieceLength: seed.PieceSize, Pieces: standardPieces}, + Comment: seed.Comment, + CreatedBy: seed.CreatedBy, + CreationDate: time.Now().Unix(), + OpenList: seed, + } + if len(seed.Trackers) > 0 { + t.Announce = seed.Trackers[0] + for _, tracker := range seed.Trackers { + t.AnnounceList = append(t.AnnounceList, []string{tracker}) + } + } + if len(seed.Files) == 1 { + file := seed.Files[0] + t.Info.Name = stdpath.Base(file.Path) + t.Info.Length = file.Size + t.Info.MD5Sum = file.Hashes.MD5 + if file.CASSliceMD5 != "" { + t.SetCASInfo(&torrent.CASInfo{ + FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5), + SliceSize: torrent.DefaultPieceSize, Cloud: "189", + }) + } else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 { + t.SetCASInfo(torrent.BuildCASInfoFromMD5s(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize)) + } + } else { + for _, file := range seed.Files { + t.Info.Files = append(t.Info.Files, torrent.TorrentFile{Length: file.Size, Path: strings.Split(file.Path, "/"), MD5Sum: file.Hashes.MD5}) + } + } + return t.Encode() +} + +// GenerateSeedArtifacts reads each file once while computing the requested +// hashes, then emits one or more seed containers. It has no HTTP dependency and +// can be driven synchronously or from a background task. +func GenerateSeedArtifacts(ctx context.Context, user *model.User, params SeedGenerateParams) ([]SeedArtifact, *torrent.Seed, error) { + if len(params.Paths) == 0 || len(params.Paths) > torrent.DefaultMaxSeedFiles { + return nil, nil, fmt.Errorf("invalid seed file count") + } + formats, err := seedFormats(params) + if err != nil { + return nil, nil, err + } + matrix := normalizedSeedMatrix(params.HashMatrix, formats) + pieceSize := params.PieceSize + if pieceSize <= 0 { + pieceSize = torrent.DefaultPieceSize + } + if slices.Contains(formats, "cas") { + pieceSize = torrent.DefaultPieceSize + } + seedName := strings.TrimSpace(params.Name) + if seedName == "" { + seedName = DeriveSeedName(params.Paths) + } + seed := torrent.NewSeed(seedName, "OpenList", pieceSize) + seed.Comment = params.Comment + seed.Trackers = params.Trackers + seed.Channels = params.Channels + shareSet := make(map[string]bool, len(params.ShareFiles)) + for _, p := range params.ShareFiles { + if strings.TrimSpace(p) != "" { + shareSet[p] = true + } + } + directSet := make(map[string]bool, len(params.DirectFiles)) + for _, p := range params.DirectFiles { + if strings.TrimSpace(p) != "" { + directSet[p] = true + } + } + useGlobalShare := len(shareSet) == 0 && params.IncludeShare + useGlobalDirect := len(directSet) == 0 && params.IncludeDirectSource + hasShare := useGlobalShare || len(shareSet) > 0 + hasDirect := useGlobalDirect || len(directSet) > 0 + if hasShare && !user.CanShare() { + return nil, nil, errs.PermissionDenied + } + if hasDirect && setting.GetBool(conf.SignAll) && !hasShare { + return nil, nil, fmt.Errorf("direct sources require an automatic share when global signing is enabled") + } + if (hasShare || hasDirect) && strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)) == "" { + return nil, nil, fmt.Errorf("seed_site_url must be configured before embedding download sources") + } + globalHasher := torrent.NewHashWriter(pieceSize, pieceSize) + fullPaths := make([]string, 0, len(params.Paths)) + var total int64 + for _, requestedPath := range params.Paths { + fullPath, err := user.JoinPath(requestedPath) + if err != nil { + return nil, nil, err + } + meta, err := op.GetNearestMeta(fullPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, nil, err + } + if !common.CanRead(user, meta, fullPath) { + return nil, nil, errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return nil, nil, err + } + obj, err := op.Get(ctx, storage, actualPath) + if err != nil || obj.IsDir() { + return nil, nil, fmt.Errorf("seed path must be a readable file: %s", requestedPath) + } + total += obj.GetSize() + modified := "" + if !obj.ModTime().IsZero() { + modified = obj.ModTime().UTC().Format(time.RFC3339) + } + seedPath := stdpath.Base(requestedPath) + if len(params.Paths) > 1 { + seedPath = strings.TrimPrefix(stdpath.Clean(requestedPath), "/") + } + + hashInfo := obj.GetHash() + if canReuseListedHashes(hashInfo, matrix) { + seedFile := torrent.SeedFile{ + Path: seedPath, + Size: obj.GetSize(), + Modified: modified, + Hashes: torrent.SeedHashes{ + MD5: strings.ToLower(hashInfo.GetHash(utils.MD5)), + SHA1: strings.ToLower(hashInfo.GetHash(utils.SHA1)), + SHA256: strings.ToLower(hashInfo.GetHash(utils.SHA256)), + }, + } + applySeedMatrix(&seedFile, matrix) + if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" { + seedFile.Comment = comment + } else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" { + seedFile.Comment = comment + } + if useGlobalDirect || directSet[requestedPath] { + baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}} + } + seed.Files = append(seed.Files, seedFile) + fullPaths = append(fullPaths, fullPath) + continue + } + + link, _, err := op.Link(ctx, storage, actualPath, model.LinkArgs{}) + if err != nil { + return nil, nil, fmt.Errorf("storage cannot stream %s: %v", requestedPath, err) + } + rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link) + if err != nil { + return nil, nil, fmt.Errorf("storage cannot stream %s", requestedPath) + } + rc, err := rangeReader.RangeRead(ctx, http_range.Range{Length: obj.GetSize()}) + if err != nil { + return nil, nil, err + } + fileHasher := torrent.NewHashWriter(pieceSize, pieceSize) + n, copyErr := io.Copy(io.MultiWriter(globalHasher, fileHasher), rc) + _ = rc.Close() + if copyErr != nil { + return nil, nil, fmt.Errorf("read %s: %w", requestedPath, copyErr) + } + if n != obj.GetSize() { + return nil, nil, fmt.Errorf("read %s: got %d of %d bytes", requestedPath, n, obj.GetSize()) + } + fileHasher.Finish() + seedFile := fileHasher.BuildSeedFile(seedPath, modified) + applySeedMatrix(&seedFile, matrix) + if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" { + seedFile.Comment = comment + } else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" { + seedFile.Comment = comment + } + if useGlobalDirect || directSet[requestedPath] { + baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}} + } + seed.Files = append(seed.Files, seedFile) + fullPaths = append(fullPaths, fullPath) + } + globalHasher.Finish() + createdShares := make([]string, 0, len(seed.Files)) + keepCreatedShares := false + defer func() { + if !keepCreatedShares { + for _, createdID := range createdShares { + _ = op.DeleteSharing(createdID) + } + } + }() + if hasShare { + for index, fullPath := range fullPaths { + if !useGlobalShare && !shareSet[params.Paths[index]] { + continue + } + sharing := &model.Sharing{ + SharingDB: &model.SharingDB{Remark: "Transfer seed source"}, + Files: []string{fullPath}, Creator: user, + } + shareID, createErr := op.CreateSharing(sharing) + if createErr != nil { + return nil, nil, fmt.Errorf("create seed share: %w", createErr) + } + createdShares = append(createdShares, shareID) + seed.Files[index].Sources = []torrent.SeedSource{{ + Type: "openlist-share", URL: strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + "/sd/" + shareID, + ShareID: shareID, + }} + } + } + if err := torrent.ValidateSeed(seed, torrent.DefaultParseLimits()); err != nil { + return nil, nil, err + } + outputPath := strings.TrimSpace(params.OutputPath) + artifacts := make([]SeedArtifact, 0, len(formats)) + seenFormats := make(map[string]struct{}, len(formats)) + + // writeArtifact persists the encoded container into the destination folder + // when an output path is configured, returning the fully-populated artifact. + writeArtifact := func(format, fileName string, data []byte) (SeedArtifact, error) { + artifact := SeedArtifact{ + Format: format, + Name: fileName, + FileName: fileName, + SeedData: base64.StdEncoding.EncodeToString(data), + Size: len(data), + } + if outputPath == "" { + return artifact, nil + } + dstDir, err := user.JoinPath(outputPath) + if err != nil { + return artifact, err + } + meta, metaErr := op.GetNearestMeta(dstDir) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + return artifact, metaErr + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) { + return artifact, errs.PermissionDenied + } + fileStream := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()}, + Reader: bytes.NewReader(data), Mimetype: "application/octet-stream", + } + if err = PutDirectly(ctx, dstDir, fileStream); err != nil { + return artifact, err + } + artifact.Path = stdpath.Join(outputPath, fileName) + return artifact, nil + } + + for _, requestedFormat := range formats { + format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(requestedFormat), ".")) + if format == "bt" { + format = "torrent" + } + if _, exists := seenFormats[format]; exists { + continue + } + seenFormats[format] = struct{}{} + + data, err := EncodeGeneratedSeed(seed, format, globalHasher.GetPieceHashes()) + if err != nil { + return nil, nil, fmt.Errorf("generate %s seed: %w", format, err) + } + fileName := stdpath.Base(seed.Name) + "." + format + artifact, err := writeArtifact(format, fileName, data) + if err != nil { + return nil, nil, err + } + artifacts = append(artifacts, artifact) + } + keepCreatedShares = true + return artifacts, seed, nil +} + +// SeedGenerateTask generates seed containers asynchronously, writing them into +// params.OutputPath so they appear in the target folder once done. +type SeedGenerateTask struct { + task.TaskExtension + params SeedGenerateParams +} + +func (t *SeedGenerateTask) GetName() string { + if len(t.params.Paths) == 1 { + return fmt.Sprintf("generate seed for %s", stdpath.Base(t.params.Paths[0])) + } + return fmt.Sprintf("generate seed for %d files", len(t.params.Paths)) +} + +func (t *SeedGenerateTask) GetStatus() string { + return "generating seed" +} + +func (t *SeedGenerateTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + _, _, err := GenerateSeedArtifacts(t.Ctx(), t.Creator, t.params) + return err +} + +var SeedGenerateTaskManager *tache.Manager[*SeedGenerateTask] + +// AddSeedGenerateTask schedules an asynchronous seed generation task. +func AddSeedGenerateTask(ctx context.Context, user *model.User, params SeedGenerateParams) (task.TaskExtensionInfo, error) { + t := &SeedGenerateTask{ + TaskExtension: task.TaskExtension{ + Creator: user, + ApiUrl: common.GetApiUrl(ctx), + }, + params: params, + } + SeedGenerateTaskManager.Add(t) + return t, nil +} diff --git a/internal/model/storage.go b/internal/model/storage.go index a6b4745a60..6bda023f1f 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -19,6 +19,7 @@ type Storage struct { Disabled bool `json:"disabled"` // if disabled DisableIndex bool `json:"disable_index"` EnableSign bool `json:"enable_sign"` + SeedPolicy string `json:"seed_policy" gorm:"default:inherit"` Sort Proxy } diff --git a/internal/op/driver.go b/internal/op/driver.go index 5b79b0aed6..38fe2f56b5 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -173,6 +173,14 @@ func getMainItems(config driver.Config) []driver.Item { Default: "false", Required: true, }) + items = append(items, driver.Item{ + Name: "seed_policy", + Type: conf.TypeSelect, + Options: "inherit,on,off", + Default: "inherit", + Required: true, + Help: "Override automatic transfer-seed generation for this storage", + }) return items } func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item { diff --git a/pkg/torrent/bencode.go b/pkg/torrent/bencode.go index 2d4fd782c6..8189bc8ed5 100644 --- a/pkg/torrent/bencode.go +++ b/pkg/torrent/bencode.go @@ -145,15 +145,24 @@ func bencodeEncodeOrderedDict(w io.Writer, d OrderedDict) error { // BencodeDecode 从字节数组解码 bencode 数据 func BencodeDecode(data []byte) (interface{}, error) { + if int64(len(data)) > DefaultMaxSeedSize { + return nil, fmt.Errorf("bencode: input exceeds %d bytes", DefaultMaxSeedSize) + } reader := bytes.NewReader(data) - val, err := bencodeDecodeValue(reader) + val, err := bencodeDecodeValue(reader, 0) if err != nil { return nil, err } + if reader.Len() != 0 { + return nil, fmt.Errorf("bencode: trailing data") + } return val, nil } -func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { +func bencodeDecodeValue(r *bytes.Reader, depth int) (interface{}, error) { + if depth > DefaultParseLimits().MaxDepth { + return nil, fmt.Errorf("bencode: nesting depth exceeds limit") + } b, err := r.ReadByte() if err != nil { return nil, err @@ -163,9 +172,9 @@ func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) { case b == 'i': return bencodeDecodeInt(r) case b == 'l': - return bencodeDecodeList(r) + return bencodeDecodeList(r, depth+1) case b == 'd': - return bencodeDecodeDict(r) + return bencodeDecodeDict(r, depth+1) case b >= '0' && b <= '9': r.UnreadByte() return bencodeDecodeString(r) @@ -218,9 +227,12 @@ func bencodeDecodeString(r *bytes.Reader) ([]byte, error) { return data, nil } -func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { +func bencodeDecodeList(r *bytes.Reader, depth int) ([]interface{}, error) { var list []interface{} for { + if len(list) >= DefaultMaxSeedFiles*4 { + return nil, fmt.Errorf("bencode: list item limit exceeded") + } b, err := r.ReadByte() if err != nil { return nil, err @@ -228,8 +240,8 @@ func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { if b == 'e' { return list, nil } - r.UnreadByte() - val, err := bencodeDecodeValue(r) + _ = r.UnreadByte() + val, err := bencodeDecodeValue(r, depth) if err != nil { return nil, err } @@ -237,9 +249,12 @@ func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) { } } -func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { +func bencodeDecodeDict(r *bytes.Reader, depth int) (map[string]interface{}, error) { dict := make(map[string]interface{}) for { + if len(dict) >= DefaultMaxSeedFiles*4 { + return nil, fmt.Errorf("bencode: dictionary item limit exceeded") + } b, err := r.ReadByte() if err != nil { return nil, err @@ -247,12 +262,12 @@ func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) { if b == 'e' { return dict, nil } - r.UnreadByte() + _ = r.UnreadByte() keyBytes, err := bencodeDecodeString(r) if err != nil { return nil, err } - val, err := bencodeDecodeValue(r) + val, err := bencodeDecodeValue(r, depth) if err != nil { return nil, err } diff --git a/pkg/torrent/generate.go b/pkg/torrent/generate.go index 566cad86f7..5be5135bb7 100644 --- a/pkg/torrent/generate.go +++ b/pkg/torrent/generate.go @@ -1,8 +1,10 @@ package torrent import ( + "fmt" "io" "os" + "path" "strings" ) @@ -121,3 +123,27 @@ func GenerateFromFileWithCAS(filePath string) ([]byte, error) { return GenerateFromReaderWithCAS(f, info.Name(), info.Size(), DefaultPieceSize) } + +// GenerateSeedFromReader computes the complete OSS hash matrix in one stream pass. +func GenerateSeedFromReader(reader io.Reader, filePath string, expectedSize, pieceSize int64, createdBy string) (*Seed, error) { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + if err := validateRelativeSeedPath(filePath); err != nil { + return nil, err + } + hw := NewHashWriter(pieceSize, pieceSize) + if _, err := CopyAndHash(nil, reader, hw); err != nil { + return nil, err + } + hw.Finish() + if expectedSize >= 0 && hw.GetTotalWritten() != expectedSize { + return nil, fmt.Errorf("stream size mismatch: read %d bytes, expected %d", hw.GetTotalWritten(), expectedSize) + } + seed := NewSeed(path.Base(filePath), createdBy, pieceSize) + seed.Files = []SeedFile{hw.BuildSeedFile(filePath, "")} + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, err + } + return seed, nil +} diff --git a/pkg/torrent/hash_writer.go b/pkg/torrent/hash_writer.go index a62f42c648..efd50dd161 100644 --- a/pkg/torrent/hash_writer.go +++ b/pkg/torrent/hash_writer.go @@ -3,6 +3,7 @@ package torrent import ( "crypto/md5" "crypto/sha1" + "crypto/sha256" "encoding/hex" "fmt" "hash" @@ -15,10 +16,15 @@ import ( type HashWriter struct { // 整文件 MD5 fileMD5 hash.Hash + // fileSHA1 and fileSHA256 complete the portable full-file matrix. + fileSHA1 hash.Hash + fileSHA256 hash.Hash // 当前分片 MD5 sliceMD5 hash.Hash - // 当前 piece 的 SHA-1 - pieceSHA1 hash.Hash + // Per-piece hashers are updated in the same pass as whole-file hashes. + pieceMD5 hash.Hash + pieceSHA1 hash.Hash + pieceSHA256 hash.Hash // 分片大小（默认 10MB） sliceSize int64 @@ -34,8 +40,12 @@ type HashWriter struct { // 每个分片的 MD5（大写十六进制） sliceMD5Hexs []string - // 所有 piece 的 SHA-1 哈希拼接 + // all standard BitTorrent SHA-1 piece hashes concatenated pieceHashes []byte + // portable per-file piece matrix + pieceMD5Hexs []string + pieceSHA1Hexs []string + pieceSHA256Hexs []string } // NewHashWriter 创建一个新的 HashWriter @@ -49,11 +59,15 @@ func NewHashWriter(sliceSize, pieceSize int64) *HashWriter { pieceSize = DefaultPieceSize } return &HashWriter{ - fileMD5: md5.New(), - sliceMD5: md5.New(), - pieceSHA1: sha1.New(), - sliceSize: sliceSize, - pieceSize: pieceSize, + fileMD5: md5.New(), + fileSHA1: sha1.New(), + fileSHA256: sha256.New(), + sliceMD5: md5.New(), + pieceMD5: md5.New(), + pieceSHA1: sha1.New(), + pieceSHA256: sha256.New(), + sliceSize: sliceSize, + pieceSize: pieceSize, } } @@ -76,12 +90,14 @@ func (hw *HashWriter) Write(p []byte) (n int, err error) { chunk := p[offset : offset+int(canWrite)] - // 写入整文件 MD5 - hw.fileMD5.Write(chunk) - // 写入当前分片 MD5 - hw.sliceMD5.Write(chunk) - // 写入当前 piece SHA-1 - hw.pieceSHA1.Write(chunk) + // Write all whole-file and boundary-specific hashes in one pass. + _, _ = hw.fileMD5.Write(chunk) + _, _ = hw.fileSHA1.Write(chunk) + _, _ = hw.fileSHA256.Write(chunk) + _, _ = hw.sliceMD5.Write(chunk) + _, _ = hw.pieceMD5.Write(chunk) + _, _ = hw.pieceSHA1.Write(chunk) + _, _ = hw.pieceSHA256.Write(chunk) hw.sliceWritten += canWrite hw.pieceWritten += canWrite @@ -112,8 +128,16 @@ func (hw *HashWriter) finishSlice() { // finishPiece 完成当前 piece 的 SHA-1 计算 func (hw *HashWriter) finishPiece() { - hw.pieceHashes = append(hw.pieceHashes, hw.pieceSHA1.Sum(nil)...) + md5Sum := hw.pieceMD5.Sum(nil) + sha1Sum := hw.pieceSHA1.Sum(nil) + sha256Sum := hw.pieceSHA256.Sum(nil) + hw.pieceMD5Hexs = append(hw.pieceMD5Hexs, hex.EncodeToString(md5Sum)) + hw.pieceSHA1Hexs = append(hw.pieceSHA1Hexs, hex.EncodeToString(sha1Sum)) + hw.pieceSHA256Hexs = append(hw.pieceSHA256Hexs, hex.EncodeToString(sha256Sum)) + hw.pieceHashes = append(hw.pieceHashes, sha1Sum...) + hw.pieceMD5.Reset() hw.pieceSHA1.Reset() + hw.pieceSHA256.Reset() hw.pieceWritten = 0 } @@ -134,6 +158,50 @@ func (hw *HashWriter) GetFileMD5() string { return strings.ToUpper(hex.EncodeToString(hw.fileMD5.Sum(nil))) } +// GetFileSHA1 returns the lowercase whole-file SHA-1 digest. +func (hw *HashWriter) GetFileSHA1() string { + return hex.EncodeToString(hw.fileSHA1.Sum(nil)) +} + +// GetFileSHA256 returns the lowercase whole-file SHA-256 digest. +func (hw *HashWriter) GetFileSHA256() string { + return hex.EncodeToString(hw.fileSHA256.Sum(nil)) +} + +// GetPieceMD5s returns independent per-file MD5 piece hashes. +func (hw *HashWriter) GetPieceMD5s() []string { + return append([]string(nil), hw.pieceMD5Hexs...) +} + +// GetPieceSHA1s returns independent per-file SHA-1 piece hashes. +func (hw *HashWriter) GetPieceSHA1s() []string { + return append([]string(nil), hw.pieceSHA1Hexs...) +} + +// GetPieceSHA256s returns independent per-file SHA-256 piece hashes. +func (hw *HashWriter) GetPieceSHA256s() []string { + return append([]string(nil), hw.pieceSHA256Hexs...) +} + +// BuildSeedFile exports all hashes accumulated during this single stream pass. +func (hw *HashWriter) BuildSeedFile(filePath string, modified string) SeedFile { + return SeedFile{ + Path: filePath, + Size: hw.totalWritten, + Modified: modified, + Hashes: SeedHashes{ + MD5: strings.ToLower(hw.GetFileMD5()), + SHA1: hw.GetFileSHA1(), + SHA256: hw.GetFileSHA256(), + Pieces: &SeedPieceHashes{ + MD5: hw.GetPieceMD5s(), + SHA1: hw.GetPieceSHA1s(), + SHA256: hw.GetPieceSHA256s(), + }, + }, + } +} + // GetSliceMD5s 获取所有分片的 MD5 列表 func (hw *HashWriter) GetSliceMD5s() []string { return hw.sliceMD5Hexs diff --git a/pkg/torrent/seed_test.go b/pkg/torrent/seed_test.go new file mode 100644 index 0000000000..f9aa7be54d --- /dev/null +++ b/pkg/torrent/seed_test.go @@ -0,0 +1,117 @@ +package torrent + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" + "time" +) + +func testSeed() *Seed { + return &Seed{ + Format: OSSFormat, + Version: OSSVersion, + Name: "example.bin", + CreatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339), + CreatedBy: "OpenList", + PieceSize: DefaultPieceSize, + Files: []SeedFile{{ + Path: "example.bin", + Size: DefaultPieceSize + 1, + Hashes: SeedHashes{ + MD5: strings.Repeat("1", 32), + SHA1: strings.Repeat("2", 40), + SHA256: strings.Repeat("3", 64), + Pieces: &SeedPieceHashes{ + MD5: []string{strings.Repeat("4", 32), strings.Repeat("5", 32)}, + SHA1: []string{strings.Repeat("6", 40), strings.Repeat("7", 40)}, + SHA256: []string{strings.Repeat("8", 64), strings.Repeat("9", 64)}, + }, + }, + }}, + } +} + +func TestOSSRoundTrip(t *testing.T) { + encoded, err := EncodeOSS(testSeed()) + if err != nil { + t.Fatalf("EncodeOSS() error = %v", err) + } + decoded, err := DecodeOSS(encoded, DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeOSS() error = %v", err) + } + if decoded.Name != "example.bin" || len(decoded.Files) != 1 || decoded.Files[0].Hashes.SHA256 == "" { + t.Fatalf("DecodeOSS() returned incomplete seed: %#v", decoded) + } +} + +func TestCASWireFormatIsLegacyCompatible(t *testing.T) { + encoded, err := EncodeCAS(testSeed()) + if err != nil { + t.Fatalf("EncodeCAS() error = %v", err) + } + decodedJSON, err := base64.StdEncoding.DecodeString(string(encoded)) + if err != nil { + t.Fatalf("base64.DecodeString() error = %v", err) + } + var payload map[string]any + if err = json.Unmarshal(decodedJSON, &payload); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + // The five legacy fields must always be present so the reference client can + // parse the payload. slice_md5s / slice_size are optional extensions. + for _, key := range []string{"name", "size", "md5", "sliceMd5", "create_time"} { + if _, ok := payload[key]; !ok { + t.Fatalf("CAS payload missing %q: %#v", key, payload) + } + } + decoded, err := DecodeCAS(encoded, DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeCAS() error = %v", err) + } + if decoded.Files[0].Hashes.MD5 != strings.Repeat("1", 32) { + t.Fatalf("DecodeCAS() MD5 = %q", decoded.Files[0].Hashes.MD5) + } + // Per-piece MD5 list must round-trip through the slice_md5s extension. + if decoded.Files[0].Hashes.Pieces == nil || len(decoded.Files[0].Hashes.Pieces.MD5) != 2 { + t.Fatalf("DecodeCAS() piece MD5 list = %#v", decoded.Files[0].Hashes.Pieces) + } +} + +func TestTorrentRoundTripPreservesOpenListExtension(t *testing.T) { + encoded, err := EncodeSeed(testSeed(), "torrent") + if err != nil { + t.Fatalf("EncodeSeed(torrent) error = %v", err) + } + decoded, err := DecodeSeed(encoded, "torrent", DefaultParseLimits()) + if err != nil { + t.Fatalf("DecodeSeed(torrent) error = %v", err) + } + if decoded.Files[0].Hashes.SHA256 != strings.Repeat("3", 64) { + t.Fatalf("torrent extension lost SHA-256: %#v", decoded.Files[0].Hashes) + } +} + +func TestValidateSeedRejectsTraversalAndInvalidHash(t *testing.T) { + seed := testSeed() + seed.Files[0].Path = "../secret" + if err := ValidateSeed(seed, DefaultParseLimits()); err == nil { + t.Fatal("ValidateSeed() accepted path traversal") + } + seed = testSeed() + seed.Files[0].Hashes.MD5 = "not-a-hash" + if err := ValidateSeed(seed, DefaultParseLimits()); err == nil { + t.Fatal("ValidateSeed() accepted an invalid hash") + } +} + +func TestDecodeSeedHonorsSizeLimit(t *testing.T) { + data := []byte(`{"format":"openlist-sharing-seed"}`) + limits := DefaultParseLimits() + limits.MaxBytes = int64(len(data) - 1) + if _, err := DecodeSeed(data, "oss", limits); err == nil { + t.Fatal("DecodeSeed() accepted input above MaxBytes") + } +} diff --git a/pkg/torrent/torrent.go b/pkg/torrent/torrent.go index 8744e6362e..4e97200a5a 100644 --- a/pkg/torrent/torrent.go +++ b/pkg/torrent/torrent.go @@ -1,12 +1,19 @@ package torrent import ( + "bytes" "crypto/md5" "crypto/sha1" + "encoding/base64" "encoding/hex" + "encoding/json" "fmt" + "io" + "net/url" + "path" "strings" "time" + "unicode/utf8" ) const ( @@ -26,8 +33,113 @@ const ( CASFileMD5Key = "file_md5" // CASCloudKey 云盘类型 key CASCloudKey = "cloud" + + // OpenListExtensionKey is the optional root-level extension key. + OpenListExtensionKey = "x-openlist" + // OSSFormat identifies OpenList sharing seed JSON documents. + OSSFormat = "openlist-sharing-seed" + // OSSVersion is the currently supported sharing seed schema version. + OSSVersion = 1 + // DefaultMaxSeedSize bounds untrusted seed documents. + DefaultMaxSeedSize int64 = 10 * 1024 * 1024 + // DefaultMaxSeedFiles bounds file-list fan-out in untrusted seeds. + DefaultMaxSeedFiles = 100000 ) +// Seed describes the portable OpenList sharing seed contract. +type Seed struct { + Format string `json:"format"` + Version int `json:"version"` + Name string `json:"name"` + Comment string `json:"comment,omitempty"` + CreatedAt string `json:"created_at"` + CreatedBy string `json:"created_by"` + PieceSize int64 `json:"piece_size"` + Trackers []string `json:"trackers,omitempty"` + Channels []SeedChannel `json:"channels,omitempty"` + Files []SeedFile `json:"files"` +} + +// SeedChannel contains only public storage discovery metadata. +type SeedChannel struct { + Driver string `json:"driver"` + MountPath string `json:"mount_path,omitempty"` +} + +// SeedFile describes one relative file in a sharing seed. +type SeedFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + Modified string `json:"modified,omitempty"` + Comment string `json:"comment,omitempty"` + Hashes SeedHashes `json:"hashes"` + Sources []SeedSource `json:"sources,omitempty"` + CASSliceMD5 string `json:"cas_slice_md5,omitempty"` + CASCreateTime string `json:"cas_create_time,omitempty"` + MissingChannels []string `json:"missing_channels,omitempty"` +} + +// SeedHashes contains whole-file and optional per-piece hashes. +type SeedHashes struct { + MD5 string `json:"md5,omitempty"` + SHA1 string `json:"sha1,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Pieces *SeedPieceHashes `json:"pieces,omitempty"` +} + +// SeedPieceHashes contains independent per-file piece hashes. +type SeedPieceHashes struct { + MD5 []string `json:"md5,omitempty"` + SHA1 []string `json:"sha1,omitempty"` + SHA256 []string `json:"sha256,omitempty"` +} + +// SeedSource is an optional retrievable public or signed source URL. +type SeedSource struct { + Type string `json:"type"` + URL string `json:"url"` + ExpiresAt string `json:"expires_at,omitempty"` + ShareID string `json:"share_id,omitempty"` +} + +// CASFileEntry describes one file inside a multi-file .cas payload. +type CASFileEntry struct { + Name string `json:"name"` + Size int64 `json:"size"` + MD5 string `json:"md5"` + SliceMD5 string `json:"sliceMd5"` + CreateTime string `json:"create_time"` + SliceMD5s []string `json:"slice_md5s,omitempty"` + SliceSize int64 `json:"slice_size,omitempty"` +} + +// CASPayload matches the reference .cas JSON payload. The five legacy fields +// describe a single file (byte-for-byte compatible with the reference project); +// the optional "files" array extends it to multi-file seeds, and the optional +// slice_md5s/slice_size preserve the per-piece MD5 list. +type CASPayload struct { + Name string `json:"name"` + Size int64 `json:"size"` + MD5 string `json:"md5"` + SliceMD5 string `json:"sliceMd5"` + CreateTime string `json:"create_time"` + SliceMD5s []string `json:"slice_md5s,omitempty"` + SliceSize int64 `json:"slice_size,omitempty"` + Files []CASFileEntry `json:"files,omitempty"` +} + +// ParseLimits controls resource use while parsing untrusted seeds. +type ParseLimits struct { + MaxBytes int64 + MaxFiles int + MaxDepth int +} + +// DefaultParseLimits returns conservative public API limits. +func DefaultParseLimits() ParseLimits { + return ParseLimits{MaxBytes: DefaultMaxSeedSize, MaxFiles: DefaultMaxSeedFiles, MaxDepth: 64} +} + // CASInfo 天翼云 CAS 秒传所需信息 type CASInfo struct { // FileMD5 整文件 MD5（大写十六进制） @@ -86,6 +198,8 @@ type Torrent struct { CreatedBy string // CAS 天翼云 CAS 扩展信息（存储在 info 字典外部，不影响 info_hash） CAS *CASInfo + // OpenList stores the portable extension outside info so info_hash stays standard. + OpenList *Seed } // NewTorrent 创建一个新的 torrent 结构 @@ -200,6 +314,12 @@ func (t *Torrent) Encode() ([]byte, error) { } rootDict[CASExtensionKey] = casDict } + if t.OpenList != nil { + if err := ValidateSeed(t.OpenList, DefaultParseLimits()); err != nil { + return nil, fmt.Errorf("validate x-openlist: %w", err) + } + rootDict[OpenListExtensionKey] = seedToBencode(t.OpenList) + } return BencodeEncode(rootDict) } @@ -376,6 +496,20 @@ func Decode(data []byte) (*Torrent, error) { } } + if extension, ok := rootDict[OpenListExtensionKey]; ok { + seed, err := seedFromBencode(extension) + if err != nil { + return nil, fmt.Errorf("decode x-openlist: %w", err) + } + if err = ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, fmt.Errorf("validate x-openlist: %w", err) + } + t.OpenList = seed + } + if err := ValidateTorrent(t, DefaultParseLimits()); err != nil { + return nil, err + } + return t, nil } @@ -416,11 +550,14 @@ func (t *Torrent) HasCASInfo() bool { // BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息 func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *CASInfo { + fileMD5 = strings.ToUpper(fileMD5) + sliceMD5s = upperStrings(sliceMD5s) sliceMD5 := fileMD5 - if len(sliceMD5s) > 1 { - // 所有分片 MD5 用 \n 拼接后再取 MD5 - joined := strings.Join(sliceMD5s, "\n") - sliceMD5 = strings.ToUpper(GetMD5Str(joined)) + if len(sliceMD5s) == 1 { + sliceMD5 = sliceMD5s[0] + } else if len(sliceMD5s) > 1 { + // All piece MD5 values are joined with newlines before hashing. + sliceMD5 = strings.ToUpper(GetMD5Str(strings.Join(sliceMD5s, "\n"))) } return &CASInfo{ FileMD5: fileMD5, @@ -437,3 +574,865 @@ func GetMD5Str(data string) string { h.Write([]byte(data)) return strings.ToUpper(hex.EncodeToString(h.Sum(nil))) } + +// ValidateTorrent checks standard BitTorrent invariants and portable paths. +func ValidateTorrent(t *Torrent, limits ParseLimits) error { + if t == nil { + return fmt.Errorf("missing torrent") + } + if limits.MaxFiles <= 0 { + limits.MaxFiles = DefaultMaxSeedFiles + } + if err := validateSeedName(t.Info.Name); err != nil { + return fmt.Errorf("torrent: %w", err) + } + if t.Info.PieceLength <= 0 || t.Info.PieceLength > 1<<30 { + return fmt.Errorf("torrent: invalid piece length") + } + if len(t.Info.Pieces)%sha1.Size != 0 { + return fmt.Errorf("torrent: pieces length must be a multiple of %d", sha1.Size) + } + if len(t.Info.Files) > limits.MaxFiles { + return fmt.Errorf("torrent: too many files") + } + var total int64 + if len(t.Info.Files) == 0 { + if t.Info.Length < 0 { + return fmt.Errorf("torrent: invalid file size") + } + total = t.Info.Length + } else { + seen := make(map[string]struct{}, len(t.Info.Files)) + for _, file := range t.Info.Files { + filePath := strings.Join(file.Path, "/") + if err := validateRelativeSeedPath(filePath); err != nil { + return fmt.Errorf("torrent: %w", err) + } + if _, ok := seen[filePath]; ok { + return fmt.Errorf("torrent: duplicate file path %q", filePath) + } + seen[filePath] = struct{}{} + if file.Length < 0 || total > int64(^uint64(0)>>1)-file.Length { + return fmt.Errorf("torrent: invalid file size") + } + total += file.Length + } + } + expectedPieces := int64(0) + if total > 0 { + expectedPieces = (total + t.Info.PieceLength - 1) / t.Info.PieceLength + } + if int64(len(t.Info.Pieces)/sha1.Size) != expectedPieces { + return fmt.Errorf("torrent: piece count mismatch") + } + return nil +} + +// NewSeed creates a versioned OSS document with stable defaults. +func NewSeed(name, createdBy string, pieceSize int64) *Seed { + if pieceSize <= 0 { + pieceSize = DefaultPieceSize + } + return &Seed{ + Format: OSSFormat, + Version: OSSVersion, + Name: name, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + CreatedBy: createdBy, + PieceSize: pieceSize, + } +} + +// ValidateSeed rejects unsafe paths, malformed hashes and unreasonable resource use. +func ValidateSeed(seed *Seed, limits ParseLimits) error { + if seed == nil { + return fmt.Errorf("missing seed") + } + if limits.MaxBytes <= 0 { + limits.MaxBytes = DefaultMaxSeedSize + } + if limits.MaxFiles <= 0 { + limits.MaxFiles = DefaultMaxSeedFiles + } + if seed.Format != OSSFormat { + return fmt.Errorf("unsupported seed format %q", seed.Format) + } + if seed.Version != OSSVersion { + return fmt.Errorf("unsupported seed version %d", seed.Version) + } + if err := validateSeedName(seed.Name); err != nil { + return err + } + if seed.CreatedAt == "" || seed.CreatedBy == "" { + return fmt.Errorf("created_at and created_by are required") + } + if seed.PieceSize < 16*1024 || seed.PieceSize > 64*1024*1024 { + return fmt.Errorf("piece_size must be between 16384 and 67108864") + } + if len(seed.Files) == 0 || len(seed.Files) > limits.MaxFiles { + return fmt.Errorf("file count must be between 1 and %d", limits.MaxFiles) + } + seen := make(map[string]struct{}, len(seed.Files)) + var total int64 + for i := range seed.Files { + file := &seed.Files[i] + if err := validateRelativeSeedPath(file.Path); err != nil { + return fmt.Errorf("file %d: %w", i, err) + } + if _, ok := seen[file.Path]; ok { + return fmt.Errorf("duplicate file path %q", file.Path) + } + seen[file.Path] = struct{}{} + if file.Size < 0 || total > int64(^uint64(0)>>1)-file.Size { + return fmt.Errorf("file %q has invalid size", file.Path) + } + total += file.Size + if err := validateSeedHashes(file.Hashes, file.Size, seed.PieceSize); err != nil { + return fmt.Errorf("file %q: %w", file.Path, err) + } + if file.CASSliceMD5 != "" && !validHexHash(file.CASSliceMD5, 32) { + return fmt.Errorf("file %q has an invalid CAS slice MD5", file.Path) + } + for _, source := range file.Sources { + if source.Type == "" || source.URL == "" { + return fmt.Errorf("file %q has an incomplete source", file.Path) + } + u, err := url.Parse(source.URL) + if err != nil || u.Scheme == "" { + return fmt.Errorf("file %q has an invalid source URL", file.Path) + } + } + for _, channel := range file.MissingChannels { + if strings.TrimSpace(channel) == "" || strings.ContainsAny(channel, "/\\\x00") { + return fmt.Errorf("file %q has an invalid missing channel", file.Path) + } + } + } + for _, channel := range seed.Channels { + if strings.TrimSpace(channel.Driver) == "" { + return fmt.Errorf("channel driver is required") + } + if strings.ContainsAny(channel.MountPath, "?#\x00") { + return fmt.Errorf("channel mount_path contains invalid characters") + } + } + return nil +} + +func validateSeedName(name string) error { + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\\x00") { + return fmt.Errorf("invalid seed name %q", name) + } + return nil +} + +func validateRelativeSeedPath(name string) error { + if name == "" || !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || strings.HasPrefix(name, "/") { + return fmt.Errorf("invalid relative path %q", name) + } + cleaned := path.Clean(name) + if cleaned != name || cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return fmt.Errorf("invalid relative path %q", name) + } + for _, part := range strings.Split(name, "/") { + if part == "" || part == "." || part == ".." { + return fmt.Errorf("invalid relative path %q", name) + } + } + return nil +} + +func validateSeedHashes(hashes SeedHashes, size, pieceSize int64) error { + for name, value := range map[string]string{"md5": hashes.MD5, "sha1": hashes.SHA1, "sha256": hashes.SHA256} { + if value != "" && !validHexHash(value, map[string]int{"md5": 32, "sha1": 40, "sha256": 64}[name]) { + return fmt.Errorf("invalid %s hash", name) + } + } + if hashes.Pieces == nil { + return nil + } + expected := 0 + if size > 0 { + expected = int((size + pieceSize - 1) / pieceSize) + } + pieceSets := []struct { + name string + width int + values []string + }{ + {"md5", 32, hashes.Pieces.MD5}, + {"sha1", 40, hashes.Pieces.SHA1}, + {"sha256", 64, hashes.Pieces.SHA256}, + } + for _, set := range pieceSets { + if len(set.values) != 0 && len(set.values) != expected { + return fmt.Errorf("%s piece count is %d, expected %d", set.name, len(set.values), expected) + } + for _, value := range set.values { + if !validHexHash(value, set.width) { + return fmt.Errorf("invalid %s piece hash", set.name) + } + } + } + return nil +} + +func validHexHash(value string, width int) bool { + if len(value) != width { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +// EncodeOSS serializes a validated seed as UTF-8 JSON. +func EncodeOSS(seed *Seed) ([]byte, error) { + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, err + } + return json.MarshalIndent(seed, "", " ") +} + +// DecodeOSS parses a bounded UTF-8 OSS document. +func DecodeOSS(data []byte, limits ParseLimits) (*Seed, error) { + if limits.MaxBytes <= 0 { + limits.MaxBytes = DefaultMaxSeedSize + } + if int64(len(data)) > limits.MaxBytes { + return nil, fmt.Errorf("seed exceeds %d bytes", limits.MaxBytes) + } + data = bytes.TrimPrefix(data, []byte{0xef, 0xbb, 0xbf}) + if !utf8.Valid(data) { + return nil, fmt.Errorf("seed is not valid UTF-8") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var seed Seed + if err := decoder.Decode(&seed); err != nil { + return nil, fmt.Errorf("decode OSS: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nil, fmt.Errorf("decode OSS: trailing data") + } + if err := ValidateSeed(&seed, limits); err != nil { + return nil, err + } + return &seed, nil +} + +// buildCASFileEntry computes the reference-compatible five fields for one file, +// preserving the per-piece MD5 list and piece size when available. +func buildCASFileEntry(file SeedFile, pieceSize int64) (CASFileEntry, error) { + if file.Hashes.MD5 == "" { + return CASFileEntry{}, fmt.Errorf("CAS requires a whole-file MD5 for %s", file.Path) + } + var sliceMD5s []string + if file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 { + sliceMD5s = upperStrings(file.Hashes.Pieces.MD5) + } + sliceMD5 := strings.ToUpper(file.CASSliceMD5) + if sliceMD5 == "" && len(sliceMD5s) > 0 && pieceSize == DefaultPieceSize { + sliceMD5 = sliceMD5s[0] + if len(sliceMD5s) > 1 { + sliceMD5 = strings.ToUpper(GetMD5Str(strings.Join(sliceMD5s, "\n"))) + } + } + if sliceMD5 == "" { + if file.Size > DefaultPieceSize { + return CASFileEntry{}, fmt.Errorf("CAS requires a legacy slice MD5 or complete 10 MiB MD5 pieces for %s", file.Path) + } + sliceMD5 = strings.ToUpper(file.Hashes.MD5) + } + createTime := file.CASCreateTime + if createTime == "" { + createTime = fmt.Sprintf("%d", time.Now().Unix()) + } + entry := CASFileEntry{ + Name: path.Base(file.Path), Size: file.Size, MD5: strings.ToUpper(file.Hashes.MD5), + SliceMD5: sliceMD5, CreateTime: createTime, + } + if len(sliceMD5s) > 0 { + entry.SliceMD5s = sliceMD5s + entry.SliceSize = pieceSize + if entry.SliceSize <= 0 { + entry.SliceSize = DefaultPieceSize + } + } + return entry, nil +} + +// EncodeCAS writes the reference-compatible base64 encoded JSON payload. A +// single-file seed uses the legacy five fields; multiple files are stored in +// the "files" array. +func EncodeCAS(seed *Seed) ([]byte, error) { + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return nil, err + } + if len(seed.Files) == 0 { + return nil, fmt.Errorf("CAS requires at least one file") + } + var payload CASPayload + if len(seed.Files) == 1 { + entry, err := buildCASFileEntry(seed.Files[0], seed.PieceSize) + if err != nil { + return nil, err + } + payload = CASPayload{ + Name: entry.Name, Size: entry.Size, MD5: entry.MD5, + SliceMD5: entry.SliceMD5, CreateTime: entry.CreateTime, + SliceMD5s: entry.SliceMD5s, SliceSize: entry.SliceSize, + } + } else { + entries := make([]CASFileEntry, 0, len(seed.Files)) + var totalSize int64 + for _, file := range seed.Files { + entry, err := buildCASFileEntry(file, seed.PieceSize) + if err != nil { + return nil, err + } + entries = append(entries, entry) + totalSize += entry.Size + } + payload = CASPayload{Name: seed.Name, Size: totalSize, Files: entries} + } + content, err := json.Marshal(payload) + if err != nil { + return nil, err + } + encoded := make([]byte, base64.StdEncoding.EncodedLen(len(content))) + base64.StdEncoding.Encode(encoded, content) + return encoded, nil +} + +// DecodeCAS accepts both reference base64 JSON and raw JSON payloads. +func DecodeCAS(data []byte, limits ParseLimits) (*Seed, error) { + if limits.MaxBytes <= 0 { + limits.MaxBytes = DefaultMaxSeedSize + } + if int64(len(data)) > limits.MaxBytes { + return nil, fmt.Errorf("CAS seed exceeds %d bytes", limits.MaxBytes) + } + data = bytes.TrimSpace(data) + decoded, err := base64.StdEncoding.DecodeString(string(data)) + if err == nil { + data = decoded + } + var payload CASPayload + if err = json.Unmarshal(data, &payload); err != nil { + return nil, fmt.Errorf("decode CAS: %w", err) + } + seed := NewSeed(payload.Name, "OpenList CAS", DefaultPieceSize) + if len(payload.Files) > 0 { + // Multi-file extension. + if len(payload.Files) > limits.MaxFiles { + return nil, fmt.Errorf("CAS seed exceeds %d files", limits.MaxFiles) + } + for _, entry := range payload.Files { + file, err := casEntryToSeedFile(entry.Name, entry.Size, entry.MD5, entry.SliceMD5, entry.CreateTime, entry.SliceMD5s) + if err != nil { + return nil, err + } + if entry.SliceSize > 0 { + seed.PieceSize = entry.SliceSize + } + seed.Files = append(seed.Files, file) + } + return seed, ValidateSeed(seed, limits) + } + file, err := casEntryToSeedFile(payload.Name, payload.Size, payload.MD5, payload.SliceMD5, payload.CreateTime, payload.SliceMD5s) + if err != nil { + return nil, err + } + if payload.SliceSize > 0 { + seed.PieceSize = payload.SliceSize + } + seed.Files = []SeedFile{file} + return seed, ValidateSeed(seed, limits) +} + +// casEntryToSeedFile converts a CAS payload entry into a SeedFile, restoring the +// per-piece MD5 list when it is present. +func casEntryToSeedFile(name string, size int64, md5Hex, sliceMD5Hex, createTime string, sliceMD5s []string) (SeedFile, error) { + if name == "" || size < 0 || !validHexHash(md5Hex, 32) { + return SeedFile{}, fmt.Errorf("invalid CAS payload") + } + sliceMD5 := sliceMD5Hex + if sliceMD5 == "" { + sliceMD5 = md5Hex + } + if !validHexHash(sliceMD5, 32) { + return SeedFile{}, fmt.Errorf("invalid CAS sliceMd5") + } + file := SeedFile{ + Path: name, Size: size, CASCreateTime: createTime, + CASSliceMD5: strings.ToLower(sliceMD5), + Hashes: SeedHashes{MD5: strings.ToLower(md5Hex)}, + } + if len(sliceMD5s) > 0 { + file.Hashes.Pieces = &SeedPieceHashes{MD5: lowerStrings(sliceMD5s)} + } + return file, nil +} + +// DetectFormat determines the seed container from a file name and content. +func DetectFormat(fileName string, data []byte) string { + switch strings.ToLower(path.Ext(fileName)) { + case ".oss": + return "oss" + case ".torrent": + return "torrent" + case ".cas": + return "cas" + } + trimmed := bytes.TrimSpace(data) + if len(trimmed) > 0 && (trimmed[0] == '{' || bytes.HasPrefix(trimmed, []byte{0xef, 0xbb, 0xbf, '{'})) { + return "oss" + } + if len(trimmed) > 0 && trimmed[0] == 'd' { + return "torrent" + } + return "cas" +} + +// DecodeSeed normalizes OSS, torrent and CAS containers to the OSS contract. +func DecodeSeed(data []byte, format string, limits ParseLimits) (*Seed, error) { + if limits.MaxBytes <= 0 { + limits = DefaultParseLimits() + } + if int64(len(data)) > limits.MaxBytes { + return nil, fmt.Errorf("seed exceeds %d bytes", limits.MaxBytes) + } + switch strings.ToLower(strings.TrimPrefix(format, ".")) { + case "oss": + return DecodeOSS(data, limits) + case "torrent": + t, err := Decode(data) + if err != nil { + return nil, err + } + return SeedFromTorrent(t, limits) + case "cas": + return DecodeCAS(data, limits) + default: + return nil, fmt.Errorf("unsupported seed format %q", format) + } +} + +// EncodeSeed converts a normalized seed to the requested container. +func EncodeSeed(seed *Seed, format string) ([]byte, error) { + switch strings.ToLower(strings.TrimPrefix(format, ".")) { + case "oss": + return EncodeOSS(seed) + case "torrent": + t, diagnostics := TorrentFromSeed(seed) + if len(diagnostics) > 0 { + return nil, fmt.Errorf("cannot convert to torrent: %s", strings.Join(diagnostics, "; ")) + } + return t.Encode() + case "cas": + return EncodeCAS(seed) + default: + return nil, fmt.Errorf("unsupported seed format %q", format) + } +} + +// DiagnoseConversion reports information missing for a lossless target conversion. +func DiagnoseConversion(seed *Seed, format string) []string { + if err := ValidateSeed(seed, DefaultParseLimits()); err != nil { + return []string{err.Error()} + } + var diagnostics []string + switch strings.ToLower(strings.TrimPrefix(format, ".")) { + case "torrent": + for i, file := range seed.Files { + if file.Hashes.Pieces == nil || len(file.Hashes.Pieces.SHA1) == 0 { + diagnostics = append(diagnostics, fmt.Sprintf("%s: missing SHA-1 piece hashes", file.Path)) + } + if len(seed.Files) > 1 && i < len(seed.Files)-1 && file.Size%seed.PieceSize != 0 { + diagnostics = append(diagnostics, fmt.Sprintf("%s: piece boundary crosses the next file", file.Path)) + } + } + case "cas": + for _, file := range seed.Files { + if file.Hashes.MD5 == "" { + diagnostics = append(diagnostics, file.Path+": missing whole-file MD5") + } + if file.Size > DefaultPieceSize && file.CASSliceMD5 == "" && + (seed.PieceSize != DefaultPieceSize || file.Hashes.Pieces == nil || len(file.Hashes.Pieces.MD5) == 0) { + diagnostics = append(diagnostics, file.Path+": missing legacy slice MD5 or complete 10 MiB MD5 pieces") + } + } + case "oss": + default: + diagnostics = append(diagnostics, fmt.Sprintf("unsupported target format %q", format)) + } + return diagnostics +} + +// TorrentFromSeed constructs standard BitTorrent info/pieces plus x-openlist. +func TorrentFromSeed(seed *Seed) (*Torrent, []string) { + diagnostics := DiagnoseConversion(seed, "torrent") + if len(diagnostics) > 0 { + return nil, diagnostics + } + t := &Torrent{ + Info: TorrentInfo{PieceLength: seed.PieceSize, Name: seed.Name}, + Comment: seed.Comment, + CreatedBy: seed.CreatedBy, + CreationDate: time.Now().Unix(), + OpenList: seed, + } + if parsed, err := time.Parse(time.RFC3339, seed.CreatedAt); err == nil { + t.CreationDate = parsed.Unix() + } + if len(seed.Trackers) > 0 { + t.Announce = seed.Trackers[0] + for _, tracker := range seed.Trackers { + t.AnnounceList = append(t.AnnounceList, []string{tracker}) + } + } + for _, file := range seed.Files { + for _, piece := range file.Hashes.Pieces.SHA1 { + raw, _ := hex.DecodeString(piece) + t.Info.Pieces = append(t.Info.Pieces, raw...) + } + if len(seed.Files) == 1 { + t.Info.Length = file.Size + t.Info.MD5Sum = file.Hashes.MD5 + } else { + t.Info.Files = append(t.Info.Files, TorrentFile{ + Length: file.Size, + Path: strings.Split(file.Path, "/"), + MD5Sum: file.Hashes.MD5, + }) + } + } + if len(seed.Files) == 1 { + file := seed.Files[0] + if file.Hashes.MD5 != "" && file.CASSliceMD5 != "" { + t.CAS = &CASInfo{ + FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5), + SliceSize: DefaultPieceSize, Cloud: "189", + } + } else if file.Hashes.MD5 != "" && seed.PieceSize == DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 { + t.CAS = BuildCASInfoFromMD5s(file.Hashes.MD5, upperStrings(file.Hashes.Pieces.MD5), DefaultPieceSize) + } + } + return t, nil +} + +func validateOpenListTorrentConsistency(t *Torrent, seed *Seed) error { + if seed.Name != t.Info.Name || seed.PieceSize != t.Info.PieceLength { + return fmt.Errorf("x-openlist metadata conflicts with torrent info") + } + if len(t.Info.Files) == 0 { + if len(seed.Files) != 1 || seed.Files[0].Size != t.Info.Length { + return fmt.Errorf("x-openlist file list conflicts with torrent info") + } + } else { + if len(seed.Files) != len(t.Info.Files) { + return fmt.Errorf("x-openlist file count conflicts with torrent info") + } + for i, file := range t.Info.Files { + if seed.Files[i].Path != strings.Join(file.Path, "/") || seed.Files[i].Size != file.Length { + return fmt.Errorf("x-openlist file %d conflicts with torrent info", i) + } + } + } + standardPieces := t.GetPieceHashes() + extensionPieces := make([]string, 0, len(standardPieces)) + pieceBoundariesAligned := true + for i, file := range seed.Files { + if i < len(seed.Files)-1 && file.Size%seed.PieceSize != 0 { + pieceBoundariesAligned = false + } + if file.Hashes.Pieces != nil { + extensionPieces = append(extensionPieces, file.Hashes.Pieces.SHA1...) + } + } + if pieceBoundariesAligned && len(extensionPieces) > 0 { + if len(extensionPieces) != len(standardPieces) { + return fmt.Errorf("x-openlist SHA-1 pieces conflict with torrent info") + } + for i, piece := range standardPieces { + if !strings.EqualFold(extensionPieces[i], hex.EncodeToString(piece)) { + return fmt.Errorf("x-openlist SHA-1 piece %d conflicts with torrent info", i) + } + } + } + return nil +} + +// SeedFromTorrent normalizes a standard torrent and preserves x-openlist when present. +func SeedFromTorrent(t *Torrent, limits ParseLimits) (*Seed, error) { + if t == nil { + return nil, fmt.Errorf("missing torrent") + } + if t.OpenList != nil { + if err := ValidateSeed(t.OpenList, limits); err != nil { + return nil, err + } + if err := validateOpenListTorrentConsistency(t, t.OpenList); err != nil { + return nil, err + } + return t.OpenList, nil + } + seed := NewSeed(t.Info.Name, t.CreatedBy, t.Info.PieceLength) + if seed.CreatedBy == "" { + seed.CreatedBy = "BitTorrent" + } + if t.CreationDate > 0 { + seed.CreatedAt = time.Unix(t.CreationDate, 0).UTC().Format(time.RFC3339) + } + seed.Comment = t.Comment + seed.Trackers = append(seed.Trackers, t.Announce) + for _, tier := range t.AnnounceList { + seed.Trackers = append(seed.Trackers, tier...) + } + seed.Trackers = uniqueNonEmpty(seed.Trackers) + pieceHex := make([]string, 0, len(t.Info.Pieces)/sha1.Size) + for _, piece := range t.GetPieceHashes() { + pieceHex = append(pieceHex, hex.EncodeToString(piece)) + } + if len(t.Info.Files) == 0 { + hashes := SeedHashes{MD5: t.Info.MD5Sum} + if len(pieceHex) > 0 { + hashes.Pieces = &SeedPieceHashes{SHA1: pieceHex} + } + if t.CAS != nil { + hashes.MD5 = t.CAS.FileMD5 + if hashes.Pieces == nil { + hashes.Pieces = &SeedPieceHashes{} + } + hashes.Pieces.MD5 = append([]string(nil), t.CAS.SliceMD5s...) + } + seed.Files = []SeedFile{{Path: t.Info.Name, Size: t.Info.Length, Hashes: hashes}} + } else { + pieceOffset := 0 + for i, file := range t.Info.Files { + hashes := SeedHashes{MD5: file.MD5Sum} + pieceCount := 0 + if file.Length > 0 && t.Info.PieceLength > 0 { + pieceCount = int((file.Length + t.Info.PieceLength - 1) / t.Info.PieceLength) + } + aligned := i == len(t.Info.Files)-1 || file.Length%t.Info.PieceLength == 0 + if aligned && pieceOffset+pieceCount <= len(pieceHex) { + hashes.Pieces = &SeedPieceHashes{SHA1: append([]string(nil), pieceHex[pieceOffset:pieceOffset+pieceCount]...)} + } + pieceOffset += pieceCount + seed.Files = append(seed.Files, SeedFile{Path: strings.Join(file.Path, "/"), Size: file.Length, Hashes: hashes}) + } + } + if err := ValidateSeed(seed, limits); err != nil { + return nil, err + } + return seed, nil +} + +func seedToBencode(seed *Seed) map[string]interface{} { + root := map[string]interface{}{ + "format": seed.Format, "version": int64(seed.Version), "name": seed.Name, + "created_at": seed.CreatedAt, "created_by": seed.CreatedBy, "piece_size": seed.PieceSize, + } + if seed.Comment != "" { + root["comment"] = seed.Comment + } + if len(seed.Trackers) > 0 { + root["trackers"] = stringsToInterfaces(seed.Trackers) + } + channels := make([]interface{}, 0, len(seed.Channels)) + for _, channel := range seed.Channels { + item := map[string]interface{}{"driver": channel.Driver} + if channel.MountPath != "" { + item["mount_path"] = channel.MountPath + } + channels = append(channels, item) + } + if len(channels) > 0 { + root["channels"] = channels + } + files := make([]interface{}, 0, len(seed.Files)) + for _, file := range seed.Files { + item := map[string]interface{}{"path": file.Path, "size": file.Size, "hashes": hashesToBencode(file.Hashes)} + if file.Modified != "" { + item["modified"] = file.Modified + } + if file.Comment != "" { + item["comment"] = file.Comment + } + if file.CASSliceMD5 != "" { + item["cas_slice_md5"] = file.CASSliceMD5 + } + if file.CASCreateTime != "" { + item["cas_create_time"] = file.CASCreateTime + } + if len(file.MissingChannels) > 0 { + item["missing_channels"] = stringsToInterfaces(file.MissingChannels) + } + sources := make([]interface{}, 0, len(file.Sources)) + for _, source := range file.Sources { + s := map[string]interface{}{"type": source.Type, "url": source.URL} + if source.ExpiresAt != "" { + s["expires_at"] = source.ExpiresAt + } + if source.ShareID != "" { + s["share_id"] = source.ShareID + } + sources = append(sources, s) + } + if len(sources) > 0 { + item["sources"] = sources + } + files = append(files, item) + } + root["files"] = files + return root +} + +func hashesToBencode(hashes SeedHashes) map[string]interface{} { + result := make(map[string]interface{}) + if hashes.MD5 != "" { + result["md5"] = hashes.MD5 + } + if hashes.SHA1 != "" { + result["sha1"] = hashes.SHA1 + } + if hashes.SHA256 != "" { + result["sha256"] = hashes.SHA256 + } + if hashes.Pieces != nil { + pieces := make(map[string]interface{}) + if len(hashes.Pieces.MD5) > 0 { + pieces["md5"] = stringsToInterfaces(hashes.Pieces.MD5) + } + if len(hashes.Pieces.SHA1) > 0 { + pieces["sha1"] = stringsToInterfaces(hashes.Pieces.SHA1) + } + if len(hashes.Pieces.SHA256) > 0 { + pieces["sha256"] = stringsToInterfaces(hashes.Pieces.SHA256) + } + if len(pieces) > 0 { + result["pieces"] = pieces + } + } + return result +} + +func seedFromBencode(value interface{}) (*Seed, error) { + root, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("extension is not a dictionary") + } + seed := &Seed{ + Format: bString(root["format"]), Version: int(bInt(root["version"])), Name: bString(root["name"]), + Comment: bString(root["comment"]), CreatedAt: bString(root["created_at"]), + CreatedBy: bString(root["created_by"]), PieceSize: bInt(root["piece_size"]), + Trackers: bStrings(root["trackers"]), + } + for _, value := range bList(root["channels"]) { + item, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("channel is not a dictionary") + } + seed.Channels = append(seed.Channels, SeedChannel{Driver: bString(item["driver"]), MountPath: bString(item["mount_path"])}) + } + for _, value := range bList(root["files"]) { + item, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("file is not a dictionary") + } + file := SeedFile{ + Path: bString(item["path"]), Size: bInt(item["size"]), Modified: bString(item["modified"]), Comment: bString(item["comment"]), + CASSliceMD5: bString(item["cas_slice_md5"]), CASCreateTime: bString(item["cas_create_time"]), + MissingChannels: bStrings(item["missing_channels"]), + } + if hashes, ok := item["hashes"].(map[string]interface{}); ok { + file.Hashes = SeedHashes{MD5: bString(hashes["md5"]), SHA1: bString(hashes["sha1"]), SHA256: bString(hashes["sha256"])} + if pieces, ok := hashes["pieces"].(map[string]interface{}); ok { + file.Hashes.Pieces = &SeedPieceHashes{MD5: bStrings(pieces["md5"]), SHA1: bStrings(pieces["sha1"]), SHA256: bStrings(pieces["sha256"])} + } + } + for _, sourceValue := range bList(item["sources"]) { + source, ok := sourceValue.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("source is not a dictionary") + } + file.Sources = append(file.Sources, SeedSource{Type: bString(source["type"]), URL: bString(source["url"]), ExpiresAt: bString(source["expires_at"]), ShareID: bString(source["share_id"])}) + } + seed.Files = append(seed.Files, file) + } + return seed, nil +} + +func stringsToInterfaces(values []string) []interface{} { + result := make([]interface{}, len(values)) + for i := range values { + result[i] = values[i] + } + return result +} + +func bString(value interface{}) string { + switch value := value.(type) { + case []byte: + return string(value) + case string: + return value + default: + return "" + } +} + +func bInt(value interface{}) int64 { + valueInt, _ := value.(int64) + return valueInt +} + +func bList(value interface{}) []interface{} { + list, _ := value.([]interface{}) + return list +} + +func bStrings(value interface{}) []string { + list := bList(value) + result := make([]string, 0, len(list)) + for _, item := range list { + if text := bString(item); text != "" { + result = append(result, text) + } + } + return result +} + +func upperStrings(values []string) []string { + result := make([]string, len(values)) + for i, value := range values { + result[i] = strings.ToUpper(value) + } + return result +} + +func lowerStrings(values []string) []string { + result := make([]string, len(values)) + for i, value := range values { + result[i] = strings.ToLower(value) + } + return result +} + +func uniqueNonEmpty(values []string) []string { + seen := make(map[string]struct{}, len(values)) + result := make([]string, 0, len(values)) + for _, value := range values { + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index d97d36d24a..1fd3b3795d 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -63,6 +63,7 @@ type MoveCopyReq struct { Overwrite bool `json:"overwrite"` SkipExisting bool `json:"skip_existing"` Merge bool `json:"merge"` + FollowSeed bool `json:"follow_seed"` } // FsMove performs batch move (individual item permission checks skipped for performance). @@ -152,6 +153,14 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if req.FollowSeed { + seedTasks, followErr := followSeedTransfer(c, "move", p, dstDir) + if followErr != nil { + common.ErrorResp(c, followErr, 500) + return + } + addedTasks = append(addedTasks, seedTasks...) + } } // Return immediately with task information @@ -261,6 +270,14 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if req.FollowSeed { + seedTasks, followErr := followSeedTransfer(c, "copy", p, dstDir) + if followErr != nil { + common.ErrorResp(c, followErr, 500) + return + } + addedTasks = append(addedTasks, seedTasks...) + } } // Return immediately with task information @@ -277,9 +294,10 @@ func FsCopy(c *gin.Context) { } type RenameReq struct { - Path string `json:"path"` - Name string `json:"name"` - Overwrite bool `json:"overwrite"` + Path string `json:"path"` + Name string `json:"name"` + Overwrite bool `json:"overwrite"` + FollowSeed bool `json:"follow_seed"` } func FsRename(c *gin.Context) { @@ -324,6 +342,12 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if req.FollowSeed { + if err := followSeedRename(c, reqPath, req.Name); err != nil { + common.ErrorResp(c, err, 500) + return + } + } common.SuccessResp(c) } @@ -335,8 +359,9 @@ func checkRelativePath(path string) error { } type RemoveReq struct { - Dir string `json:"dir"` - Names []string `json:"names"` + Dir string `json:"dir"` + Names []string `json:"names"` + FollowSeed bool `json:"follow_seed"` } // FsRemove performs batch remove (individual item permission checks skipped for performance). @@ -384,16 +409,88 @@ func FsRemove(c *gin.Context) { if path == "" { continue } + source, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}) err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return } + if req.FollowSeed && source != nil && !source.IsDir() { + if err = followSeedRemove(c, path); err != nil { + common.ErrorResp(c, err, 500) + return + } + } } //fs.ClearCache(req.Dir) common.SuccessResp(c) } +func seedSidecarPaths(filePath string) []string { + return []string{filePath + ".oss", filePath + ".torrent", filePath + ".cas", filePath + ".cas.torrent"} +} + +func followSeedTransfer(c *gin.Context, operation, srcPath, dstDir string) ([]task.TaskExtensionInfo, error) { + source, err := fs.Get(c.Request.Context(), srcPath, &fs.GetArgs{NoLog: true}) + if err != nil || source == nil || source.IsDir() { + return nil, nil + } + var tasks []task.TaskExtensionInfo + for _, sidecarPath := range seedSidecarPaths(srcPath) { + obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true}) + if err != nil || obj == nil || obj.IsDir() { + continue + } + var current task.TaskExtensionInfo + switch operation { + case "copy": + current, err = fs.Copy(c.Request.Context(), sidecarPath, dstDir, true) + case "move": + current, err = fs.Move(c.Request.Context(), sidecarPath, dstDir, true) + default: + return nil, fmt.Errorf("unsupported seed sidecar operation %q", operation) + } + if err != nil { + return tasks, fmt.Errorf("%s seed sidecar %s: %w", operation, sidecarPath, err) + } + if current != nil { + tasks = append(tasks, current) + } + } + return tasks, nil +} + +func followSeedRename(c *gin.Context, srcPath, newName string) error { + source, err := fs.Get(c.Request.Context(), srcPath, &fs.GetArgs{NoLog: true}) + if err != nil || source == nil || source.IsDir() { + return nil + } + for _, sidecarPath := range seedSidecarPaths(srcPath) { + obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true}) + if err != nil || obj == nil || obj.IsDir() { + continue + } + suffix := strings.TrimPrefix(sidecarPath, srcPath) + if err = fs.Rename(c.Request.Context(), sidecarPath, newName+suffix, true); err != nil { + return fmt.Errorf("rename seed sidecar %s: %w", sidecarPath, err) + } + } + return nil +} + +func followSeedRemove(c *gin.Context, srcPath string) error { + for _, sidecarPath := range seedSidecarPaths(srcPath) { + obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true}) + if err != nil || obj == nil || obj.IsDir() { + continue + } + if err = fs.Remove(c.Request.Context(), sidecarPath); err != nil { + return fmt.Errorf("remove seed sidecar %s: %w", sidecarPath, err) + } + } + return nil +} + type RemoveEmptyDirectoryReq struct { SrcDir string `json:"src_dir"` } diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398cdf..8854b3de49 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -1,19 +1,25 @@ package handles import ( + "bytes" + "encoding/json" + "fmt" "io" "net/url" stdpath "path" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/pkg/torrent" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" @@ -97,6 +103,17 @@ func FsStream(c *gin.Context) { if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) } + generateSeed := shouldGenerateUploadSeed(c, dir) + if generateSeed && asTask { + common.ErrorStrResp(c, "seed sidecar generation requires synchronous upload", 400) + return + } + var seedHasher *torrent.HashWriter + var uploadReader io.Reader = c.Request.Body + if generateSeed { + seedHasher = torrent.NewHashWriter(seedPieceSize(c), seedPieceSize(c)) + uploadReader = io.TeeReader(c.Request.Body, seedHasher) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, @@ -104,7 +121,7 @@ func FsStream(c *gin.Context) { Modified: getLastModified(c), HashInfo: utils.NewHashInfoByMap(h), }, - Reader: c.Request.Body, + Reader: uploadReader, Mimetype: mimetype, WebPutAsTask: asTask, } @@ -118,6 +135,12 @@ func FsStream(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if generateSeed { + if err = writeUploadSeedSidecar(c, dir, name, size, seedHasher); err != nil { + common.ErrorResp(c, err, 500) + return + } + } if t == nil { common.SuccessResp(c) return @@ -194,6 +217,17 @@ func FsForm(c *gin.Context) { if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) } + generateSeed := shouldGenerateUploadSeed(c, dir) + if generateSeed && asTask { + common.ErrorStrResp(c, "seed sidecar generation requires synchronous upload", 400) + return + } + var seedHasher *torrent.HashWriter + var uploadReader io.Reader = f + if generateSeed { + seedHasher = torrent.NewHashWriter(seedPieceSize(c), seedPieceSize(c)) + uploadReader = io.TeeReader(f, seedHasher) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, @@ -201,7 +235,7 @@ func FsForm(c *gin.Context) { Modified: getLastModified(c), HashInfo: utils.NewHashInfoByMap(h), }, - Reader: f, + Reader: uploadReader, Mimetype: mimetype, WebPutAsTask: asTask, } @@ -218,6 +252,12 @@ func FsForm(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if generateSeed { + if err = writeUploadSeedSidecar(c, dir, name, file.Size, seedHasher); err != nil { + common.ErrorResp(c, err, 500) + return + } + } if t == nil { common.SuccessResp(c) return @@ -226,3 +266,107 @@ func FsForm(c *gin.Context) { "task": getTaskInfo(t), }) } + +func shouldGenerateUploadSeed(c *gin.Context, path string) bool { + if strings.TrimSpace(c.GetHeader("X-Seed-Sidecars")) != "" { + return true + } + policy := strings.ToLower(strings.TrimSpace(c.GetHeader("X-Generate-Seed"))) + if policy != "" && policy != "inherit" { + return policy == "on" || policy == "true" || policy == "1" + } + if storage := op.GetBalancedStorage(path); storage != nil { + policy = strings.ToLower(strings.TrimSpace(storage.GetStorage().SeedPolicy)) + } + if policy == "" || policy == "inherit" { + policy = strings.ToLower(setting.GetStr(conf.SeedAutoGeneratePolicy, "off")) + } + return (policy == "on" || policy == "true" || policy == "1") && configuredSeedFormats() != "" +} + +func seedPieceSize(c *gin.Context) int64 { + for _, format := range strings.Split(strings.ToLower(c.GetHeader("X-Seed-Sidecars")), ",") { + if strings.TrimSpace(format) == "cas" { + return torrent.DefaultPieceSize + } + } + value := c.GetHeader("X-Seed-Piece-Size") + if value == "" { + return torrent.DefaultPieceSize + } + size, err := strconv.ParseInt(value, 10, 64) + if err != nil || size <= 0 || size > 1<<30 { + return torrent.DefaultPieceSize + } + return size +} + +func configuredSeedFormats() string { + policies := make(map[string]string) + if err := json.Unmarshal([]byte(setting.GetStr(conf.SeedFormatPolicies)), &policies); err != nil { + return "" + } + formats := make([]string, 0, 3) + for _, format := range []string{"oss", "torrent", "cas"} { + if strings.EqualFold(strings.TrimSpace(policies[format]), "on") { + formats = append(formats, format) + } + } + return strings.Join(formats, ",") +} + +func writeUploadSeedSidecar(c *gin.Context, dir, name string, expectedSize int64, hasher *torrent.HashWriter) error { + if hasher == nil { + return nil + } + hasher.Finish() + if expectedSize >= 0 && hasher.GetTotalWritten() != expectedSize { + return fmt.Errorf("seed sidecar requires a complete stream: read %d of %d bytes", hasher.GetTotalWritten(), expectedSize) + } + seed := torrent.NewSeed(name, "OpenList", seedPieceSize(c)) + formatsHeader := strings.TrimSpace(c.GetHeader("X-Seed-Sidecars")) + if formatsHeader == "" { + formatsHeader = strings.TrimSpace(c.GetHeader("X-Seed-Format")) + } + if formatsHeader == "" { + formatsHeader = configuredSeedFormats() + } + formats, err := fs.NormalizeSeedFormats(strings.Split(formatsHeader, ",")) + if err != nil { + return err + } + matrix := SeedHashMatrix{} + if rawMatrix := strings.TrimSpace(c.GetHeader("X-Seed-Hash-Matrix")); rawMatrix != "" { + if err = json.Unmarshal([]byte(rawMatrix), &matrix); err != nil { + return fmt.Errorf("invalid seed hash matrix: %w", err) + } + } + matrix = normalizedSeedMatrix(matrix, formats) + seedFile := hasher.BuildSeedFile(name, getLastModified(c).UTC().Format(time.RFC3339)) + applySeedMatrix(&seedFile, matrix) + seed.Files = []torrent.SeedFile{seedFile} + seen := make(map[string]struct{}, 3) + for _, rawFormat := range formats { + format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), ".")) + if format == "bt" { + format = "torrent" + } + if _, exists := seen[format]; exists { + continue + } + seen[format] = struct{}{} + data, err := fs.EncodeGeneratedSeed(seed, format, hasher.GetPieceHashes()) + if err != nil { + return fmt.Errorf("generate %s seed sidecar: %w", format, err) + } + sidecar := &stream.FileStream{ + Ctx: c.Request.Context(), + Obj: &model.Object{Name: name + "." + format, Size: int64(len(data)), Modified: time.Now()}, + Reader: bytes.NewReader(data), Mimetype: "application/octet-stream", + } + if err = fs.PutDirectly(c.Request.Context(), dir, sidecar, true); err != nil { + return fmt.Errorf("upload %s seed sidecar: %w", format, err) + } + } + return nil +} diff --git a/server/handles/torrent.go b/server/handles/torrent.go index 8b6ee1b6bf..8473b460d7 100644 --- a/server/handles/torrent.go +++ b/server/handles/torrent.go @@ -1,18 +1,30 @@ package handles import ( + "bytes" + "context" "encoding/base64" + "encoding/json" "fmt" "io" + "net/url" + stdpath "path" "strings" + "time" _189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/torrent" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" @@ -172,10 +184,14 @@ func TorrentRapidUpload(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - if !common.CanWrite(user, meta, reqPath) { + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, reqPath)) || !common.CanWrite(user, meta, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + if len(req.TorrentData) > maxTorrentBase64Len { + common.ErrorStrResp(c, "torrent data exceeds the maximum size", 413) + return + } // Base64 解码 torrentData, err := base64.StdEncoding.DecodeString(req.TorrentData) @@ -392,13 +408,15 @@ func GenerateTorrentForPath(c *gin.Context) { defer link.Close() // 通过 RangeReader 获取文件内容并计算哈希生成 torrent - if link.RangeReader == nil { - common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取，无法生成 torrent（请先下载文件到本地）"), 400) + // 对于仅返回 URL（无 RangeReader）的驱动，GetRangeReaderFromLink 会退化为 HTTP Range 流式读取 + rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link) + if err != nil { + common.ErrorResp(c, fmt.Errorf("该存储不支持流式读取，无法生成 torrent: %w", err), 400) return } // 读取整个文件 - rc, err := link.RangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + rc, err := rangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) if err != nil { common.ErrorResp(c, fmt.Errorf("读取文件失败: %w", err), 500) return @@ -431,3 +449,1189 @@ func GenerateTorrentForPath(c *gin.Context) { "with_cas": req.WithCAS, }) } + +// SeedDataReq carries a bounded base64-encoded seed document. +// SeedData is intentionally NOT bound with `required` here: some requests that +// embed SeedDataReq (e.g. SeedCapabilityReq preflight) accept `paths` instead of +// `seed_data`. Required-ness is enforced inside decodeSeedData for the handlers +// that actually consume a seed document. +type SeedDataReq struct { + SeedData string `json:"seed_data"` + Format string `json:"format"` + FileName string `json:"file_name"` +} + +// SeedConvertReq requests a container conversion without changing metadata. +type SeedConvertReq struct { + SeedDataReq + TargetFormat string `json:"target_format"` + Format string `json:"format"` + Path string `json:"path"` +} + +// SeedHashSelection controls whole-file and piece hash inclusion. +type SeedHashSelection struct { + Whole bool `json:"whole"` + Pieces bool `json:"pieces"` +} + +// SeedHashMatrix controls the optional hash metadata stored in a seed. +type SeedHashMatrix struct { + MD5 SeedHashSelection `json:"md5"` + SHA1 SeedHashSelection `json:"sha1"` + SHA256 SeedHashSelection `json:"sha256"` +} + +// SeedGenerateReq generates one or more seed containers for existing files. +type SeedGenerateReq struct { + Paths []string `json:"paths" binding:"required"` + Format string `json:"format"` + Formats []string `json:"formats"` + Name string `json:"name"` + Comment string `json:"comment"` + FileComments map[string]string `json:"file_comments"` + HashMatrix SeedHashMatrix `json:"hash_matrix"` + PieceSize int64 `json:"piece_size"` + Trackers []string `json:"trackers"` + Channels []torrent.SeedChannel `json:"channels"` + OutputPath string `json:"output_path"` + SavePath string `json:"save_path"` + IncludeShare bool `json:"include_share"` + IncludeDirectSource bool `json:"include_direct_source"` + ShareFiles []string `json:"share_files"` + DirectFiles []string `json:"direct_files"` +} + +// SeedCapabilityReq supports source preflight and destination import planning. +type SeedCapabilityReq struct { + SeedDataReq + Paths []string `json:"paths"` + Path string `json:"path"` + Override string `json:"policy"` +} + +// SeedUpdateChannelsReq replaces only public channel metadata. +type SeedUpdateChannelsReq struct { + SeedDataReq + Channels []torrent.SeedChannel `json:"channels"` +} + +// SeedRecalcFile maps one seed file to a server-side path for re-hashing. +type SeedRecalcFile struct { + Path string `json:"path"` // seed file relative path + SourcePath string `json:"source_path"` // server-side readable file path +} + +// SeedUpdateReq edits metadata and optionally recalculates hashes. +type SeedUpdateReq struct { + SeedDataReq + Comment *string `json:"comment"` + Trackers []string `json:"trackers"` + Channels []torrent.SeedChannel `json:"channels"` + FileComments map[string]string `json:"file_comments"` + FileSources map[string][]torrent.SeedSource `json:"file_sources"` + RemoveFiles []string `json:"remove_files"` + Recalculate bool `json:"recalculate"` + RecalcFiles []SeedRecalcFile `json:"recalc_files"` + HashMatrix SeedHashMatrix `json:"hash_matrix"` + PieceSize int64 `json:"piece_size"` + OutputPath string `json:"output_path"` + SavePath string `json:"save_path"` + Options map[string]any `json:"options"` +} + +// SeedQuickSaveReq imports selected seed files using rapid upload or an existing source URL. +type SeedQuickSaveReq struct { + SeedDataReq + Path string `json:"path" binding:"required"` + Files []string `json:"files"` + SelectedFiles []int `json:"selected_files"` + Tool string `json:"tool"` + DeletePolicy string `json:"delete_policy"` + Overwrite bool `json:"overwrite"` + TransitPath string `json:"transit_path"` + UpdateChannel bool `json:"update_channel"` + Options map[string]any `json:"options"` +} + +func decodeSeedData(req SeedDataReq) ([]byte, *torrent.Seed, string, error) { + if strings.TrimSpace(req.SeedData) == "" { + return nil, nil, "", fmt.Errorf("seed_data is required") + } + if len(req.SeedData) > maxTorrentBase64Len { + return nil, nil, "", fmt.Errorf("seed data is too large") + } + data, err := base64.StdEncoding.DecodeString(req.SeedData) + if err != nil { + return nil, nil, "", fmt.Errorf("invalid base64 seed data: %w", err) + } + format := strings.ToLower(strings.TrimPrefix(req.Format, ".")) + if format == "" { + format = torrent.DetectFormat(req.FileName, data) + } + seed, err := torrent.DecodeSeed(data, format, torrent.DefaultParseLimits()) + if err != nil { + return nil, nil, "", err + } + return data, seed, format, nil +} + +// ParseSeed parses OSS, torrent or CAS data into one preview contract. +func ParseSeed(c *gin.Context) { + var req SeedDataReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, format, err := decodeSeedData(req) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + hasSource := false + hasRapidHashes := false + for _, file := range seed.Files { + hasSource = hasSource || firstUsableSeedSource(file) != "" + hasRapidHashes = hasRapidHashes || file.Hashes.MD5 != "" || file.Hashes.SHA1 != "" || file.Hashes.SHA256 != "" + } + common.SuccessResp(c, gin.H{ + "format": format, + "seed": seed, + "files": seed.Files, + "total_size": seedTotalSize(seed), + "diagnostics": seedDiagnostics(seed), + "conversions": seedConversionStates(seed), + "capabilities": gin.H{ + "rapid_upload": hasRapidHashes, "offline_download": hasSource || format == "torrent", + "transfer": hasSource, "convert": true, "edit": true, "recalculate": true, + }, + "direct_preview": len(seed.Files) == 1 && setting.GetBool(conf.SeedSingleDirectPreview), + }) +} + +// UploadSeedAndParse parses a multipart seed upload with the same limits as JSON parsing. +func UploadSeedAndParse(c *gin.Context) { + file, err := c.FormFile("seed") + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if file.Size < 0 || file.Size > torrent.DefaultMaxSeedSize { + common.ErrorStrResp(c, "seed file is too large", 400) + return + } + r, err := file.Open() + if err != nil { + common.ErrorResp(c, err, 400) + return + } + defer r.Close() + data, err := io.ReadAll(io.LimitReader(r, torrent.DefaultMaxSeedSize+1)) + if err != nil || int64(len(data)) > torrent.DefaultMaxSeedSize { + common.ErrorStrResp(c, "failed to read bounded seed file", 400) + return + } + format := torrent.DetectFormat(file.Filename, data) + seed, err := torrent.DecodeSeed(data, format, torrent.DefaultParseLimits()) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, gin.H{ + "format": format, + "seed": seed, + "seed_data": base64.StdEncoding.EncodeToString(data), + "diagnostics": seedDiagnostics(seed), + }) +} + +// ConvertSeed converts between OSS, standard torrent+x-openlist and CAS containers. +func ConvertSeed(c *gin.Context) { + var req SeedConvertReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, _, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + targetFormat := req.TargetFormat + if targetFormat == "" { + targetFormat = req.Format + } + if targetFormat == "" { + common.ErrorStrResp(c, "target format is required", 400) + return + } + diagnostics := torrent.DiagnoseConversion(seed, targetFormat) + if len(diagnostics) > 0 { + common.SuccessResp(c, gin.H{"convertible": false, "diagnostics": diagnostics}) + return + } + data, err := torrent.EncodeSeed(seed, targetFormat) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + format := strings.ToLower(strings.TrimPrefix(targetFormat, ".")) + fileName := stdpath.Base(seed.Name) + "." + format + result := gin.H{ + "convertible": true, "format": format, "name": fileName, + "data": base64.StdEncoding.EncodeToString(data), "seed_data": base64.StdEncoding.EncodeToString(data), "size": len(data), + } + if req.Path != "" { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + dstDir, joinErr := user.JoinPath(req.Path) + if joinErr != nil { + common.ErrorResp(c, joinErr, 403) + return + } + meta, metaErr := op.GetNearestMeta(dstDir) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + common.ErrorResp(c, metaErr, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + fileStream := &stream.FileStream{Ctx: c.Request.Context(), Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()}, Reader: bytes.NewReader(data), Mimetype: "application/octet-stream"} + if err = fs.PutDirectly(c.Request.Context(), dstDir, fileStream); err != nil { + common.ErrorResp(c, err, 500) + return + } + result["path"] = stdpath.Join(req.Path, fileName) + } + common.SuccessResp(c, result) +} + +// DiagnoseSeed reports conversion requirements for every supported container. +func DiagnoseSeed(c *gin.Context) { + var req SeedDataReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, _, err := decodeSeedData(req) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c, seedDiagnostics(seed)) +} + +func seedMatrixEmpty(matrix SeedHashMatrix) bool { + return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces +} + +// loadSeedDefaultMatrix parses the configured default right-click hash matrix. +func loadSeedDefaultMatrix() SeedHashMatrix { + var matrix SeedHashMatrix + if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" { + _ = json.Unmarshal([]byte(raw), &matrix) + } + return matrix +} + +func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix { + if seedMatrixEmpty(matrix) { + matrix = loadSeedDefaultMatrix() + } + if seedMatrixEmpty(matrix) { + matrix = SeedHashMatrix{ + MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true}, + SHA256: SeedHashSelection{Whole: true, Pieces: true}, + } + } + for _, format := range formats { + switch format { + case "torrent": + matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true} + case "cas": + matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true} + } + } + return matrix +} + +func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) { + if !matrix.MD5.Whole { + file.Hashes.MD5 = "" + } + if !matrix.SHA1.Whole { + file.Hashes.SHA1 = "" + } + if !matrix.SHA256.Whole { + file.Hashes.SHA256 = "" + } + if file.Hashes.Pieces == nil { + return + } + if !matrix.MD5.Pieces { + file.Hashes.Pieces.MD5 = nil + } + if !matrix.SHA1.Pieces { + file.Hashes.Pieces.SHA1 = nil + } + if !matrix.SHA256.Pieces { + file.Hashes.Pieces.SHA256 = nil + } + if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 { + file.Hashes.Pieces = nil + } +} + +// toSeedGenerateParams converts the HTTP request into the transport-agnostic +// params consumed by fs.GenerateSeedArtifacts. +func toSeedGenerateParams(req SeedGenerateReq) fs.SeedGenerateParams { + return fs.SeedGenerateParams{ + Paths: req.Paths, + Formats: req.Formats, + Name: req.Name, + Comment: req.Comment, + FileComments: req.FileComments, + HashMatrix: fs.SeedHashMatrix{ + MD5: fs.SeedHashSelection(req.HashMatrix.MD5), + SHA1: fs.SeedHashSelection(req.HashMatrix.SHA1), + SHA256: fs.SeedHashSelection(req.HashMatrix.SHA256), + }, + PieceSize: req.PieceSize, + Trackers: req.Trackers, + Channels: req.Channels, + OutputPath: req.OutputPath, + IncludeShare: req.IncludeShare, + IncludeDirectSource: req.IncludeDirectSource, + ShareFiles: req.ShareFiles, + DirectFiles: req.DirectFiles, + } +} + +// GenerateSeedForPaths reads each file once while calculating complete hashes. +// Requests exceeding the synchronous size limit are queued as a background task. +func GenerateSeedForPaths(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedGenerateReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Paths) == 0 || len(req.Paths) > torrent.DefaultMaxSeedFiles { + common.ErrorStrResp(c, "invalid seed file count", 400) + return + } + if len(req.Formats) == 0 { + req.Formats = []string{req.Format} + } + params := toSeedGenerateParams(req) + async, err := fs.SeedGenerateNeedsAsync(c.Request.Context(), user, req.Paths) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if async { + outputPath := strings.TrimSpace(req.OutputPath) + if outputPath == "" { + outputPath = strings.TrimSpace(req.SavePath) + } + if outputPath == "" { + common.ErrorStrResp(c, "asynchronous seed generation requires an output_path", 400) + return + } + params.OutputPath = outputPath + taskInfo, addErr := fs.AddSeedGenerateTask(c.Request.Context(), user, params) + if addErr != nil { + common.ErrorResp(c, addErr, 500) + return + } + common.SuccessResp(c, gin.H{"task": getTaskInfo(taskInfo), "async": true}) + return + } + artifacts, seed, genErr := fs.GenerateSeedArtifacts(c.Request.Context(), user, params) + if genErr != nil { + status := 400 + if errors.Is(genErr, errs.PermissionDenied) { + status = 403 + } + common.ErrorResp(c, genErr, status) + return + } + common.SuccessResp(c, gin.H{"artifacts": artifacts, "seed": seed}) +} + +// loadSeedDefaultTrackers returns the configured default tracker list, one per line. +func loadSeedDefaultTrackers() []string { + raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultTrackers)) + if raw == "" { + return nil + } + var trackers []string + for _, line := range strings.Split(raw, "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + trackers = append(trackers, trimmed) + } + } + return trackers +} + +// directSourceAvailable reports whether an unauthenticated /d/ direct source can be embedded. +// Direct sources are only safe when the file is not encrypted and guest access is allowed. +func directSourceAvailable(meta *model.Meta, path string) bool { + if setting.GetBool(conf.SignAll) { + return false + } + if isEncrypt(meta, path) { + return false + } + guest, err := op.GetGuest() + return err == nil && !guest.Disabled +} + +// SeedCapabilities reports whether files can use hashes, source URLs or require downloading. +func SeedCapabilities(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedCapabilityReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Paths) > 0 { + if len(req.Paths) > torrent.DefaultMaxSeedFiles { + common.ErrorStrResp(c, "invalid seed file count", 400) + return + } + files := make([]gin.H, 0, len(req.Paths)) + var estimatedTraffic int64 + existing := make(map[string]bool) + for _, requestedPath := range req.Paths { + fullPath, joinErr := user.JoinPath(requestedPath) + if joinErr != nil { + common.ErrorResp(c, joinErr, 403) + return + } + meta, metaErr := op.GetNearestMeta(fullPath) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + common.ErrorResp(c, metaErr, 500, true) + return + } + if !common.CanRead(user, meta, fullPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + storage, actualPath, getErr := op.GetStorageAndActualPath(fullPath) + if getErr != nil { + common.ErrorResp(c, getErr, 400) + return + } + obj, getErr := op.Get(c.Request.Context(), storage, actualPath) + if getErr != nil || obj.IsDir() { + common.ErrorResp(c, fmt.Errorf("seed path must be a readable file: %s", requestedPath), 400) + return + } + available := make([]string, 0, 3) + hashInfo := obj.GetHash() + for _, hashType := range []*utils.HashType{utils.MD5, utils.SHA1, utils.SHA256} { + if hashInfo.GetHash(hashType) != "" { + available = append(available, hashType.Name) + existing[hashType.Name] = true + } + } + requiresDownload := len(available) < 3 + var fileTraffic int64 + if requiresDownload { + fileTraffic = obj.GetSize() + estimatedTraffic += fileTraffic + } + // Probe whether the storage can be streamed for server-side hashing. + streamable := false + if link, _, linkErr := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}); linkErr == nil && link != nil { + if _, rrErr := stream.GetRangeReaderFromLink(obj.GetSize(), link); rrErr == nil { + streamable = true + } + _ = link.Close() + } + files = append(files, gin.H{ + "path": requestedPath, "name": obj.GetName(), "size": obj.GetSize(), + "available_hashes": available, "requires_download": requiresDownload, + "requires_fetch": requiresDownload, "estimated_traffic": fileTraffic, + "streamable": streamable, "share_available": user.CanShare(), + "direct_source_available": directSourceAvailable(meta, fullPath), + }) + } + existingHashes := make([]string, 0, len(existing)) + for _, name := range []string{"md5", "sha1", "sha256"} { + if existing[name] { + existingHashes = append(existingHashes, name) + } + } + common.SuccessResp(c, gin.H{ + "formats": gin.H{"oss": true, "torrent": true, "cas": true}, + "files": files, "existing_hashes": existingHashes, "estimated_traffic": estimatedTraffic, + "default_matrix": loadSeedDefaultMatrix(), "trackers": loadSeedDefaultTrackers(), + }) + return + } + _, seed, _, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + dstPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(dstPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstPath)) || !common.CanWrite(user, meta, dstPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + storage, _, err := op.GetStorageAndActualPath(dstPath) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, rapid189 := storage.(*_189pc.Cloud189PC) + _, putURL := storage.(driver.PutURL) + _, putURLResult := storage.(driver.PutURLResult) + files := make([]gin.H, 0, len(seed.Files)) + for _, file := range seed.Files { + hasCAS := rapid189 && file.Hashes.MD5 != "" && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 + hasSource := firstUsableSeedSource(file) != "" + method := "download_required" + if hasCAS { + method = "189pc_cas" + } else if hasSource && (putURL || putURLResult) { + method = "put_url" + } else if hasSource { + method = "offline_download" + } + files = append(files, gin.H{"path": file.Path, "method": method, "requires_download": method == "download_required"}) + } + globalPolicy := setting.GetStr(conf.SeedAutoGeneratePolicy, "off") + policy := strings.ToLower(strings.TrimSpace(req.Override)) + if policy == "" || policy == "inherit" { + policy = globalPolicy + } + if policy != "on" && policy != "off" { + common.ErrorStrResp(c, "policy must be off, on, or inherit", 400) + return + } + // Describe the destination driver's rapid-transfer capability surface so the + // frontend can show which hashes are reusable for instant upload. + driverSupports := gin.H{ + "cas_rapid": rapid189, + "put_url": putURL || putURLResult, + "offline_download": true, + "rapid_hash_algos": []string{"md5", "sha1"}, + "rapid_uses_pieces": rapid189, + } + common.SuccessResp(c, gin.H{ + "driver": storage.Config().Name, "global_policy": globalPolicy, + "resolved_policy": policy, "files": files, "driver_supports": driverSupports, + }) +} + +// UpdateSeedChannels updates public discovery metadata and keeps the requested container. +func UpdateSeedChannels(c *gin.Context) { + UpdateSeed(c) +} + +// applySeedUpdateOptions folds the frontend options passthrough into typed fields. +func applySeedUpdateOptions(req *SeedUpdateReq) { + if req.Options == nil { + return + } + if req.Comment == nil { + if v, ok := req.Options["comment"].(string); ok { + req.Comment = &v + } + } + if !req.Recalculate { + if v, ok := req.Options["recalculate"].(bool); ok { + req.Recalculate = v + } + } +} + +// validateSeedSource enforces the metadata contract for editable share/direct sources. +func validateSeedSource(src torrent.SeedSource) error { + if src.Type != "openlist-direct" && src.Type != "openlist-share" { + return fmt.Errorf("unsupported seed source type %q", src.Type) + } + u, err := url.Parse(src.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil { + return fmt.Errorf("invalid seed source URL for %q", src.Type) + } + if configured := strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)); configured != "" { + if site, parseErr := url.Parse(configured); parseErr == nil && !strings.EqualFold(u.Host, site.Host) { + return fmt.Errorf("seed source host must match the configured site URL") + } + } + if src.Type == "openlist-direct" && !strings.HasPrefix(u.EscapedPath(), "/d/") { + return fmt.Errorf("openlist-direct source URL must start with /d/") + } + if src.Type == "openlist-share" && !strings.HasPrefix(u.EscapedPath(), "/sd/") { + return fmt.Errorf("openlist-share source URL must start with /sd/") + } + return nil +} + +// rehashSeedFile reads one server-side file and recomputes its whole and piece hashes. +func rehashSeedFile(c *gin.Context, user *model.User, sourcePath string, pieceSize int64) (torrent.SeedFile, error) { + var out torrent.SeedFile + fullPath, err := user.JoinPath(sourcePath) + if err != nil { + return out, err + } + meta, err := op.GetNearestMeta(fullPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return out, err + } + if !common.CanRead(user, meta, fullPath) { + return out, errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return out, err + } + obj, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil || obj.IsDir() { + return out, fmt.Errorf("recalculate path must be a readable file: %s", sourcePath) + } + if obj.GetSize() < 0 || obj.GetSize() > maxTorrentGenFileSize { + return out, fmt.Errorf("recalculate file exceeds 1GB limit: %s", sourcePath) + } + link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{}) + if err != nil { + return out, fmt.Errorf("storage cannot stream %s: %v", sourcePath, err) + } + rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link) + if err != nil { + return out, fmt.Errorf("storage cannot stream %s", sourcePath) + } + rc, err := rangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()}) + if err != nil { + return out, err + } + defer rc.Close() + hasher := torrent.NewHashWriter(pieceSize, pieceSize) + n, copyErr := io.Copy(hasher, io.LimitReader(rc, obj.GetSize()+1)) + if copyErr != nil { + return out, fmt.Errorf("read %s: %w", sourcePath, copyErr) + } + if n != obj.GetSize() { + return out, fmt.Errorf("read %s: got %d of %d bytes", sourcePath, n, obj.GetSize()) + } + hasher.Finish() + modified := "" + if !obj.ModTime().IsZero() { + modified = obj.ModTime().UTC().Format(time.RFC3339) + } + return hasher.BuildSeedFile(stdpath.Base(sourcePath), modified), nil +} + +// UpdateSeed edits seed metadata and optionally recalculates hashes from server files. +func UpdateSeed(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedUpdateReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + applySeedUpdateOptions(&req) + _, seed, format, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.Comment != nil { + seed.Comment = strings.TrimSpace(*req.Comment) + } + if req.Trackers != nil { + seed.Trackers = req.Trackers + } + if req.Channels != nil { + seed.Channels = req.Channels + } + for i := range seed.Files { + if comment, ok := req.FileComments[seed.Files[i].Path]; ok { + seed.Files[i].Comment = strings.TrimSpace(comment) + } + if sources, ok := req.FileSources[seed.Files[i].Path]; ok { + for _, src := range sources { + if srcErr := validateSeedSource(src); srcErr != nil { + common.ErrorResp(c, srcErr, 400) + return + } + } + seed.Files[i].Sources = sources + } + } + if len(req.RemoveFiles) > 0 { + removeSet := make(map[string]struct{}, len(req.RemoveFiles)) + for _, path := range req.RemoveFiles { + if trimmed := strings.TrimSpace(path); trimmed != "" { + removeSet[trimmed] = struct{}{} + } + } + if len(removeSet) == 0 { + common.ErrorStrResp(c, "remove_files requires at least one non-empty path", 400) + return + } + kept := make([]torrent.SeedFile, 0, len(seed.Files)) + for _, file := range seed.Files { + if _, removed := removeSet[file.Path]; !removed { + kept = append(kept, file) + } + } + if len(kept) == 0 { + common.ErrorStrResp(c, "cannot remove all files from a seed", 400) + return + } + seed.Files = kept + } + if req.Recalculate { + pieceSize := req.PieceSize + if pieceSize <= 0 { + pieceSize = seed.PieceSize + } + if pieceSize <= 0 { + pieceSize = torrent.DefaultPieceSize + } + if pieceSize < 16*1024 || pieceSize > 64*1024*1024 { + common.ErrorStrResp(c, "piece_size must be between 16384 and 67108864", 400) + return + } + seed.PieceSize = pieceSize + matrix := normalizedSeedMatrix(req.HashMatrix, []string{format}) + recalcMap := make(map[string]string, len(req.RecalcFiles)) + for _, rf := range req.RecalcFiles { + if strings.TrimSpace(rf.SourcePath) != "" { + recalcMap[rf.Path] = rf.SourcePath + } + } + if len(recalcMap) == 0 { + common.ErrorStrResp(c, "recalculate requires at least one source_path", 400) + return + } + for i := range seed.Files { + srcPath, ok := recalcMap[seed.Files[i].Path] + if !ok { + continue + } + rehashed, rehashErr := rehashSeedFile(c, user, srcPath, pieceSize) + if rehashErr != nil { + common.ErrorResp(c, rehashErr, 400) + return + } + rehashed.Path = seed.Files[i].Path + rehashed.Comment = seed.Files[i].Comment + rehashed.Sources = seed.Files[i].Sources + rehashed.CASSliceMD5 = "" + rehashed.CASCreateTime = "" + applySeedMatrix(&rehashed, matrix) + seed.Files[i] = rehashed + } + } + data, err := torrent.EncodeSeed(seed, format) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + result := gin.H{"format": format, "seed_data": base64.StdEncoding.EncodeToString(data), "seed": seed} + outputPath := strings.TrimSpace(req.OutputPath) + if outputPath == "" { + outputPath = strings.TrimSpace(req.SavePath) + } + if outputPath != "" { + fileName := stdpath.Base(seed.Name) + "." + format + dstDir, joinErr := user.JoinPath(outputPath) + if joinErr != nil { + common.ErrorResp(c, joinErr, 403) + return + } + meta, metaErr := op.GetNearestMeta(dstDir) + if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) { + common.ErrorResp(c, metaErr, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + fileStream := &stream.FileStream{Ctx: c.Request.Context(), Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()}, Reader: bytes.NewReader(data), Mimetype: "application/octet-stream"} + if putErr := fs.PutDirectly(c.Request.Context(), dstDir, fileStream); putErr != nil { + common.ErrorResp(c, putErr, 500) + return + } + result["path"] = stdpath.Join(outputPath, fileName) + } + shareStatus := checkSeedShareValidity(seed) + if len(shareStatus) > 0 { + result["share_status"] = shareStatus + } + common.SuccessResp(c, result) +} + +// saveSeedFilesToPath saves selected seed files into one target directory. +func saveSeedFilesToPath(c *gin.Context, user *model.User, seed *torrent.Seed, req SeedQuickSaveReq, dstPath string, rapidOnly bool) ([]gin.H, string, string, error) { + results := make([]gin.H, 0, len(seed.Files)) + fullPath, err := user.JoinPath(dstPath) + if err != nil { + return nil, "", "", err + } + meta, err := op.GetNearestMeta(fullPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, "", "", err + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, fullPath)) || !common.CanWrite(user, meta, fullPath) { + return nil, "", "", errs.PermissionDenied + } + storage, actualPath, err := op.GetStorageAndActualPath(fullPath) + if err != nil { + return nil, "", "", err + } + dstDir, err := op.Get(c.Request.Context(), storage, actualPath) + if err != nil || !dstDir.IsDir() { + return nil, "", "", errs.NotFolder + } + driverName, mountPath := "", "" + if mounted := storage.GetStorage(); mounted != nil { + driverName = strings.TrimSpace(mounted.Driver) + mountPath = strings.TrimSpace(mounted.MountPath) + } + selected := make(map[string]struct{}, len(req.Files)) + for _, name := range req.Files { + selected[name] = struct{}{} + } + selectedIndexes := make(map[int]struct{}, len(req.SelectedFiles)) + for _, index := range req.SelectedFiles { + if index < 0 || index >= len(seed.Files) { + return nil, "", "", fmt.Errorf("selected_files contains an invalid index") + } + selectedIndexes[index] = struct{}{} + } + cloud189, is189 := storage.(*_189pc.Cloud189PC) + _, putURL := storage.(driver.PutURL) + _, putURLResult := storage.(driver.PutURLResult) + for index, file := range seed.Files { + if len(selected) > 0 { + if _, ok := selected[file.Path]; !ok { + continue + } + } else if len(selectedIndexes) > 0 { + if _, ok := selectedIndexes[index]; !ok { + continue + } + } + name := stdpath.Base(file.Path) + if is189 && file.Hashes.MD5 != "" && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 && len(file.Hashes.Pieces.SHA1) > 0 { + one := *seed + one.Name = name + one.Files = []torrent.SeedFile{file} + if data, encodeErr := torrent.EncodeSeed(&one, "torrent"); encodeErr == nil { + if obj, rapidErr := cloud189.RapidUploadFromTorrent(c.Request.Context(), dstDir, data, req.Overwrite); rapidErr == nil { + results = append(results, gin.H{"path": file.Path, "name": obj.GetName(), "method": "189pc_cas"}) + continue + } + } + } + if rapidOnly { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "unavailable", "error": "target driver cannot reuse the available hashes"}) + continue + } + source := firstUsableSeedSource(file) + if source == "" { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "unavailable", "error": "no usable source URL or compatible rapid-upload hashes"}) + continue + } + if putURL || putURLResult { + if putErr := fs.PutURL(c.Request.Context(), fullPath, name, source); putErr == nil { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "put_url"}) + continue + } + } + if !user.CanAddOfflineDownloadTasks() { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": "offline download permission is required"}) + continue + } + if req.Tool == "" { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": "offline download tool is required"}) + continue + } + t, addErr := tool.AddURL(c, &tool.AddURLArgs{URL: source, DstDirPath: fullPath, Tool: req.Tool, DeletePolicy: tool.DeletePolicy(req.DeletePolicy)}) + if addErr != nil { + results = append(results, gin.H{"path": file.Path, "name": name, "method": "offline_download", "error": addErr.Error()}) + continue + } + result := gin.H{"path": file.Path, "name": name, "method": "offline_download"} + if t != nil { + result["task"] = getTaskInfo(t) + } + results = append(results, result) + } + return results, driverName, mountPath, nil +} + +// resolveSeedTransitPath extracts the optional transit destination for a relayed save. +func resolveSeedTransitPath(req SeedQuickSaveReq) string { + transitPath := strings.TrimSpace(req.TransitPath) + if transitPath == "" && req.Options != nil { + if mode, ok := req.Options["mode"].(string); ok && mode == "transfer" { + if tp, ok := req.Options["transit_path"].(string); ok { + transitPath = strings.TrimSpace(tp) + } + } + } + return transitPath +} + +// transferSeedViaTransit first saves to an intermediate directory then relays to the final target. +func transferSeedViaTransit(c *gin.Context, user *model.User, seed *torrent.Seed, req SeedQuickSaveReq, transitPath string) { + if !user.CanCopy() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + transitResults, _, _, err := saveSeedFilesToPath(c, user, seed, req, transitPath, false) + if err != nil { + common.ErrorResp(c, err, seedSaveErrorCode(err)) + return + } + finalDst, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + finalMeta, err := op.GetNearestMeta(finalDst) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(finalMeta, finalDst)) || !common.CanWrite(user, finalMeta, finalDst) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + transitFullPath, err := user.JoinPath(transitPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + finalResults := make([]gin.H, 0, len(transitResults)) + for _, res := range transitResults { + method, _ := res["method"].(string) + if method == "unavailable" { + finalResults = append(finalResults, res) + continue + } + name, _ := res["name"].(string) + if name == "" { + continue + } + // Asynchronous offline downloads cannot be relayed synchronously. + if method == "offline_download" { + finalResults = append(finalResults, gin.H{"path": res["path"], "name": name, "method": "transfer", "error": "transit requires a synchronous intermediate save (rapid upload or PutURL)"}) + continue + } + srcObjPath := stdpath.Join(transitFullPath, name) + _, copyErr := fs.Copy(context.WithValue(c.Request.Context(), conf.NoTaskKey, struct{}{}), srcObjPath, finalDst) + if copyErr != nil { + finalResults = append(finalResults, gin.H{"path": res["path"], "name": name, "method": "transfer", "error": copyErr.Error()}) + continue + } + finalResults = append(finalResults, gin.H{"path": res["path"], "name": name, "method": "transfer"}) + } + common.SuccessResp(c, gin.H{"results": finalResults}) +} + +// seedSaveErrorCode maps a save error to an HTTP status code. +func seedSaveErrorCode(err error) int { + if errors.Is(errors.Cause(err), errs.PermissionDenied) { + return 403 + } + if errors.Is(errors.Cause(err), errs.NotFolder) || errors.Is(errors.Cause(err), errs.ObjectNotFound) { + return 400 + } + return 500 +} + +// checkSeedShareValidity reports whether each openlist-share source still resolves to a valid sharing. +func checkSeedShareValidity(seed *torrent.Seed) gin.H { + status := make(gin.H) + for _, file := range seed.Files { + for _, source := range file.Sources { + if source.Type != "openlist-share" || strings.TrimSpace(source.ShareID) == "" { + continue + } + shareID := strings.TrimSpace(source.ShareID) + if _, checked := status[shareID]; checked { + continue + } + valid := false + if sharing, err := op.GetSharingById(shareID, true); err == nil && sharing != nil { + valid = sharing.Valid() + } + status[shareID] = valid + } + } + return status +} + +// applyChannelUpdate reflects successful saves as channels and failed saves as missing channels. +func applyChannelUpdate(seed *torrent.Seed, driverName, mountPath string, results []gin.H) { + if driverName == "" { + return + } + successByPath := make(map[string]bool) + for _, res := range results { + filePath, _ := res["path"].(string) + if filePath == "" { + continue + } + method, _ := res["method"].(string) + successByPath[filePath] = method != "unavailable" + } + anySuccess := false + for _, ok := range successByPath { + if ok { + anySuccess = true + break + } + } + if anySuccess { + exists := false + for _, channel := range seed.Channels { + if channel.Driver == driverName { + exists = true + break + } + } + if !exists { + seed.Channels = append(seed.Channels, torrent.SeedChannel{Driver: driverName, MountPath: mountPath}) + } + } + for i := range seed.Files { + file := &seed.Files[i] + ok, tracked := successByPath[file.Path] + if !tracked { + continue + } + if ok { + file.MissingChannels = removeString(file.MissingChannels, driverName) + } else if !containsString(file.MissingChannels, driverName) { + file.MissingChannels = append(file.MissingChannels, driverName) + } + } +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func removeString(values []string, target string) []string { + result := values[:0] + for _, value := range values { + if value != target { + result = append(result, value) + } + } + return result +} + +// QuickSaveSeed imports selected files without downloading when the target supports it. +func QuickSaveSeed(c *gin.Context) { + user := c.Request.Context().Value(conf.UserKey).(*model.User) + var req SeedQuickSaveReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + _, seed, format, err := decodeSeedData(req.SeedDataReq) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + rapidOnly := strings.HasSuffix(c.FullPath(), "/rapid_upload") + if transitPath := resolveSeedTransitPath(req); transitPath != "" { + transferSeedViaTransit(c, user, seed, req, transitPath) + return + } + results, driverName, mountPath, err := saveSeedFilesToPath(c, user, seed, req, req.Path, rapidOnly) + if err != nil { + common.ErrorResp(c, err, seedSaveErrorCode(err)) + return + } + resp := gin.H{"results": results} + if req.UpdateChannel { + applyChannelUpdate(seed, driverName, mountPath, results) + if data, encodeErr := torrent.EncodeSeed(seed, format); encodeErr == nil { + resp["seed_data"] = base64.StdEncoding.EncodeToString(data) + resp["seed"] = seed + } + } + common.SuccessResp(c, resp) +} + +func firstUsableSeedSource(file torrent.SeedFile) string { + configuredSite, err := url.Parse(strings.TrimSpace(setting.GetStr(conf.SeedSiteURL))) + if err != nil || configuredSite.Scheme == "" || configuredSite.Hostname() == "" { + return "" + } + for _, source := range file.Sources { + candidate, parseErr := url.Parse(source.URL) + if parseErr != nil || (candidate.Scheme != "http" && candidate.Scheme != "https") || candidate.User != nil { + continue + } + if !strings.EqualFold(candidate.Scheme, configuredSite.Scheme) || !strings.EqualFold(candidate.Host, configuredSite.Host) { + continue + } + validPath := (source.Type == "openlist-direct" && strings.HasPrefix(candidate.EscapedPath(), "/d/")) || + (source.Type == "openlist-share" && strings.HasPrefix(candidate.EscapedPath(), "/sd/")) + if !validPath { + continue + } + if source.ExpiresAt != "" { + expires, parseErr := time.Parse(time.RFC3339, source.ExpiresAt) + if parseErr != nil || time.Now().After(expires) { + continue + } + } + return candidate.String() + } + return "" +} + +func seedTotalSize(seed *torrent.Seed) int64 { + var total int64 + for _, file := range seed.Files { + total += file.Size + } + return total +} + +func seedDiagnostics(seed *torrent.Seed) gin.H { + return gin.H{ + "oss": torrent.DiagnoseConversion(seed, "oss"), + "torrent": torrent.DiagnoseConversion(seed, "torrent"), + "cas": torrent.DiagnoseConversion(seed, "cas"), + } +} + +func seedConversionStates(seed *torrent.Seed) gin.H { + states := make(gin.H, 3) + for _, format := range []string{"oss", "torrent", "cas"} { + missing := torrent.DiagnoseConversion(seed, format) + states[format] = gin.H{"feasible": len(missing) == 0, "missing": missing} + } + return states +} diff --git a/server/router.go b/server/router.go index 4be0dd84a9..5fae18c0b0 100644 --- a/server/router.go +++ b/server/router.go @@ -234,6 +234,19 @@ func _fs(g *gin.RouterGroup) { g.POST("/torrent/upload_parse", handles.UploadTorrentAndParse) g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload) g.POST("/torrent/generate", handles.GenerateTorrentForPath) + // Unified transfer seed APIs. Legacy torrent routes above remain supported. + seed := g.Group("/seed") + seed.POST("/parse", handles.ParseSeed) + seed.POST("/upload_parse", handles.UploadSeedAndParse) + seed.POST("/generate", handles.GenerateSeedForPaths) + seed.POST("/convert", handles.ConvertSeed) + seed.POST("/diagnose", handles.DiagnoseSeed) + seed.POST("/capabilities", handles.SeedCapabilities) + seed.POST("/rapid_upload", handles.QuickSaveSeed) + seed.POST("/offline_download", handles.QuickSaveSeed) + seed.POST("/update", handles.UpdateSeed) + seed.POST("/quick_save", handles.QuickSaveSeed) + seed.POST("/update_channels", handles.UpdateSeedChannels) // Direct upload (client-side upload to storage) g.POST("/get_direct_upload_info", handles.FsGetDirectUploadInfo) }