diff --git a/internal/command/album.go b/internal/command/album.go index 9339e04..6695f19 100644 --- a/internal/command/album.go +++ b/internal/command/album.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -21,10 +21,19 @@ import ( "github.com/tickstep/aliyunpan/cmder" "github.com/tickstep/aliyunpan/cmder/cmdtable" "github.com/tickstep/aliyunpan/internal/config" + "github.com/tickstep/aliyunpan/internal/file/downloader" + "github.com/tickstep/aliyunpan/internal/functions/pandownload" + "github.com/tickstep/aliyunpan/internal/taskframework" + "github.com/tickstep/aliyunpan/internal/utils" + "github.com/tickstep/aliyunpan/library/requester/transfer" + "github.com/tickstep/library-go/converter" "github.com/tickstep/library-go/logger" + "github.com/tickstep/library-go/requester/rio/speeds" "github.com/urfave/cli" "os" + "path/filepath" "strconv" + "sync/atomic" "time" ) @@ -224,6 +233,70 @@ func CmdAlbum() cli.Command { }, Flags: []cli.Flag{}, }, + { + Name: "download-file", + Aliases: []string{"df"}, + Usage: "下载相簿中的所有文件到本地", + UsageText: cmder.App().Name + " album download-file", + Description: ` +下载相簿中的所有文件 +示例: + + 下载相簿 "我的相簿2022" 里面的所有文件 + aliyunpan album download-file 我的相簿2022 + +`, + Action: func(c *cli.Context) error { + if config.Config.ActiveUser() == nil { + fmt.Println("未登录账号") + return nil + } + subArgs := c.Args() + if len(subArgs) == 0 { + fmt.Println("请指定下载的相簿名称") + return nil + } + + // 处理saveTo + var ( + saveTo string + ) + if c.String("saveto") != "" { + saveTo = filepath.Clean(c.String("saveto")) + } + + do := &DownloadOptions{ + IsPrintStatus: false, + IsExecutedPermission: false, + IsOverwrite: c.Bool("ow"), + SaveTo: saveTo, + Parallel: 0, + Load: 0, + MaxRetry: pandownload.DefaultDownloadMaxRetry, + NoCheck: false, + ShowProgress: !c.Bool("np"), + DriveId: parseDriveId(c), + ExcludeNames: []string{}, + } + + RunAlbumDownloadFile(c.Args(), do) + return nil + }, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "ow", + Usage: "overwrite, 覆盖已存在的文件", + }, + cli.StringFlag{ + Name: "saveto", + Usage: "将下载的文件直接保存到指定的目录", + }, + cli.BoolFlag{ + Name: "np", + Usage: "no progress 不展示下载进度条", + }, + }, + }, }, } } @@ -503,3 +576,178 @@ func isFileMatchCondition(fileInfo *aliyunpan.FileEntity, filterOption AlbumFile } return false } + +func RunAlbumDownloadFile(albumNames []string, options *DownloadOptions) { + if len(albumNames) == 0 { + fmt.Printf("相簿名称不能为空\n") + return + } + + activeUser := GetActiveUser() + activeUser.PanClient().EnableCache() + activeUser.PanClient().ClearCache() + defer activeUser.PanClient().DisableCache() + // pan token expired checker + continueFlag := int32(0) + atomic.StoreInt32(&continueFlag, 0) + defer func() { + atomic.StoreInt32(&continueFlag, 1) + }() + go func(flag *int32) { + for atomic.LoadInt32(flag) == 0 { + time.Sleep(time.Duration(1) * time.Minute) + if RefreshTokenInNeed(activeUser, config.Config.DeviceName) { + logger.Verboseln("update access token for download task") + } + } + }(&continueFlag) + + if options == nil { + options = &DownloadOptions{} + } + + if options.MaxRetry < 0 { + options.MaxRetry = pandownload.DefaultDownloadMaxRetry + } + options.IsExecutedPermission = false + + // 设置下载配置 + cfg := &downloader.Config{ + Mode: transfer.RangeGenMode_BlockSize, + CacheSize: config.Config.CacheSize, + BlockSize: MaxDownloadRangeSize, + MaxRate: config.Config.MaxDownloadRate, + InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON, + ShowProgress: options.ShowProgress, + UseInternalUrl: config.Config.TransferUrlType == 2, + ExcludeNames: options.ExcludeNames, + } + if cfg.CacheSize == 0 { + cfg.CacheSize = int(DownloadCacheSize) + } + + // 设置下载最大并发量 + if options.Parallel < 1 { + options.Parallel = config.Config.MaxDownloadParallel + if options.Parallel == 0 { + options.Parallel = config.DefaultFileDownloadParallelNum + } + } + if options.Parallel > config.MaxFileDownloadParallelNum { + options.Parallel = config.MaxFileDownloadParallelNum + } + + // 保存文件的本地根文件夹 + originSaveRootPath := "" + if options.SaveTo != "" { + originSaveRootPath = options.SaveTo + } else { + // 使用默认的保存路径 + originSaveRootPath = GetActiveUser().GetSavePath("") + } + fi, err1 := os.Stat(originSaveRootPath) + if err1 != nil && !os.IsExist(err1) { + os.MkdirAll(originSaveRootPath, 0777) // 首先在本地创建目录 + } else { + if !fi.IsDir() { + fmt.Println("本地保存路径不是文件夹,请删除或者创建对应的文件夹:", originSaveRootPath) + return + } + } + + fmt.Printf("\n[0] 当前文件下载最大并发量为: %d, 下载缓存为: %s\n\n", options.Parallel, converter.ConvertFileSize(int64(cfg.CacheSize), 2)) + + var ( + panClient = activeUser.PanClient() + ) + cfg.MaxParallel = options.Parallel + + var ( + executor = taskframework.TaskExecutor{ + IsFailedDeque: true, // 统计失败的列表 + } + statistic = &pandownload.DownloadStatistic{} + ) + // 配置执行器任务并发数,即同时下载文件并发数 + executor.SetParallel(cfg.MaxParallel) + + // 全局速度统计 + globalSpeedsStat := &speeds.Speeds{} + + // 处理队列 + for k := range albumNames { + record := getAlbumFromName(activeUser, albumNames[k]) + if record == nil { + continue + } + // 获取相簿下的所有文件 + fileList, er := activeUser.PanClient().AlbumListFileGetAll(&aliyunpan.AlbumListFileParam{ + AlbumId: record.AlbumId, + }) + if er != nil { + fmt.Printf("获取相簿文件出错,请稍后重试: %s\n", albumNames[k]) + continue + } + if fileList == nil || len(fileList) == 0 { + fmt.Printf("相簿里面没有文件: %s\n", albumNames[k]) + continue + } + for _, f := range fileList { + // 补全虚拟网盘路径,规则:/<相簿名称>/文件名称 + f.Path = "/" + albumNames[k] + "/" + f.FileName + + // 生成下载项 + newCfg := *cfg + unit := pandownload.DownloadTaskUnit{ + Cfg: &newCfg, // 复制一份新的cfg + PanClient: panClient, + VerbosePrinter: panCommandVerbose, + PrintFormat: downloadPrintFormat(options.Load), + ParentTaskExecutor: &executor, + DownloadStatistic: statistic, + IsPrintStatus: options.IsPrintStatus, + IsExecutedPermission: options.IsExecutedPermission, + IsOverwrite: options.IsOverwrite, + NoCheck: options.NoCheck, + FilePanPath: f.Path, + DriveId: f.DriveId, // 必须使用文件的DriveId,因为一个相簿的文件会来自多个网盘(资源库/备份盘) + GlobalSpeedsStat: globalSpeedsStat, + FileRecorder: nil, + } + // 设置相簿文件信息 + unit.SetFileInfo(pandownload.AlbumFileSource, f) + + // 设置储存的路径 + if options.SaveTo != "" { + unit.OriginSaveRootPath = options.SaveTo + unit.SavePath = filepath.Join(options.SaveTo, f.Path) + } else { + // 使用默认的保存路径 + unit.OriginSaveRootPath = GetActiveUser().GetSavePath("") + unit.SavePath = GetActiveUser().GetSavePath(f.Path) + } + info := executor.Append(&unit, options.MaxRetry) + fmt.Printf("[%s] 加入下载队列: %s\n", info.Id(), f.Path) + } + } + + // 开始计时 + statistic.StartTimer() + + // 开始执行 + executor.Execute() + + fmt.Printf("\n下载结束, 时间: %s, 数据总量: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize(), 2)) + + // 输出失败的文件列表 + failedList := executor.FailedDeque() + if failedList.Size() != 0 { + fmt.Printf("以下文件下载失败: \n") + tb := cmdtable.NewTable(os.Stdout) + for e := failedList.Shift(); e != nil; e = failedList.Shift() { + item := e.(*taskframework.TaskInfoItem) + tb.Append([]string{item.Info.Id(), item.Unit.(*pandownload.DownloadTaskUnit).FilePanPath}) + } + tb.Render() + } +} diff --git a/internal/functions/pandownload/download_task_unit.go b/internal/functions/pandownload/download_task_unit.go index ae69cfe..2d2a59c 100644 --- a/internal/functions/pandownload/download_task_unit.go +++ b/internal/functions/pandownload/download_task_unit.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -43,6 +43,8 @@ import ( ) type ( + FileSourceType string + // DownloadTaskUnit 下载的任务单元 DownloadTaskUnit struct { taskInfo *taskframework.TaskInfo // 任务信息 @@ -67,7 +69,8 @@ type ( OriginSaveRootPath string // 文件保存在本地的根目录路径 DriveId string - fileInfo *aliyunpan.FileEntity // 文件或目录详情 + fileInfo *aliyunpan.FileEntity // 文件或目录详情 + downloadFileSource FileSourceType // 下载项类型,File-普通文件(备份盘/资源库),Album-相册文件 // 下载文件记录器 FileRecorder *log.FileRecorder @@ -89,8 +92,20 @@ const ( StrDownloadChecksumFailed = "检测文件有效性失败" // DefaultDownloadMaxRetry 默认下载失败最大重试次数 DefaultDownloadMaxRetry = 3 + + // BackupFileSource 备份盘文件 + BackupFileSource FileSourceType = "backup" + // ResourceFileSource 资源库文件 + ResourceFileSource FileSourceType = "resource" + // AlbumFileSource 相册文件 + AlbumFileSource FileSourceType = "album" ) +func (dtu *DownloadTaskUnit) SetFileInfo(fileType FileSourceType, info *aliyunpan.FileEntity) { + dtu.downloadFileSource = fileType + dtu.fileInfo = info +} + func (dtu *DownloadTaskUnit) SetTaskInfo(info *taskframework.TaskInfo) { dtu.taskInfo = info } @@ -277,7 +292,7 @@ func (dtu *DownloadTaskUnit) download() (err error) { return nil } -//panHTTPClient 获取包含特定User-Agent的HTTPClient +// panHTTPClient 获取包含特定User-Agent的HTTPClient func (dtu *DownloadTaskUnit) panHTTPClient() (client *requester.HTTPClient) { client = requester.NewHTTPClient() client.CheckRedirect = func(req *http.Request, via []*http.Request) error { @@ -317,7 +332,7 @@ func (dtu *DownloadTaskUnit) handleError(result *taskframework.TaskUnitRunResult time.Sleep(1 * time.Second) } -//checkFileValid 检测文件有效性 +// checkFileValid 检测文件有效性 func (dtu *DownloadTaskUnit) checkFileValid(result *taskframework.TaskUnitRunResult) (ok bool) { if dtu.NoCheck { // 不检测文件有效性 @@ -378,12 +393,14 @@ func (dtu *DownloadTaskUnit) OnSuccess(lastRunResult *taskframework.TaskUnitRunR // 下载文件数据记录 if config.Config.FileRecordConfig == "1" { if dtu.fileInfo.IsFile() { - dtu.FileRecorder.Append(&log.FileRecordItem{ - Status: "成功", - TimeStr: utils.NowTimeStr(), - FileSize: dtu.fileInfo.FileSize, - FilePath: dtu.fileInfo.Path, - }) + if dtu.FileRecorder != nil { + dtu.FileRecorder.Append(&log.FileRecordItem{ + Status: "成功", + TimeStr: utils.NowTimeStr(), + FileSize: dtu.fileInfo.FileSize, + FilePath: dtu.fileInfo.Path, + }) + } } } } @@ -440,19 +457,26 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { result = &taskframework.TaskUnitRunResult{} // 获取文件信息 var apierr *apierror.ApiError - if dtu.fileInfo == nil || dtu.taskInfo.Retry() > 0 { - // 没有获取文件信息 - // 如果是动态添加的下载任务, 是会写入文件信息的 - // 如果该任务重试过, 则应该再获取一次文件信息 - dtu.fileInfo, apierr = dtu.PanClient.FileInfoByPath(dtu.DriveId, dtu.FilePanPath) - if apierr != nil { - // 如果不是未登录或文件不存在, 则不重试 - result.ResultMessage = "获取下载路径信息错误" - result.Err = apierr - dtu.handleError(result) - return + if dtu.downloadFileSource != AlbumFileSource { // 相簿文件信息是传递进来的,无法在这里获取 + if dtu.fileInfo == nil || dtu.taskInfo.Retry() > 0 { + // 没有获取文件信息 + // 如果是动态添加的下载任务, 是会写入文件信息的 + // 如果该任务重试过, 则应该再获取一次文件信息 + dtu.fileInfo, apierr = dtu.PanClient.FileInfoByPath(dtu.DriveId, dtu.FilePanPath) + if apierr != nil { + // 如果不是未登录或文件不存在, 则不重试 + result.ResultMessage = "获取下载路径信息错误" + result.Err = apierr + dtu.handleError(result) + return + } + time.Sleep(1 * time.Second) + } + } else { + if dtu.taskInfo.Retry() > 0 { + // 延时避免风控 + time.Sleep(2 * time.Second) } - time.Sleep(1 * time.Second) } // 输出文件信息 @@ -620,6 +644,14 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { return result } + // 文件下载成功,更改文件修改时间 + if dtu.downloadFileSource == AlbumFileSource { + // 只有相册文件才需要更改时间 + if err := os.Chtimes(dtu.SavePath, utils.ParseTimeStr(dtu.fileInfo.CreatedAt), utils.ParseTimeStr(dtu.fileInfo.CreatedAt)); err != nil { + logger.Verbosef(err.Error()) + } + } + // 统计下载 dtu.DownloadStatistic.AddTotalSize(dtu.fileInfo.FileSize) // 下载成功