mirror of
https://github.com/tickstep/aliyunpan.git
synced 2025-01-23 14:32:14 +08:00
add album download command
This commit is contained in:
xiaoyaofenfen
parent
b559f41e47
commit
19afa16bfe
@ -4,7 +4,7 @@
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@ -21,10 +21,19 @@ import (
|
||||
"github.com/tickstep/aliyunpan/cmder"
|
||||
"github.com/tickstep/aliyunpan/cmder/cmdtable"
|
||||
"github.com/tickstep/aliyunpan/internal/config"
|
||||
"github.com/tickstep/aliyunpan/internal/file/downloader"
|
||||
"github.com/tickstep/aliyunpan/internal/functions/pandownload"
|
||||
"github.com/tickstep/aliyunpan/internal/taskframework"
|
||||
"github.com/tickstep/aliyunpan/internal/utils"
|
||||
"github.com/tickstep/aliyunpan/library/requester/transfer"
|
||||
"github.com/tickstep/library-go/converter"
|
||||
"github.com/tickstep/library-go/logger"
|
||||
"github.com/tickstep/library-go/requester/rio/speeds"
|
||||
"github.com/urfave/cli"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -224,6 +233,70 @@ func CmdAlbum() cli.Command {
|
||||
},
|
||||
Flags: []cli.Flag{},
|
||||
},
|
||||
{
|
||||
Name: "download-file",
|
||||
Aliases: []string{"df"},
|
||||
Usage: "下载相簿中的所有文件到本地",
|
||||
UsageText: cmder.App().Name + " album download-file",
|
||||
Description: `
|
||||
下载相簿中的所有文件
|
||||
示例:
|
||||
|
||||
下载相簿 "我的相簿2022" 里面的所有文件
|
||||
aliyunpan album download-file 我的相簿2022
|
||||
|
||||
`,
|
||||
Action: func(c *cli.Context) error {
|
||||
if config.Config.ActiveUser() == nil {
|
||||
fmt.Println("未登录账号")
|
||||
return nil
|
||||
}
|
||||
subArgs := c.Args()
|
||||
if len(subArgs) == 0 {
|
||||
fmt.Println("请指定下载的相簿名称")
|
||||
return nil
|
||||
}
|
||||
|
||||
// 处理saveTo
|
||||
var (
|
||||
saveTo string
|
||||
)
|
||||
if c.String("saveto") != "" {
|
||||
saveTo = filepath.Clean(c.String("saveto"))
|
||||
}
|
||||
|
||||
do := &DownloadOptions{
|
||||
IsPrintStatus: false,
|
||||
IsExecutedPermission: false,
|
||||
IsOverwrite: c.Bool("ow"),
|
||||
SaveTo: saveTo,
|
||||
Parallel: 0,
|
||||
Load: 0,
|
||||
MaxRetry: pandownload.DefaultDownloadMaxRetry,
|
||||
NoCheck: false,
|
||||
ShowProgress: !c.Bool("np"),
|
||||
DriveId: parseDriveId(c),
|
||||
ExcludeNames: []string{},
|
||||
}
|
||||
|
||||
RunAlbumDownloadFile(c.Args(), do)
|
||||
return nil
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
cli.BoolFlag{
|
||||
Name: "ow",
|
||||
Usage: "overwrite, 覆盖已存在的文件",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "saveto",
|
||||
Usage: "将下载的文件直接保存到指定的目录",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "np",
|
||||
Usage: "no progress 不展示下载进度条",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -503,3 +576,178 @@ func isFileMatchCondition(fileInfo *aliyunpan.FileEntity, filterOption AlbumFile
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func RunAlbumDownloadFile(albumNames []string, options *DownloadOptions) {
|
||||
if len(albumNames) == 0 {
|
||||
fmt.Printf("相簿名称不能为空\n")
|
||||
return
|
||||
}
|
||||
|
||||
activeUser := GetActiveUser()
|
||||
activeUser.PanClient().EnableCache()
|
||||
activeUser.PanClient().ClearCache()
|
||||
defer activeUser.PanClient().DisableCache()
|
||||
// pan token expired checker
|
||||
continueFlag := int32(0)
|
||||
atomic.StoreInt32(&continueFlag, 0)
|
||||
defer func() {
|
||||
atomic.StoreInt32(&continueFlag, 1)
|
||||
}()
|
||||
go func(flag *int32) {
|
||||
for atomic.LoadInt32(flag) == 0 {
|
||||
time.Sleep(time.Duration(1) * time.Minute)
|
||||
if RefreshTokenInNeed(activeUser, config.Config.DeviceName) {
|
||||
logger.Verboseln("update access token for download task")
|
||||
}
|
||||
}
|
||||
}(&continueFlag)
|
||||
|
||||
if options == nil {
|
||||
options = &DownloadOptions{}
|
||||
}
|
||||
|
||||
if options.MaxRetry < 0 {
|
||||
options.MaxRetry = pandownload.DefaultDownloadMaxRetry
|
||||
}
|
||||
options.IsExecutedPermission = false
|
||||
|
||||
// 设置下载配置
|
||||
cfg := &downloader.Config{
|
||||
Mode: transfer.RangeGenMode_BlockSize,
|
||||
CacheSize: config.Config.CacheSize,
|
||||
BlockSize: MaxDownloadRangeSize,
|
||||
MaxRate: config.Config.MaxDownloadRate,
|
||||
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
|
||||
ShowProgress: options.ShowProgress,
|
||||
UseInternalUrl: config.Config.TransferUrlType == 2,
|
||||
ExcludeNames: options.ExcludeNames,
|
||||
}
|
||||
if cfg.CacheSize == 0 {
|
||||
cfg.CacheSize = int(DownloadCacheSize)
|
||||
}
|
||||
|
||||
// 设置下载最大并发量
|
||||
if options.Parallel < 1 {
|
||||
options.Parallel = config.Config.MaxDownloadParallel
|
||||
if options.Parallel == 0 {
|
||||
options.Parallel = config.DefaultFileDownloadParallelNum
|
||||
}
|
||||
}
|
||||
if options.Parallel > config.MaxFileDownloadParallelNum {
|
||||
options.Parallel = config.MaxFileDownloadParallelNum
|
||||
}
|
||||
|
||||
// 保存文件的本地根文件夹
|
||||
originSaveRootPath := ""
|
||||
if options.SaveTo != "" {
|
||||
originSaveRootPath = options.SaveTo
|
||||
} else {
|
||||
// 使用默认的保存路径
|
||||
originSaveRootPath = GetActiveUser().GetSavePath("")
|
||||
}
|
||||
fi, err1 := os.Stat(originSaveRootPath)
|
||||
if err1 != nil && !os.IsExist(err1) {
|
||||
os.MkdirAll(originSaveRootPath, 0777) // 首先在本地创建目录
|
||||
} else {
|
||||
if !fi.IsDir() {
|
||||
fmt.Println("本地保存路径不是文件夹，请删除或者创建对应的文件夹：", originSaveRootPath)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n[0] 当前文件下载最大并发量为: %d, 下载缓存为: %s\n\n", options.Parallel, converter.ConvertFileSize(int64(cfg.CacheSize), 2))
|
||||
|
||||
var (
|
||||
panClient = activeUser.PanClient()
|
||||
)
|
||||
cfg.MaxParallel = options.Parallel
|
||||
|
||||
var (
|
||||
executor = taskframework.TaskExecutor{
|
||||
IsFailedDeque: true, // 统计失败的列表
|
||||
}
|
||||
statistic = &pandownload.DownloadStatistic{}
|
||||
)
|
||||
// 配置执行器任务并发数，即同时下载文件并发数
|
||||
executor.SetParallel(cfg.MaxParallel)
|
||||
|
||||
// 全局速度统计
|
||||
globalSpeedsStat := &speeds.Speeds{}
|
||||
|
||||
// 处理队列
|
||||
for k := range albumNames {
|
||||
record := getAlbumFromName(activeUser, albumNames[k])
|
||||
if record == nil {
|
||||
continue
|
||||
}
|
||||
// 获取相簿下的所有文件
|
||||
fileList, er := activeUser.PanClient().AlbumListFileGetAll(&aliyunpan.AlbumListFileParam{
|
||||
AlbumId: record.AlbumId,
|
||||
})
|
||||
if er != nil {
|
||||
fmt.Printf("获取相簿文件出错，请稍后重试: %s\n", albumNames[k])
|
||||
continue
|
||||
}
|
||||
if fileList == nil || len(fileList) == 0 {
|
||||
fmt.Printf("相簿里面没有文件: %s\n", albumNames[k])
|
||||
continue
|
||||
}
|
||||
for _, f := range fileList {
|
||||
// 补全虚拟网盘路径，规则：/<相簿名称>/文件名称
|
||||
f.Path = "/" + albumNames[k] + "/" + f.FileName
|
||||
|
||||
// 生成下载项
|
||||
newCfg := *cfg
|
||||
unit := pandownload.DownloadTaskUnit{
|
||||
Cfg: &newCfg, // 复制一份新的cfg
|
||||
PanClient: panClient,
|
||||
VerbosePrinter: panCommandVerbose,
|
||||
PrintFormat: downloadPrintFormat(options.Load),
|
||||
ParentTaskExecutor: &executor,
|
||||
DownloadStatistic: statistic,
|
||||
IsPrintStatus: options.IsPrintStatus,
|
||||
IsExecutedPermission: options.IsExecutedPermission,
|
||||
IsOverwrite: options.IsOverwrite,
|
||||
NoCheck: options.NoCheck,
|
||||
FilePanPath: f.Path,
|
||||
DriveId: f.DriveId, // 必须使用文件的DriveId,因为一个相簿的文件会来自多个网盘（资源库/备份盘）
|
||||
GlobalSpeedsStat: globalSpeedsStat,
|
||||
FileRecorder: nil,
|
||||
}
|
||||
// 设置相簿文件信息
|
||||
unit.SetFileInfo(pandownload.AlbumFileSource, f)
|
||||
|
||||
// 设置储存的路径
|
||||
if options.SaveTo != "" {
|
||||
unit.OriginSaveRootPath = options.SaveTo
|
||||
unit.SavePath = filepath.Join(options.SaveTo, f.Path)
|
||||
} else {
|
||||
// 使用默认的保存路径
|
||||
unit.OriginSaveRootPath = GetActiveUser().GetSavePath("")
|
||||
unit.SavePath = GetActiveUser().GetSavePath(f.Path)
|
||||
}
|
||||
info := executor.Append(&unit, options.MaxRetry)
|
||||
fmt.Printf("[%s] 加入下载队列: %s\n", info.Id(), f.Path)
|
||||
}
|
||||
}
|
||||
|
||||
// 开始计时
|
||||
statistic.StartTimer()
|
||||
|
||||
// 开始执行
|
||||
executor.Execute()
|
||||
|
||||
fmt.Printf("\n下载结束, 时间: %s, 数据总量: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize(), 2))
|
||||
|
||||
// 输出失败的文件列表
|
||||
failedList := executor.FailedDeque()
|
||||
if failedList.Size() != 0 {
|
||||
fmt.Printf("以下文件下载失败: \n")
|
||||
tb := cmdtable.NewTable(os.Stdout)
|
||||
for e := failedList.Shift(); e != nil; e = failedList.Shift() {
|
||||
item := e.(*taskframework.TaskInfoItem)
|
||||
tb.Append([]string{item.Info.Id(), item.Unit.(*pandownload.DownloadTaskUnit).FilePanPath})
|
||||
}
|
||||
tb.Render()
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@ -43,6 +43,8 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
FileSourceType string
|
||||
|
||||
// DownloadTaskUnit 下载的任务单元
|
||||
DownloadTaskUnit struct {
|
||||
taskInfo *taskframework.TaskInfo // 任务信息
|
||||
@ -67,7 +69,8 @@ type (
|
||||
OriginSaveRootPath string // 文件保存在本地的根目录路径
|
||||
DriveId string
|
||||
|
||||
fileInfo *aliyunpan.FileEntity // 文件或目录详情
|
||||
fileInfo *aliyunpan.FileEntity // 文件或目录详情
|
||||
downloadFileSource FileSourceType // 下载项类型，File-普通文件（备份盘/资源库），Album-相册文件
|
||||
|
||||
// 下载文件记录器
|
||||
FileRecorder *log.FileRecorder
|
||||
@ -89,8 +92,20 @@ const (
|
||||
StrDownloadChecksumFailed = "检测文件有效性失败"
|
||||
// DefaultDownloadMaxRetry 默认下载失败最大重试次数
|
||||
DefaultDownloadMaxRetry = 3
|
||||
|
||||
// BackupFileSource 备份盘文件
|
||||
BackupFileSource FileSourceType = "backup"
|
||||
// ResourceFileSource 资源库文件
|
||||
ResourceFileSource FileSourceType = "resource"
|
||||
// AlbumFileSource 相册文件
|
||||
AlbumFileSource FileSourceType = "album"
|
||||
)
|
||||
|
||||
func (dtu *DownloadTaskUnit) SetFileInfo(fileType FileSourceType, info *aliyunpan.FileEntity) {
|
||||
dtu.downloadFileSource = fileType
|
||||
dtu.fileInfo = info
|
||||
}
|
||||
|
||||
func (dtu *DownloadTaskUnit) SetTaskInfo(info *taskframework.TaskInfo) {
|
||||
dtu.taskInfo = info
|
||||
}
|
||||
@ -277,7 +292,7 @@ func (dtu *DownloadTaskUnit) download() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
//panHTTPClient 获取包含特定User-Agent的HTTPClient
|
||||
// panHTTPClient 获取包含特定User-Agent的HTTPClient
|
||||
func (dtu *DownloadTaskUnit) panHTTPClient() (client *requester.HTTPClient) {
|
||||
client = requester.NewHTTPClient()
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
@ -317,7 +332,7 @@ func (dtu *DownloadTaskUnit) handleError(result *taskframework.TaskUnitRunResult
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
//checkFileValid 检测文件有效性
|
||||
// checkFileValid 检测文件有效性
|
||||
func (dtu *DownloadTaskUnit) checkFileValid(result *taskframework.TaskUnitRunResult) (ok bool) {
|
||||
if dtu.NoCheck {
|
||||
// 不检测文件有效性
|
||||
@ -378,12 +393,14 @@ func (dtu *DownloadTaskUnit) OnSuccess(lastRunResult *taskframework.TaskUnitRunR
|
||||
// 下载文件数据记录
|
||||
if config.Config.FileRecordConfig == "1" {
|
||||
if dtu.fileInfo.IsFile() {
|
||||
dtu.FileRecorder.Append(&log.FileRecordItem{
|
||||
Status: "成功",
|
||||
TimeStr: utils.NowTimeStr(),
|
||||
FileSize: dtu.fileInfo.FileSize,
|
||||
FilePath: dtu.fileInfo.Path,
|
||||
})
|
||||
if dtu.FileRecorder != nil {
|
||||
dtu.FileRecorder.Append(&log.FileRecordItem{
|
||||
Status: "成功",
|
||||
TimeStr: utils.NowTimeStr(),
|
||||
FileSize: dtu.fileInfo.FileSize,
|
||||
FilePath: dtu.fileInfo.Path,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -440,19 +457,26 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
|
||||
result = &taskframework.TaskUnitRunResult{}
|
||||
// 获取文件信息
|
||||
var apierr *apierror.ApiError
|
||||
if dtu.fileInfo == nil || dtu.taskInfo.Retry() > 0 {
|
||||
// 没有获取文件信息
|
||||
// 如果是动态添加的下载任务, 是会写入文件信息的
|
||||
// 如果该任务重试过, 则应该再获取一次文件信息
|
||||
dtu.fileInfo, apierr = dtu.PanClient.FileInfoByPath(dtu.DriveId, dtu.FilePanPath)
|
||||
if apierr != nil {
|
||||
// 如果不是未登录或文件不存在, 则不重试
|
||||
result.ResultMessage = "获取下载路径信息错误"
|
||||
result.Err = apierr
|
||||
dtu.handleError(result)
|
||||
return
|
||||
if dtu.downloadFileSource != AlbumFileSource { // 相簿文件信息是传递进来的，无法在这里获取
|
||||
if dtu.fileInfo == nil || dtu.taskInfo.Retry() > 0 {
|
||||
// 没有获取文件信息
|
||||
// 如果是动态添加的下载任务, 是会写入文件信息的
|
||||
// 如果该任务重试过, 则应该再获取一次文件信息
|
||||
dtu.fileInfo, apierr = dtu.PanClient.FileInfoByPath(dtu.DriveId, dtu.FilePanPath)
|
||||
if apierr != nil {
|
||||
// 如果不是未登录或文件不存在, 则不重试
|
||||
result.ResultMessage = "获取下载路径信息错误"
|
||||
result.Err = apierr
|
||||
dtu.handleError(result)
|
||||
return
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
} else {
|
||||
if dtu.taskInfo.Retry() > 0 {
|
||||
// 延时避免风控
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// 输出文件信息
|
||||
@ -620,6 +644,14 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
|
||||
return result
|
||||
}
|
||||
|
||||
// 文件下载成功，更改文件修改时间
|
||||
if dtu.downloadFileSource == AlbumFileSource {
|
||||
// 只有相册文件才需要更改时间
|
||||
if err := os.Chtimes(dtu.SavePath, utils.ParseTimeStr(dtu.fileInfo.CreatedAt), utils.ParseTimeStr(dtu.fileInfo.CreatedAt)); err != nil {
|
||||
logger.Verbosef(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// 统计下载
|
||||
dtu.DownloadStatistic.AddTotalSize(dtu.fileInfo.FileSize)
|
||||
// 下载成功
|
||||
|
||||
Loading…
Reference in New Issue
Block a user