diff --git a/README.md b/README.md index 973f765..81d97e5 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,11 @@ aliyunpan ll /我的文档 ``` ## 下载文件/目录 +下载支持两种链接类型：1-默认类型 2-阿里ECS环境类型 +在普通网络下，下载速度可以达到10MB/s，在阿里ECS（必须是"经典网络"类型的机器）环境下，下载速度单文件可以轻松达到20MB/s，多文件可以达到100MB/s +![](./assets/images/download_file_ecs_speed_screenshot.gif) +![](./assets/images/download_file_speed_screenshot.gif) + ``` aliyunpan download <网盘文件或目录的路径1> <文件或目录2> <文件或目录3> ... aliyunpan d <网盘文件或目录的路径1> <文件或目录2> <文件或目录3> ... diff --git a/aliyunpan.exe.manifest b/aliyunpan.exe.manifest index 6fa433d..8444ba2 100644 --- a/aliyunpan.exe.manifest +++ b/aliyunpan.exe.manifest @@ -1,6 +1,6 @@ - + diff --git a/assets/images/download_file_ecs_speed_screenshot.gif b/assets/images/download_file_ecs_speed_screenshot.gif new file mode 100755 index 0000000..1c84ca4 Binary files /dev/null and b/assets/images/download_file_ecs_speed_screenshot.gif differ diff --git a/assets/images/download_file_speed_screenshot.gif b/assets/images/download_file_speed_screenshot.gif new file mode 100755 index 0000000..e6e824f Binary files /dev/null and b/assets/images/download_file_speed_screenshot.gif differ diff --git a/internal/command/download.go b/internal/command/download.go index febb97c..43f64f4 100644 --- a/internal/command/download.go +++ b/internal/command/download.go @@ -23,13 +23,16 @@ import ( "github.com/tickstep/aliyunpan/internal/file/downloader" "github.com/tickstep/aliyunpan/internal/functions/pandownload" "github.com/tickstep/aliyunpan/internal/taskframework" + "github.com/tickstep/aliyunpan/internal/utils" "github.com/tickstep/aliyunpan/library/requester/transfer" "github.com/tickstep/library-go/converter" + "github.com/tickstep/library-go/logger" "github.com/tickstep/library-go/requester/rio/speeds" "github.com/urfave/cli" "os" "path/filepath" "runtime" + "time" ) type ( @@ -181,6 +184,17 @@ func downloadPrintFormat(load int) string { // RunDownload 执行下载网盘内文件 func RunDownload(paths []string, options *DownloadOptions) { + activeUser := GetActiveUser() + // pan token expired checker + go func() { + for { + time.Sleep(time.Duration(1) * time.Minute) + if RefreshTokenInNeed(activeUser) { + logger.Verboseln("update access token for download task") + } + } + }() + if options == nil { options = &DownloadOptions{} } @@ -225,10 +239,12 @@ func RunDownload(paths []string, options *DownloadOptions) { fmt.Printf("\n[0] 当前文件下载最大并发量为: %d, 下载缓存为: %s\n", options.Parallel, converter.ConvertFileSize(int64(cfg.CacheSize), 2)) var ( - panClient = GetActivePanClient() + panClient = activeUser.PanClient() loadCount = 0 + loadSize = int64(0) ) + fmt.Printf("[0] 正在计算需要下载的文件数量和大小...\n") // 预测要下载的文件数量 for k := range paths { // 使用递归获取文件的方法计算路径包含的文件的总数量 @@ -241,11 +257,13 @@ func RunDownload(paths []string, options *DownloadOptions) { // 忽略统计文件夹数量 if !fd.IsFolder() { loadCount++ + loadSize += fd.FileSize } + time.Sleep(500 * time.Millisecond) return true }) } - fmt.Printf("[0] 预计总共需要下载的文件数量: %d\n", loadCount) + fmt.Printf("[0] 预计总共需要下载的文件数量: %d, 总大小：%s\n\n", loadCount, converter.ConvertFileSize(loadSize, 2)) cfg.MaxParallel = options.Parallel var ( @@ -298,7 +316,7 @@ func RunDownload(paths []string, options *DownloadOptions) { // 开始执行 executor.Execute() - fmt.Printf("\n下载结束, 时间: %s, 数据总量: %s\n", statistic.Elapsed()/1e6*1e6, converter.ConvertFileSize(statistic.TotalSize())) + fmt.Printf("\n下载结束, 时间: %s, 数据总量: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize(), 2)) // 输出失败的文件列表 failedList := executor.FailedDeque() diff --git a/internal/command/upload.go b/internal/command/upload.go index 0494358..b64ef73 100644 --- a/internal/command/upload.go +++ b/internal/command/upload.go @@ -234,18 +234,10 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { activeUser := GetActiveUser() // pan token expired checker go func() { - cz := time.FixedZone("CST", 8*3600) // 东8区 for { time.Sleep(time.Duration(1) * time.Minute) - expiredTime, _ := time.ParseInLocation("2006-01-02 15:04:05", activeUser.WebToken.ExpireTime, cz) - now := time.Now() - if (expiredTime.Unix() - now.Unix()) <= (10 * 60) { - // need refresh token - if wt, er := aliyunpan.GetAccessTokenFromRefreshToken(activeUser.RefreshToken); er == nil { - activeUser.WebToken = *wt - activeUser.PanClient().UpdateToken(*wt) - logger.Verboseln("update access token for upload task") - } + if RefreshTokenInNeed(activeUser) { + logger.Verboseln("update access token for upload task") } } }() @@ -269,6 +261,8 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { } opt.UseInternalUrl = config.Config.TransferUrlType == 2 + fmt.Printf("\n[0] 当前文件上传最大并发量为: %d, 上传分片大小为: %s\n", opt.AllParallel, converter.ConvertFileSize(opt.BlockSize, 2)) + savePath = activeUser.PathJoin(opt.DriveId, savePath) _, err1 := activeUser.PanClient().FileInfoByPath(opt.DriveId, savePath) if err1 != nil { @@ -438,7 +432,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { } fmt.Printf("\n") - fmt.Printf("上传结束, 时间: %s, 总大小: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize())) + fmt.Printf("上传结束, 时间: %s, 数据总量: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize(), 2)) // 输出上传失败的文件列表 for _, failed := range failedList { diff --git a/internal/command/utils.go b/internal/command/utils.go index 2759219..962d23d 100644 --- a/internal/command/utils.go +++ b/internal/command/utils.go @@ -116,4 +116,31 @@ func EscapeStr(s string) string { func UnescapeStr(s string) string { r,_ := url.PathUnescape(s) return r +} + +// RefreshTokenInNeed 刷新refresh token +func RefreshTokenInNeed(activeUser *config.PanUser) bool { + if activeUser == nil { + return false + } + + // refresh expired token + if activeUser.PanClient() != nil { + if len(activeUser.WebToken.RefreshToken) > 0 { + cz := time.FixedZone("CST", 8*3600) // 东8区 + expiredTime, _ := time.ParseInLocation("2006-01-02 15:04:05", activeUser.WebToken.ExpireTime, cz) + now := time.Now() + if (expiredTime.Unix() - now.Unix()) <= (10 * 60) { // 10min + // need update refresh token + logger.Verboseln("access token expired, get new from refresh token") + if wt, er := aliyunpan.GetAccessTokenFromRefreshToken(activeUser.RefreshToken); er == nil { + activeUser.WebToken = *wt + activeUser.PanClient().UpdateToken(*wt) + logger.Verboseln("get new access token success") + return true + } + } + } + } + return false } \ No newline at end of file diff --git a/internal/file/downloader/downloader.go b/internal/file/downloader/downloader.go index 715299a..9a5d187 100644 --- a/internal/file/downloader/downloader.go +++ b/internal/file/downloader/downloader.go @@ -20,12 +20,12 @@ import ( "github.com/tickstep/aliyunpan-api/aliyunpan/apierror" "github.com/tickstep/aliyunpan/cmder/cmdutil" "github.com/tickstep/aliyunpan/internal/waitgroup" + "github.com/tickstep/aliyunpan/library/requester/transfer" "github.com/tickstep/library-go/cachepool" "github.com/tickstep/library-go/logger" "github.com/tickstep/library-go/prealloc" "github.com/tickstep/library-go/requester" "github.com/tickstep/library-go/requester/rio/speeds" - "github.com/tickstep/aliyunpan/library/requester/transfer" "io" "net/http" "sync" @@ -42,6 +42,7 @@ type ( Downloader struct { onExecuteEvent requester.Event //开始下载事件 onSuccessEvent requester.Event //成功下载事件 + onFailedEvent requester.Event //成功下载事件 onFinishEvent requester.Event //结束下载事件 onPauseEvent requester.Event //暂停下载事件 onResumeEvent requester.Event //恢复下载事件 @@ -80,7 +81,6 @@ func NewDownloader(writer io.WriterAt, config *Config, p *aliyunpan.PanClient, g panClient: p, globalSpeedsStat: globalSpeedsStat, } - return } @@ -369,23 +369,34 @@ func (der *Downloader) Execute() error { var ( writeMu = &sync.Mutex{} ) + + // 获取下载链接 + var apierr *apierror.ApiError + durl, apierr := der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{ + DriveId: der.driveId, + FileId: der.fileInfo.FileId, + }) + time.Sleep(time.Duration(200) * time.Millisecond) + if apierr != nil { + logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId) + cmdutil.Trigger(der.onCancelEvent) + return apierr + } + if durl == nil || durl.Url == "" { + logger.Verbosef("无法获取有效的下载链接: %+v\n", durl) + cmdutil.Trigger(der.onCancelEvent) + der.removeInstanceState() // 移除断点续传文件 + cmdutil.Trigger(der.onFailedEvent) + return ErrFileDownloadForbidden + } + + // 初始化下载worker for k, r := range bii.Ranges { loadBalancer := loadBalancerResponseList.SequentialGet() if loadBalancer == nil { continue } - // 获取下载链接 - var apierr *apierror.ApiError - durl, apierr := der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{ - DriveId: der.driveId, - FileId: der.fileInfo.FileId, - }) - time.Sleep(time.Duration(200) * time.Millisecond) - if apierr != nil { - logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId) - continue - } logger.Verbosef("work id: %d, download url: %s\n", k, durl) client := requester.NewHTTPClient() client.SetKeepAlive(true) @@ -488,6 +499,15 @@ func (der *Downloader) Cancel() { cmdutil.Trigger(der.monitorCancelFunc) } +//Failed 失败 +func (der *Downloader) Failed() { + if der.monitor == nil { + return + } + cmdutil.Trigger(der.onFailedEvent) + cmdutil.Trigger(der.monitorCancelFunc) +} + //OnExecute 设置开始下载事件 func (der *Downloader) OnExecute(onExecuteEvent requester.Event) { der.onExecuteEvent = onExecuteEvent @@ -498,6 +518,11 @@ func (der *Downloader) OnSuccess(onSuccessEvent requester.Event) { der.onSuccessEvent = onSuccessEvent } +//OnFailed 设置失败事件 +func (der *Downloader) OnFailed(onFailedEvent requester.Event) { + der.onFailedEvent = onFailedEvent +} + //OnFinish 设置结束下载事件 func (der *Downloader) OnFinish(onFinishEvent requester.Event) { der.onFinishEvent = onFinishEvent diff --git a/internal/file/downloader/utils.go b/internal/file/downloader/utils.go index b0c538e..3bcfa91 100644 --- a/internal/file/downloader/utils.go +++ b/internal/file/downloader/utils.go @@ -14,6 +14,7 @@ package downloader import ( + "errors" "github.com/tickstep/library-go/logger" "github.com/tickstep/library-go/requester" mathrand "math/rand" @@ -34,6 +35,9 @@ var ( // ran 一个随机数实例 ran = mathrand.New(ranSource) + + // 文件被禁止下载 + ErrFileDownloadForbidden = errors.New("文件被禁止下载") ) // RandomNumber 生成指定区间随机数 diff --git a/internal/functions/pandownload/download_task_unit.go b/internal/functions/pandownload/download_task_unit.go index 6c3b5f9..bf36865 100644 --- a/internal/functions/pandownload/download_task_unit.go +++ b/internal/functions/pandownload/download_task_unit.go @@ -188,14 +188,21 @@ func (dtu *DownloadTaskUnit) download() (err error) { }) err = der.Execute() - isComplete = true - fmt.Print("\n") - if err != nil { // check zero size file if err == downloader.ErrNoWokers && dtu.fileInfo.FileSize == 0 { // success for 0 size file dtu.verboseInfof("download success for zero size file") + } else if err == downloader.ErrFileDownloadForbidden { + // 文件被禁止下载 + isComplete = false + // 删除本地文件 + removeErr := os.Remove(dtu.SavePath) + if removeErr != nil { + dtu.verboseInfof("[%s] remove file error: %s\n", dtu.taskInfo.Id(), removeErr) + } + fmt.Printf("[%s] 下载失败，文件不合法或者被禁止下载: %s\n", dtu.taskInfo.Id(), dtu.SavePath) + return err } else { // 下载发生错误 // 下载失败, 删去空文件 @@ -211,6 +218,8 @@ func (dtu *DownloadTaskUnit) download() (err error) { } return err } + } else { + isComplete = true } // 下载成功 @@ -253,9 +262,14 @@ func (dtu *DownloadTaskUnit) handleError(result *taskframework.TaskUnitRunResult // 系统级别的错误, 可能是权限问题 result.NeedRetry = false default: - // 其他错误, 需要重试 - result.NeedRetry = true + if result.Err == downloader.ErrFileDownloadForbidden { + result.NeedRetry = false + } else { + // 其他错误, 尝试重试 + result.NeedRetry = true + } } + time.Sleep(1*time.Second) } //checkFileValid 检测文件有效性 @@ -362,7 +376,10 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { } // 获取该目录下的文件列表 - fileList := dtu.PanClient.FilesDirectoriesRecurseList(dtu.DriveId, dtu.FilePanPath, nil) + fileList := dtu.PanClient.FilesDirectoriesRecurseList(dtu.DriveId, dtu.FilePanPath, func(depth int, _ string, fd *aliyunpan.FileEntity, apiError *apierror.ApiError) bool { + time.Sleep(500 * time.Millisecond) + return true + }) if fileList == nil { result.ResultMessage = "获取目录信息错误" result.Err = err diff --git a/internal/functions/panupload/upload_task_unit.go b/internal/functions/panupload/upload_task_unit.go index 5b83cce..ef03af6 100644 --- a/internal/functions/panupload/upload_task_unit.go +++ b/internal/functions/panupload/upload_task_unit.go @@ -15,6 +15,7 @@ package panupload import ( "fmt" + "github.com/tickstep/library-go/logger" "os" "path" "path/filepath" @@ -297,7 +298,7 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { }() // 准备文件 utu.prepareFile() - fmt.Printf("[%s] %s 准备结束, 准备耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart))) + logger.Verbosef("[%s] %s 准备结束, 准备耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart))) var apierr *apierror.ApiError var rs *aliyunpan.MkdirResult @@ -346,11 +347,11 @@ StepUploadPrepareUpload: rs = &aliyunpan.MkdirResult{FileId: test.FileId} } utu.FolderCreateMutex.Unlock() - fmt.Printf("[%s] %s 检测和创建云盘文件夹完毕[from db], 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart3))) + logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕[from db], 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart3))) } if rs == nil { timeStart4 = time.Now() - fmt.Printf("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath) + logger.Verbosef("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath) utu.FolderCreateMutex.Lock() // rs, apierr = utu.PanClient.MkdirRecursive(utu.DriveId, "", "", 0, strings.Split(path.Clean(saveFilePath), "/")) // 可以直接创建的，不用循环创建 @@ -361,7 +362,7 @@ StepUploadPrepareUpload: result.ResultMessage = "创建云盘文件夹失败" return } - fmt.Printf("[%s] %s 创建云盘文件夹, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart4))) + logger.Verbosef("[%s] %s 创建云盘文件夹, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart4))) } } else { rs = &aliyunpan.MkdirResult{} @@ -369,7 +370,7 @@ StepUploadPrepareUpload: } // time.Sleep(time.Duration(2) * time.Second) // utu.FolderCreateMutex.Unlock() - fmt.Printf("[%s] %s 检测和创建云盘文件夹完毕, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart2))) + logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart2))) sha1Str = "" proofCode = "" diff --git a/main.go b/main.go index cc979cc..ab0e3ed 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,6 @@ package main import ( "fmt" "github.com/olekukonko/tablewriter" - "github.com/tickstep/aliyunpan-api/aliyunpan" "github.com/tickstep/aliyunpan/cmder" "github.com/tickstep/aliyunpan/cmder/cmdtable" "io/ioutil" @@ -51,7 +50,7 @@ const ( var ( // Version 版本号 - Version = "v0.1.0" + Version = "v0.1.1" historyFilePath = filepath.Join(config.GetConfigDir(), "aliyunpan_command_history.txt") @@ -81,22 +80,7 @@ func checkLoginExpiredAndRelogin() { cmder.TryLogin() } else { // refresh expired token - if activeUser.PanClient() != nil { - if len(activeUser.WebToken.RefreshToken) > 0 { - cz := time.FixedZone("CST", 8*3600) // 东8区 - expiredTime, _ := time.ParseInLocation("2006-01-02 15:04:05", activeUser.WebToken.ExpireTime, cz) - now := time.Now() - if (expiredTime.Unix() - now.Unix()) <= (10 * 60) { - // need refresh token - logger.Verboseln("access token expired, get new from refresh token") - if wt, er := aliyunpan.GetAccessTokenFromRefreshToken(activeUser.RefreshToken); er == nil { - activeUser.WebToken = *wt - activeUser.PanClient().UpdateToken(*wt) - logger.Verboseln("get new access token success") - } - } - } - } + command.RefreshTokenInNeed(activeUser) } cmder.SaveConfigFunc(nil) } diff --git a/resource_windows_386.syso b/resource_windows_386.syso index 48b0cf2..a02c236 100644 Binary files a/resource_windows_386.syso and b/resource_windows_386.syso differ diff --git a/resource_windows_amd64.syso b/resource_windows_amd64.syso index ec70c81..12ec10d 100644 Binary files a/resource_windows_amd64.syso and b/resource_windows_amd64.syso differ diff --git a/versioninfo.json b/versioninfo.json index eb74d2a..740f001 100644 --- a/versioninfo.json +++ b/versioninfo.json @@ -3,13 +3,13 @@ "FileVersion": { "Major": 0, "Minor": 1, - "Patch": 0, + "Patch": 1, "Build": 0 }, "ProductVersion": { "Major": 0, "Minor": 1, - "Patch": 0, + "Patch": 1, "Build": 0 }, "FileFlagsMask": "3f", @@ -22,14 +22,14 @@ "Comments": "", "CompanyName": "tickstep", "FileDescription": "阿里云盘客户端", - "FileVersion": "v0.1.0", + "FileVersion": "v0.1.1", "InternalName": "", "LegalCopyright": "© 2021 tickstep.", "LegalTrademarks": "", "OriginalFilename": "", "PrivateBuild": "", "ProductName": "aliyunpan", - "ProductVersion": "v0.1.0", + "ProductVersion": "v0.1.1", "SpecialBuild": "" }, "VarFileInfo": {