Merge branch 'tickstep:main' into main

This commit is contained in:
Stam He 2022-01-16 21:23:55 +08:00 committed by GitHub
commit 645ef7f538
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 136 additions and 61 deletions

View File

@ -285,6 +285,11 @@ aliyunpan ll /我的文档
```
## 下载文件/目录
下载支持两种链接类型1-默认类型 2-阿里ECS环境类型
在普通网络下下载速度可以达到10MB/s在阿里ECS必须是"经典网络"类型的机器环境下下载速度单文件可以轻松达到20MB/s多文件可以达到100MB/s
![](./assets/images/download_file_ecs_speed_screenshot.gif)
![](./assets/images/download_file_speed_screenshot.gif)
```
aliyunpan download <网盘文件或目录的路径1> <文件或目录2> <文件或目录3> ...
aliyunpan d <网盘文件或目录的路径1> <文件或目录2> <文件或目录3> ...

View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0" xmlns:asmv3="urn:schemas-microsoft-com:asm.v3">
<assemblyIdentity version="0.1.0.0" processorArchitecture="*" name="com.tickstep.aliyunpan" type="win32"/>
<assemblyIdentity version="0.1.1.0" processorArchitecture="*" name="com.tickstep.aliyunpan" type="win32"/>
<dependency>
<dependentAssembly>
<assemblyIdentity type="win32" name="Microsoft.Windows.Common-Controls" version="6.0.0.0" processorArchitecture="*" publicKeyToken="6595b64144ccf1df" language="*"/>

Binary file not shown.

After

Width:  |  Height:  |  Size: 395 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 301 KiB

View File

@ -23,13 +23,16 @@ import (
"github.com/tickstep/aliyunpan/internal/file/downloader"
"github.com/tickstep/aliyunpan/internal/functions/pandownload"
"github.com/tickstep/aliyunpan/internal/taskframework"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/converter"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester/rio/speeds"
"github.com/urfave/cli"
"os"
"path/filepath"
"runtime"
"time"
)
type (
@ -181,6 +184,17 @@ func downloadPrintFormat(load int) string {
// RunDownload 执行下载网盘内文件
func RunDownload(paths []string, options *DownloadOptions) {
activeUser := GetActiveUser()
// pan token expired checker
go func() {
for {
time.Sleep(time.Duration(1) * time.Minute)
if RefreshTokenInNeed(activeUser) {
logger.Verboseln("update access token for download task")
}
}
}()
if options == nil {
options = &DownloadOptions{}
}
@ -225,10 +239,12 @@ func RunDownload(paths []string, options *DownloadOptions) {
fmt.Printf("\n[0] 当前文件下载最大并发量为: %d, 下载缓存为: %s\n", options.Parallel, converter.ConvertFileSize(int64(cfg.CacheSize), 2))
var (
panClient = GetActivePanClient()
panClient = activeUser.PanClient()
loadCount = 0
loadSize = int64(0)
)
fmt.Printf("[0] 正在计算需要下载的文件数量和大小...\n")
// 预测要下载的文件数量
for k := range paths {
// 使用递归获取文件的方法计算路径包含的文件的总数量
@ -241,11 +257,13 @@ func RunDownload(paths []string, options *DownloadOptions) {
// 忽略统计文件夹数量
if !fd.IsFolder() {
loadCount++
loadSize += fd.FileSize
}
time.Sleep(500 * time.Millisecond)
return true
})
}
fmt.Printf("[0] 预计总共需要下载的文件数量: %d\n", loadCount)
fmt.Printf("[0] 预计总共需要下载的文件数量: %d, 总大小:%s\n\n", loadCount, converter.ConvertFileSize(loadSize, 2))
cfg.MaxParallel = options.Parallel
var (
@ -298,7 +316,7 @@ func RunDownload(paths []string, options *DownloadOptions) {
// 开始执行
executor.Execute()
fmt.Printf("\n下载结束, 时间: %s, 数据总量: %s\n", statistic.Elapsed()/1e6*1e6, converter.ConvertFileSize(statistic.TotalSize()))
fmt.Printf("\n下载结束, 时间: %s, 数据总量: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize(), 2))
// 输出失败的文件列表
failedList := executor.FailedDeque()

View File

@ -234,18 +234,10 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
activeUser := GetActiveUser()
// pan token expired checker
go func() {
cz := time.FixedZone("CST", 8*3600) // 东8区
for {
time.Sleep(time.Duration(1) * time.Minute)
expiredTime, _ := time.ParseInLocation("2006-01-02 15:04:05", activeUser.WebToken.ExpireTime, cz)
now := time.Now()
if (expiredTime.Unix() - now.Unix()) <= (10 * 60) {
// need refresh token
if wt, er := aliyunpan.GetAccessTokenFromRefreshToken(activeUser.RefreshToken); er == nil {
activeUser.WebToken = *wt
activeUser.PanClient().UpdateToken(*wt)
logger.Verboseln("update access token for upload task")
}
if RefreshTokenInNeed(activeUser) {
logger.Verboseln("update access token for upload task")
}
}
}()
@ -269,6 +261,8 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
}
opt.UseInternalUrl = config.Config.TransferUrlType == 2
fmt.Printf("\n[0] 当前文件上传最大并发量为: %d, 上传分片大小为: %s\n", opt.AllParallel, converter.ConvertFileSize(opt.BlockSize, 2))
savePath = activeUser.PathJoin(opt.DriveId, savePath)
_, err1 := activeUser.PanClient().FileInfoByPath(opt.DriveId, savePath)
if err1 != nil {
@ -438,7 +432,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
}
fmt.Printf("\n")
fmt.Printf("上传结束, 时间: %s, 总大小: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize()))
fmt.Printf("上传结束, 时间: %s, 数据总量: %s\n", utils.ConvertTime(statistic.Elapsed()), converter.ConvertFileSize(statistic.TotalSize(), 2))
// 输出上传失败的文件列表
for _, failed := range failedList {

View File

@ -116,4 +116,31 @@ func EscapeStr(s string) string {
func UnescapeStr(s string) string {
r,_ := url.PathUnescape(s)
return r
}
// RefreshTokenInNeed 刷新refresh token
func RefreshTokenInNeed(activeUser *config.PanUser) bool {
if activeUser == nil {
return false
}
// refresh expired token
if activeUser.PanClient() != nil {
if len(activeUser.WebToken.RefreshToken) > 0 {
cz := time.FixedZone("CST", 8*3600) // 东8区
expiredTime, _ := time.ParseInLocation("2006-01-02 15:04:05", activeUser.WebToken.ExpireTime, cz)
now := time.Now()
if (expiredTime.Unix() - now.Unix()) <= (10 * 60) { // 10min
// need update refresh token
logger.Verboseln("access token expired, get new from refresh token")
if wt, er := aliyunpan.GetAccessTokenFromRefreshToken(activeUser.RefreshToken); er == nil {
activeUser.WebToken = *wt
activeUser.PanClient().UpdateToken(*wt)
logger.Verboseln("get new access token success")
return true
}
}
}
}
return false
}

View File

@ -20,12 +20,12 @@ import (
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/cmder/cmdutil"
"github.com/tickstep/aliyunpan/internal/waitgroup"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/cachepool"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/prealloc"
"github.com/tickstep/library-go/requester"
"github.com/tickstep/library-go/requester/rio/speeds"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"io"
"net/http"
"sync"
@ -42,6 +42,7 @@ type (
Downloader struct {
onExecuteEvent requester.Event //开始下载事件
onSuccessEvent requester.Event //成功下载事件
onFailedEvent requester.Event //成功下载事件
onFinishEvent requester.Event //结束下载事件
onPauseEvent requester.Event //暂停下载事件
onResumeEvent requester.Event //恢复下载事件
@ -80,7 +81,6 @@ func NewDownloader(writer io.WriterAt, config *Config, p *aliyunpan.PanClient, g
panClient: p,
globalSpeedsStat: globalSpeedsStat,
}
return
}
@ -369,23 +369,34 @@ func (der *Downloader) Execute() error {
var (
writeMu = &sync.Mutex{}
)
// 获取下载链接
var apierr *apierror.ApiError
durl, apierr := der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: der.driveId,
FileId: der.fileInfo.FileId,
})
time.Sleep(time.Duration(200) * time.Millisecond)
if apierr != nil {
logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId)
cmdutil.Trigger(der.onCancelEvent)
return apierr
}
if durl == nil || durl.Url == "" {
logger.Verbosef("无法获取有效的下载链接: %+v\n", durl)
cmdutil.Trigger(der.onCancelEvent)
der.removeInstanceState() // 移除断点续传文件
cmdutil.Trigger(der.onFailedEvent)
return ErrFileDownloadForbidden
}
// 初始化下载worker
for k, r := range bii.Ranges {
loadBalancer := loadBalancerResponseList.SequentialGet()
if loadBalancer == nil {
continue
}
// 获取下载链接
var apierr *apierror.ApiError
durl, apierr := der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: der.driveId,
FileId: der.fileInfo.FileId,
})
time.Sleep(time.Duration(200) * time.Millisecond)
if apierr != nil {
logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId)
continue
}
logger.Verbosef("work id: %d, download url: %s\n", k, durl)
client := requester.NewHTTPClient()
client.SetKeepAlive(true)
@ -488,6 +499,15 @@ func (der *Downloader) Cancel() {
cmdutil.Trigger(der.monitorCancelFunc)
}
//Failed 失败
func (der *Downloader) Failed() {
if der.monitor == nil {
return
}
cmdutil.Trigger(der.onFailedEvent)
cmdutil.Trigger(der.monitorCancelFunc)
}
//OnExecute 设置开始下载事件
func (der *Downloader) OnExecute(onExecuteEvent requester.Event) {
der.onExecuteEvent = onExecuteEvent
@ -498,6 +518,11 @@ func (der *Downloader) OnSuccess(onSuccessEvent requester.Event) {
der.onSuccessEvent = onSuccessEvent
}
//OnFailed 设置失败事件
func (der *Downloader) OnFailed(onFailedEvent requester.Event) {
der.onFailedEvent = onFailedEvent
}
//OnFinish 设置结束下载事件
func (der *Downloader) OnFinish(onFinishEvent requester.Event) {
der.onFinishEvent = onFinishEvent

View File

@ -14,6 +14,7 @@
package downloader
import (
"errors"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester"
mathrand "math/rand"
@ -34,6 +35,9 @@ var (
// ran 一个随机数实例
ran = mathrand.New(ranSource)
// 文件被禁止下载
ErrFileDownloadForbidden = errors.New("文件被禁止下载")
)
// RandomNumber 生成指定区间随机数

View File

@ -188,14 +188,21 @@ func (dtu *DownloadTaskUnit) download() (err error) {
})
err = der.Execute()
isComplete = true
fmt.Print("\n")
if err != nil {
// check zero size file
if err == downloader.ErrNoWokers && dtu.fileInfo.FileSize == 0 {
// success for 0 size file
dtu.verboseInfof("download success for zero size file")
} else if err == downloader.ErrFileDownloadForbidden {
// 文件被禁止下载
isComplete = false
// 删除本地文件
removeErr := os.Remove(dtu.SavePath)
if removeErr != nil {
dtu.verboseInfof("[%s] remove file error: %s\n", dtu.taskInfo.Id(), removeErr)
}
fmt.Printf("[%s] 下载失败,文件不合法或者被禁止下载: %s\n", dtu.taskInfo.Id(), dtu.SavePath)
return err
} else {
// 下载发生错误
// 下载失败, 删去空文件
@ -211,6 +218,8 @@ func (dtu *DownloadTaskUnit) download() (err error) {
}
return err
}
} else {
isComplete = true
}
// 下载成功
@ -253,9 +262,14 @@ func (dtu *DownloadTaskUnit) handleError(result *taskframework.TaskUnitRunResult
// 系统级别的错误, 可能是权限问题
result.NeedRetry = false
default:
// 其他错误, 需要重试
result.NeedRetry = true
if result.Err == downloader.ErrFileDownloadForbidden {
result.NeedRetry = false
} else {
// 其他错误, 尝试重试
result.NeedRetry = true
}
}
time.Sleep(1*time.Second)
}
//checkFileValid 检测文件有效性
@ -362,7 +376,10 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
}
// 获取该目录下的文件列表
fileList := dtu.PanClient.FilesDirectoriesRecurseList(dtu.DriveId, dtu.FilePanPath, nil)
fileList := dtu.PanClient.FilesDirectoriesRecurseList(dtu.DriveId, dtu.FilePanPath, func(depth int, _ string, fd *aliyunpan.FileEntity, apiError *apierror.ApiError) bool {
time.Sleep(500 * time.Millisecond)
return true
})
if fileList == nil {
result.ResultMessage = "获取目录信息错误"
result.Err = err

View File

@ -15,6 +15,7 @@ package panupload
import (
"fmt"
"github.com/tickstep/library-go/logger"
"os"
"path"
"path/filepath"
@ -297,7 +298,7 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
}()
// 准备文件
utu.prepareFile()
fmt.Printf("[%s] %s 准备结束, 准备耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart)))
logger.Verbosef("[%s] %s 准备结束, 准备耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart)))
var apierr *apierror.ApiError
var rs *aliyunpan.MkdirResult
@ -346,11 +347,11 @@ StepUploadPrepareUpload:
rs = &aliyunpan.MkdirResult{FileId: test.FileId}
}
utu.FolderCreateMutex.Unlock()
fmt.Printf("[%s] %s 检测和创建云盘文件夹完毕[from db], 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart3)))
logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕[from db], 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart3)))
}
if rs == nil {
timeStart4 = time.Now()
fmt.Printf("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
logger.Verbosef("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
utu.FolderCreateMutex.Lock()
// rs, apierr = utu.PanClient.MkdirRecursive(utu.DriveId, "", "", 0, strings.Split(path.Clean(saveFilePath), "/"))
// 可以直接创建的,不用循环创建
@ -361,7 +362,7 @@ StepUploadPrepareUpload:
result.ResultMessage = "创建云盘文件夹失败"
return
}
fmt.Printf("[%s] %s 创建云盘文件夹, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart4)))
logger.Verbosef("[%s] %s 创建云盘文件夹, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart4)))
}
} else {
rs = &aliyunpan.MkdirResult{}
@ -369,7 +370,7 @@ StepUploadPrepareUpload:
}
// time.Sleep(time.Duration(2) * time.Second)
// utu.FolderCreateMutex.Unlock()
fmt.Printf("[%s] %s 检测和创建云盘文件夹完毕, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart2)))
logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart2)))
sha1Str = ""
proofCode = ""

20
main.go
View File

@ -16,7 +16,6 @@ package main
import (
"fmt"
"github.com/olekukonko/tablewriter"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/cmder"
"github.com/tickstep/aliyunpan/cmder/cmdtable"
"io/ioutil"
@ -51,7 +50,7 @@ const (
var (
// Version 版本号
Version = "v0.1.0"
Version = "v0.1.1"
historyFilePath = filepath.Join(config.GetConfigDir(), "aliyunpan_command_history.txt")
@ -81,22 +80,7 @@ func checkLoginExpiredAndRelogin() {
cmder.TryLogin()
} else {
// refresh expired token
if activeUser.PanClient() != nil {
if len(activeUser.WebToken.RefreshToken) > 0 {
cz := time.FixedZone("CST", 8*3600) // 东8区
expiredTime, _ := time.ParseInLocation("2006-01-02 15:04:05", activeUser.WebToken.ExpireTime, cz)
now := time.Now()
if (expiredTime.Unix() - now.Unix()) <= (10 * 60) {
// need refresh token
logger.Verboseln("access token expired, get new from refresh token")
if wt, er := aliyunpan.GetAccessTokenFromRefreshToken(activeUser.RefreshToken); er == nil {
activeUser.WebToken = *wt
activeUser.PanClient().UpdateToken(*wt)
logger.Verboseln("get new access token success")
}
}
}
}
command.RefreshTokenInNeed(activeUser)
}
cmder.SaveConfigFunc(nil)
}

Binary file not shown.

Binary file not shown.

View File

@ -3,13 +3,13 @@
"FileVersion": {
"Major": 0,
"Minor": 1,
"Patch": 0,
"Patch": 1,
"Build": 0
},
"ProductVersion": {
"Major": 0,
"Minor": 1,
"Patch": 0,
"Patch": 1,
"Build": 0
},
"FileFlagsMask": "3f",
@ -22,14 +22,14 @@
"Comments": "",
"CompanyName": "tickstep",
"FileDescription": "阿里云盘客户端",
"FileVersion": "v0.1.0",
"FileVersion": "v0.1.1",
"InternalName": "",
"LegalCopyright": "© 2021 tickstep.",
"LegalTrademarks": "",
"OriginalFilename": "",
"PrivateBuild": "",
"ProductName": "aliyunpan",
"ProductVersion": "v0.1.0",
"ProductVersion": "v0.1.1",
"SpecialBuild": ""
},
"VarFileInfo": {