fix download occurs zero speed issue #444

This commit is contained in:
tickstep 2024-08-09 10:17:02 +08:00
parent 0d823de18f
commit 7dfd89c29d
4 changed files with 41 additions and 14 deletions

View File

@ -131,7 +131,7 @@ func CmdDownload() cli.Command {
IsOverwrite: c.Bool("ow"), IsOverwrite: c.Bool("ow"),
SaveTo: saveTo, SaveTo: saveTo,
Parallel: c.Int("p"), Parallel: c.Int("p"),
SliceParallel: c.Int("sp"), SliceParallel: 3,
Load: 0, Load: 0,
MaxRetry: c.Int("retry"), MaxRetry: c.Int("retry"),
NoCheck: c.Bool("nocheck"), NoCheck: c.Bool("nocheck"),
@ -182,11 +182,11 @@ func CmdDownload() cli.Command {
Usage: "parallel,指定同时进行下载文件的数量(取值范围:1 ~ 3", Usage: "parallel,指定同时进行下载文件的数量(取值范围:1 ~ 3",
Value: 1, Value: 1,
}, },
cli.IntFlag{ //cli.IntFlag{
Name: "sp", // Name: "sp",
Usage: "slice parallel,指定单个文件下载的最大线程(分片)数(取值范围:1 ~ 3", // Usage: "slice parallel,指定单个文件下载的最大线程(分片)数(取值范围:1 ~ 3",
Value: 1, // Value: 1,
}, //},
cli.IntFlag{ cli.IntFlag{
Name: "retry", Name: "retry",
Usage: "下载失败最大重试次数", Usage: "下载失败最大重试次数",
@ -314,7 +314,7 @@ func RunDownload(paths []string, options *DownloadOptions) {
// 阿里OpenAPI规定文件分片下载的并发数为3即某用户使用 App 时,可以同时下载 1 个文件的 3 个分片,或者同时下载 3 个文件的各 1 个分片。 // 阿里OpenAPI规定文件分片下载的并发数为3即某用户使用 App 时,可以同时下载 1 个文件的 3 个分片,或者同时下载 3 个文件的各 1 个分片。
// 超过并发,调用接口,报错 http status403并且下载速度为0 // 超过并发,调用接口,报错 http status403并且下载速度为0
if options.Parallel*options.SliceParallel > 3 { if options.Parallel*options.SliceParallel > 3 {
fmt.Println("\n####### 当前文件下载的并发数已经超过阿里云盘的限制可能会导致下载速度为0并出现下载错误 #######\n") fmt.Println("\n####### 当前文件下载的并发数已经超过阿里云盘的限制可能会导致下载速度为0 #######\n")
time.Sleep(3 * time.Second) time.Sleep(3 * time.Second)
} }

View File

@ -348,6 +348,8 @@ func (mt *Monitor) ResetWorker(worker *Worker) {
case StatusCodeDownloadUrlExpired: // 下载链接已经过期 case StatusCodeDownloadUrlExpired: // 下载链接已经过期
worker.RefreshDownloadUrl() worker.RefreshDownloadUrl()
break break
case StatusCodeDownloadUrlExceedMaxConcurrency: // 下载遇到限流报错
break
} }
mt.resetController.AddResetNum() mt.resetController.AddResetNum()
@ -406,7 +408,7 @@ func (mt *Monitor) Execute(cancelCtx context.Context) {
// 是否有失败的worker // 是否有失败的worker
for _, w := range mt.workers { for _, w := range mt.workers {
if w.status.statusCode == StatusCodeDownloadUrlExpired { if w.status.statusCode == StatusCodeDownloadUrlExpired || w.status.statusCode == StatusCodeDownloadUrlExceedMaxConcurrency {
mt.ResetWorker(w) mt.ResetWorker(w)
} }
} }

View File

@ -63,11 +63,13 @@ const (
StatusCodeCanceled StatusCodeCanceled
//StatusCodeDownloadUrlExpired 下载链接已过期 //StatusCodeDownloadUrlExpired 下载链接已过期
StatusCodeDownloadUrlExpired StatusCodeDownloadUrlExpired
// StatusCodeDownloadUrlExceedMaxConcurrency 限流报错,下载链接超过最大并发数
StatusCodeDownloadUrlExceedMaxConcurrency
//StatusCodeIllegalDownloadFile 文件非法,不允许下载 //StatusCodeIllegalDownloadFile 文件非法,不允许下载
StatusCodeIllegalDownloadFile StatusCodeIllegalDownloadFile
) )
//GetStatusText 根据状态码获取状态信息 // GetStatusText 根据状态码获取状态信息
func GetStatusText(sc StatusCode) string { func GetStatusText(sc StatusCode) string {
switch sc { switch sc {
case StatusCodeInit: case StatusCodeInit:
@ -94,29 +96,33 @@ func GetStatusText(sc StatusCode) string {
return "已重设连接" return "已重设连接"
case StatusCodeCanceled: case StatusCodeCanceled:
return "已取消" return "已取消"
case StatusCodeDownloadUrlExpired:
return "链接已过期"
case StatusCodeDownloadUrlExceedMaxConcurrency:
return "遇到限流报错"
default: default:
return "未知状态码" return "未知状态码"
} }
} }
//NewWorkerStatus 初始化WorkerStatus // NewWorkerStatus 初始化WorkerStatus
func NewWorkerStatus() *WorkerStatus { func NewWorkerStatus() *WorkerStatus {
return &WorkerStatus{ return &WorkerStatus{
statusCode: StatusCodeInit, statusCode: StatusCodeInit,
} }
} }
//SetStatusCode 设置worker状态码 // SetStatusCode 设置worker状态码
func (ws *WorkerStatus) SetStatusCode(sc StatusCode) { func (ws *WorkerStatus) SetStatusCode(sc StatusCode) {
ws.statusCode = sc ws.statusCode = sc
} }
//StatusCode 返回状态码 // StatusCode 返回状态码
func (ws *WorkerStatus) StatusCode() StatusCode { func (ws *WorkerStatus) StatusCode() StatusCode {
return ws.statusCode return ws.statusCode
} }
//StatusText 返回状态信息 // StatusText 返回状态信息
func (ws *WorkerStatus) StatusText() string { func (ws *WorkerStatus) StatusText() string {
return GetStatusText(ws.statusCode) return GetStatusText(ws.statusCode)
} }

View File

@ -30,6 +30,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"sync" "sync"
"time"
) )
type ( type (
@ -351,6 +352,24 @@ func (wer *Worker) Execute() {
logger.Verboseln("download url return 403 error and expired url response") logger.Verboseln("download url return 403 error and expired url response")
wer.status.statusCode = StatusCodeDownloadUrlExpired wer.status.statusCode = StatusCodeDownloadUrlExpired
wer.err = errors.New(resp.Status) wer.err = errors.New(resp.Status)
} else if strings.Contains(respBodyStr, "ExceedMaxConcurrency") { // 遇到限流报错
// 普通应用:文件分片下载的并发数为 3即某用户使用 App 时,可以同时下载 1 个文件的 3 个分片,或者同时下载 3 个文件的各 1 个分片。
// 超过并发,再次调用接口,报错 http status403。示例报错信息如下
// <?xml version="1.0" encoding="UTF-8"?>
//<Error>
// <Code>RequestDeniedByCallback</Code>
// <Message>Callback deny this request reason: ExceedMaxConcurrency</Message>
// <RequestId>66B5720887CECD32333EB8C1</RequestId>
// <HostId>cn-beijing-data.aliyundrive.net</HostId>
// <EC>0007-00000209</EC>
// <RecommendDoc>https://api.aliyun.com/troubleshoot?q=0007-00000209</RecommendDoc>
//</Error>
logger.Verboseln("download url return 403 error and exceed max concurrency response")
wer.status.statusCode = StatusCodeDownloadUrlExceedMaxConcurrency
wer.err = errors.New(resp.Status)
// 遇到限流,本线程延迟后,再重试
time.Sleep(10 * time.Second)
} }
} }
return return