From 7dfd89c29dba6143359bd02bad4f1bffbfba561f Mon Sep 17 00:00:00 2001 From: tickstep Date: Fri, 9 Aug 2024 10:17:02 +0800 Subject: [PATCH] fix download occurs zero speed issue #444 --- internal/command/download.go | 14 +++++++------- internal/file/downloader/monitor.go | 4 +++- internal/file/downloader/status.go | 18 ++++++++++++------ internal/file/downloader/worker.go | 19 +++++++++++++++++++ 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/internal/command/download.go b/internal/command/download.go index aac6186..dfe2b10 100644 --- a/internal/command/download.go +++ b/internal/command/download.go @@ -131,7 +131,7 @@ func CmdDownload() cli.Command { IsOverwrite: c.Bool("ow"), SaveTo: saveTo, Parallel: c.Int("p"), - SliceParallel: c.Int("sp"), + SliceParallel: 3, Load: 0, MaxRetry: c.Int("retry"), NoCheck: c.Bool("nocheck"), @@ -182,11 +182,11 @@ func CmdDownload() cli.Command { Usage: "parallel,指定同时进行下载文件的数量（取值范围:1 ~ 3）", Value: 1, }, - cli.IntFlag{ - Name: "sp", - Usage: "slice parallel,指定单个文件下载的最大线程(分片)数（取值范围:1 ~ 3）", - Value: 1, - }, + //cli.IntFlag{ + // Name: "sp", + // Usage: "slice parallel,指定单个文件下载的最大线程(分片)数（取值范围:1 ~ 3）", + // Value: 1, + //}, cli.IntFlag{ Name: "retry", Usage: "下载失败最大重试次数", @@ -314,7 +314,7 @@ func RunDownload(paths []string, options *DownloadOptions) { // 阿里OpenAPI规定：文件分片下载的并发数为3，即某用户使用 App 时，可以同时下载 1 个文件的 3 个分片，或者同时下载 3 个文件的各 1 个分片。 // 超过并发，调用接口，报错 http status：403，并且下载速度为0 if options.Parallel*options.SliceParallel > 3 { - fmt.Println("\n####### 当前文件下载的并发数已经超过阿里云盘的限制，可能会导致下载速度为0并出现下载错误！ #######\n") + fmt.Println("\n####### 当前文件下载的并发数已经超过阿里云盘的限制，可能会导致下载速度为0！ #######\n") time.Sleep(3 * time.Second) } diff --git a/internal/file/downloader/monitor.go b/internal/file/downloader/monitor.go index c4b08fb..fc2a8be 100644 --- a/internal/file/downloader/monitor.go +++ b/internal/file/downloader/monitor.go @@ -348,6 +348,8 @@ func (mt *Monitor) ResetWorker(worker *Worker) { case StatusCodeDownloadUrlExpired: // 下载链接已经过期 worker.RefreshDownloadUrl() break + case StatusCodeDownloadUrlExceedMaxConcurrency: // 下载遇到限流报错 + break } mt.resetController.AddResetNum() @@ -406,7 +408,7 @@ func (mt *Monitor) Execute(cancelCtx context.Context) { // 是否有失败的worker for _, w := range mt.workers { - if w.status.statusCode == StatusCodeDownloadUrlExpired { + if w.status.statusCode == StatusCodeDownloadUrlExpired || w.status.statusCode == StatusCodeDownloadUrlExceedMaxConcurrency { mt.ResetWorker(w) } } diff --git a/internal/file/downloader/status.go b/internal/file/downloader/status.go index 7db1665..b19c96d 100644 --- a/internal/file/downloader/status.go +++ b/internal/file/downloader/status.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -63,11 +63,13 @@ const ( StatusCodeCanceled //StatusCodeDownloadUrlExpired 下载链接已过期 StatusCodeDownloadUrlExpired + // StatusCodeDownloadUrlExceedMaxConcurrency 限流报错，下载链接超过最大并发数 + StatusCodeDownloadUrlExceedMaxConcurrency //StatusCodeIllegalDownloadFile 文件非法，不允许下载 StatusCodeIllegalDownloadFile ) -//GetStatusText 根据状态码获取状态信息 +// GetStatusText 根据状态码获取状态信息 func GetStatusText(sc StatusCode) string { switch sc { case StatusCodeInit: @@ -94,29 +96,33 @@ func GetStatusText(sc StatusCode) string { return "已重设连接" case StatusCodeCanceled: return "已取消" + case StatusCodeDownloadUrlExpired: + return "链接已过期" + case StatusCodeDownloadUrlExceedMaxConcurrency: + return "遇到限流报错" default: return "未知状态码" } } -//NewWorkerStatus 初始化WorkerStatus +// NewWorkerStatus 初始化WorkerStatus func NewWorkerStatus() *WorkerStatus { return &WorkerStatus{ statusCode: StatusCodeInit, } } -//SetStatusCode 设置worker状态码 +// SetStatusCode 设置worker状态码 func (ws *WorkerStatus) SetStatusCode(sc StatusCode) { ws.statusCode = sc } -//StatusCode 返回状态码 +// StatusCode 返回状态码 func (ws *WorkerStatus) StatusCode() StatusCode { return ws.statusCode } -//StatusText 返回状态信息 +// StatusText 返回状态信息 func (ws *WorkerStatus) StatusText() string { return GetStatusText(ws.statusCode) } diff --git a/internal/file/downloader/worker.go b/internal/file/downloader/worker.go index 9012e9f..394006d 100644 --- a/internal/file/downloader/worker.go +++ b/internal/file/downloader/worker.go @@ -30,6 +30,7 @@ import ( "net/http" "strings" "sync" + "time" ) type ( @@ -351,6 +352,24 @@ func (wer *Worker) Execute() { logger.Verboseln("download url return 403 error and expired url response") wer.status.statusCode = StatusCodeDownloadUrlExpired wer.err = errors.New(resp.Status) + } else if strings.Contains(respBodyStr, "ExceedMaxConcurrency") { // 遇到限流报错 + // 普通应用：文件分片下载的并发数为 3，即某用户使用 App 时，可以同时下载 1 个文件的 3 个分片，或者同时下载 3 个文件的各 1 个分片。 + // 超过并发，再次调用接口，报错 http status：403。示例报错信息如下： + // + // + // RequestDeniedByCallback + // Callback deny this request reason: ExceedMaxConcurrency + // 66B5720887CECD32333EB8C1 + // cn-beijing-data.aliyundrive.net + // 0007-00000209 + // https://api.aliyun.com/troubleshoot?q=0007-00000209 + // + logger.Verboseln("download url return 403 error and exceed max concurrency response") + wer.status.statusCode = StatusCodeDownloadUrlExceedMaxConcurrency + wer.err = errors.New(resp.Status) + + // 遇到限流，本线程延迟后，再重试 + time.Sleep(10 * time.Second) } } return