fix download file error when file size big than 100MB

This commit is contained in:
tickstep 2024-03-02 22:38:36 +08:00
parent 9dd2482b58
commit 3bdc27ce66
6 changed files with 79 additions and 88 deletions

View File

@ -732,7 +732,7 @@ func RunAlbumDownloadFile(albumNames []string, options *DownloadOptions) {
newCfg := *cfg newCfg := *cfg
unit := pandownload.DownloadTaskUnit{ unit := pandownload.DownloadTaskUnit{
Cfg: &newCfg, // 复制一份新的cfg Cfg: &newCfg, // 复制一份新的cfg
PanClient: panClient.WebapiPanClient(), PanClient: panClient,
VerbosePrinter: panCommandVerbose, VerbosePrinter: panCommandVerbose,
PrintFormat: downloadPrintFormat(options.Load), PrintFormat: downloadPrintFormat(options.Load),
ParentTaskExecutor: &executor, ParentTaskExecutor: &executor,

View File

@ -219,9 +219,9 @@ func downloadPrintFormat(load int) string {
// RunDownload 执行下载网盘内文件 // RunDownload 执行下载网盘内文件
func RunDownload(paths []string, options *DownloadOptions) { func RunDownload(paths []string, options *DownloadOptions) {
activeUser := GetActiveUser() activeUser := GetActiveUser()
activeUser.PanClient().WebapiPanClient().EnableCache() activeUser.PanClient().OpenapiPanClient().EnableCache()
activeUser.PanClient().WebapiPanClient().ClearCache() activeUser.PanClient().OpenapiPanClient().ClearCache()
defer activeUser.PanClient().WebapiPanClient().DisableCache() defer activeUser.PanClient().OpenapiPanClient().DisableCache()
// pan token expired checker // pan token expired checker
continueFlag := int32(0) continueFlag := int32(0)
atomic.StoreInt32(&continueFlag, 0) atomic.StoreInt32(&continueFlag, 0)
@ -347,7 +347,7 @@ func RunDownload(paths []string, options *DownloadOptions) {
// 匹配的文件 // 匹配的文件
unit := pandownload.DownloadTaskUnit{ unit := pandownload.DownloadTaskUnit{
Cfg: &newCfg, // 复制一份新的cfg Cfg: &newCfg, // 复制一份新的cfg
PanClient: panClient.WebapiPanClient(), PanClient: panClient,
VerbosePrinter: panCommandVerbose, VerbosePrinter: panCommandVerbose,
PrintFormat: downloadPrintFormat(options.Load), PrintFormat: downloadPrintFormat(options.Load),
ParentTaskExecutor: &executor, ParentTaskExecutor: &executor,

View File

@ -19,6 +19,7 @@ import (
"github.com/tickstep/aliyunpan-api/aliyunpan" "github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror" "github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/cmder/cmdutil" "github.com/tickstep/aliyunpan/cmder/cmdutil"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/waitgroup" "github.com/tickstep/aliyunpan/internal/waitgroup"
"github.com/tickstep/aliyunpan/library/requester/transfer" "github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/cachepool" "github.com/tickstep/library-go/cachepool"
@ -62,7 +63,7 @@ type (
loadBalansers []string loadBalansers []string
writer io.WriterAt writer io.WriterAt
client *requester.HTTPClient client *requester.HTTPClient
panClient *aliyunpan.PanClient panClient *config.PanClient
config *Config config *Config
monitor *Monitor monitor *Monitor
instanceState *InstanceState instanceState *InstanceState
@ -75,7 +76,7 @@ type (
) )
// NewDownloader 初始化Downloader // NewDownloader 初始化Downloader
func NewDownloader(writer io.WriterAt, config *Config, p *aliyunpan.PanClient, globalSpeedsStat *speeds.Speeds) (der *Downloader) { func NewDownloader(writer io.WriterAt, config *Config, p *config.PanClient, globalSpeedsStat *speeds.Speeds) (der *Downloader) {
der = &Downloader{ der = &Downloader{
config: config, config: config,
writer: writer, writer: writer,
@ -379,22 +380,10 @@ func (der *Downloader) Execute() error {
// 获取下载链接 // 获取下载链接
var apierr *apierror.ApiError var apierr *apierror.ApiError
durl, apierr := der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{ durl, apierr := der.panClient.OpenapiPanClient().GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: der.driveId, DriveId: der.driveId,
FileId: der.fileInfo.FileId, FileId: der.fileInfo.FileId,
}) })
if apierr != nil && apierr.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
_, e := der.panClient.CreateSession(nil)
if e == nil {
// retry
durl, apierr = der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: der.driveId,
FileId: der.fileInfo.FileId,
})
} else {
logger.Verboseln("CreateSession failed")
}
}
time.Sleep(time.Duration(200) * time.Millisecond) time.Sleep(time.Duration(200) * time.Millisecond)
if apierr != nil { if apierr != nil {
logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId) logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId)

View File

@ -19,11 +19,12 @@ import (
"fmt" "fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan" "github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror" "github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/cachepool" "github.com/tickstep/library-go/cachepool"
"github.com/tickstep/library-go/logger" "github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester" "github.com/tickstep/library-go/requester"
"github.com/tickstep/library-go/requester/rio/speeds" "github.com/tickstep/library-go/requester/rio/speeds"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"io" "io"
"net/http" "net/http"
"sync" "sync"
@ -41,7 +42,7 @@ type (
driveId string driveId string
url string // 下载地址 url string // 下载地址
acceptRanges string acceptRanges string
panClient *aliyunpan.PanClient panClient *config.PanClient
client *requester.HTTPClient client *requester.HTTPClient
writerAt io.WriterAt writerAt io.WriterAt
writeMu *sync.Mutex writeMu *sync.Mutex
@ -114,7 +115,7 @@ func (wer *Worker) SetClient(c *requester.HTTPClient) {
wer.client = c wer.client = c
} }
func (wer *Worker) SetPanClient(p *aliyunpan.PanClient) { func (wer *Worker) SetPanClient(p *config.PanClient) {
wer.panClient = p wer.panClient = p
} }
@ -212,7 +213,7 @@ func (wer *Worker) Reset() {
func (wer *Worker) RefreshDownloadUrl() { func (wer *Worker) RefreshDownloadUrl() {
var apierr *apierror.ApiError var apierr *apierror.ApiError
durl, apierr := wer.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{DriveId: wer.driveId, FileId: wer.fileId}) durl, apierr := wer.panClient.OpenapiPanClient().GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{DriveId: wer.driveId, FileId: wer.fileId})
if apierr != nil { if apierr != nil {
wer.status.statusCode = StatusCodeTooManyConnections wer.status.statusCode = StatusCodeTooManyConnections
return return
@ -296,7 +297,7 @@ func (wer *Worker) Execute() {
var resp *http.Response var resp *http.Response
apierr := wer.panClient.DownloadFileData(wer.url, aliyunpan.FileDownloadRange{ apierr := wer.panClient.OpenapiPanClient().DownloadFileData(wer.url, aliyunpan.FileDownloadRange{
Offset: wer.wrange.Begin, Offset: wer.wrange.Begin,
End: wer.wrange.End - 1, End: wer.wrange.End - 1,
}, func(httpMethod, fullUrl string, headers map[string]string) (*http.Response, error) { }, func(httpMethod, fullUrl string, headers map[string]string) (*http.Response, error) {

View File

@ -50,7 +50,7 @@ type (
taskInfo *taskframework.TaskInfo // 任务信息 taskInfo *taskframework.TaskInfo // 任务信息
Cfg *downloader.Config Cfg *downloader.Config
PanClient *aliyunpan.PanClient PanClient *config.PanClient
ParentTaskExecutor *taskframework.TaskExecutor ParentTaskExecutor *taskframework.TaskExecutor
DownloadStatistic *DownloadStatistic // 下载统计 DownloadStatistic *DownloadStatistic // 下载统计
@ -462,7 +462,7 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
// 没有获取文件信息 // 没有获取文件信息
// 如果是动态添加的下载任务, 是会写入文件信息的 // 如果是动态添加的下载任务, 是会写入文件信息的
// 如果该任务重试过, 则应该再获取一次文件信息 // 如果该任务重试过, 则应该再获取一次文件信息
dtu.fileInfo, apierr = dtu.PanClient.FileInfoByPath(dtu.DriveId, dtu.FilePanPath) dtu.fileInfo, apierr = dtu.PanClient.OpenapiPanClient().FileInfoByPath(dtu.DriveId, dtu.FilePanPath)
if apierr != nil { if apierr != nil {
// 如果不是未登录或文件不存在, 则不重试 // 如果不是未登录或文件不存在, 则不重试
result.ResultMessage = "获取下载路径信息错误" result.ResultMessage = "获取下载路径信息错误"
@ -545,7 +545,7 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
} }
// 获取该目录下的文件列表 // 获取该目录下的文件列表
fileList, apierr := dtu.PanClient.FileListGetAll(&aliyunpan.FileListParam{ fileList, apierr := dtu.PanClient.OpenapiPanClient().FileListGetAll(&aliyunpan.FileListParam{
DriveId: dtu.DriveId, DriveId: dtu.DriveId,
ParentFileId: dtu.fileInfo.FileId, ParentFileId: dtu.fileInfo.FileId,
}, 1000) }, 1000)
@ -553,7 +553,7 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
// retry one more time // retry one more time
time.Sleep(3 * time.Second) time.Sleep(3 * time.Second)
fileList, apierr = dtu.PanClient.FileListGetAll(&aliyunpan.FileListParam{ fileList, apierr = dtu.PanClient.OpenapiPanClient().FileListGetAll(&aliyunpan.FileListParam{
DriveId: dtu.DriveId, DriveId: dtu.DriveId,
ParentFileId: dtu.fileInfo.FileId, ParentFileId: dtu.fileInfo.FileId,
}, 1000) }, 1000)

View File

@ -332,7 +332,8 @@ func (f *FileActionTask) downloadFile(ctx context.Context) error {
client.SetKeepAlive(true) client.SetKeepAlive(true)
client.SetTimeout(10 * time.Minute) client.SetTimeout(10 * time.Minute)
worker.SetClient(client) worker.SetClient(client)
worker.SetPanClient(f.panClient) // TODO: need fix
//worker.SetPanClient(f.panClient)
writeMu := &sync.Mutex{} writeMu := &sync.Mutex{}
worker.SetWriteMutex(writeMu) worker.SetWriteMutex(writeMu)