fix download file error when file size big than 100MB

This commit is contained in:
tickstep 2024-03-02 22:38:36 +08:00
parent 9dd2482b58
commit 3bdc27ce66
6 changed files with 79 additions and 88 deletions

View File

@ -732,7 +732,7 @@ func RunAlbumDownloadFile(albumNames []string, options *DownloadOptions) {
newCfg := *cfg
unit := pandownload.DownloadTaskUnit{
Cfg: &newCfg, // 复制一份新的cfg
PanClient: panClient.WebapiPanClient(),
PanClient: panClient,
VerbosePrinter: panCommandVerbose,
PrintFormat: downloadPrintFormat(options.Load),
ParentTaskExecutor: &executor,

View File

@ -219,9 +219,9 @@ func downloadPrintFormat(load int) string {
// RunDownload 执行下载网盘内文件
func RunDownload(paths []string, options *DownloadOptions) {
activeUser := GetActiveUser()
activeUser.PanClient().WebapiPanClient().EnableCache()
activeUser.PanClient().WebapiPanClient().ClearCache()
defer activeUser.PanClient().WebapiPanClient().DisableCache()
activeUser.PanClient().OpenapiPanClient().EnableCache()
activeUser.PanClient().OpenapiPanClient().ClearCache()
defer activeUser.PanClient().OpenapiPanClient().DisableCache()
// pan token expired checker
continueFlag := int32(0)
atomic.StoreInt32(&continueFlag, 0)
@ -347,7 +347,7 @@ func RunDownload(paths []string, options *DownloadOptions) {
// 匹配的文件
unit := pandownload.DownloadTaskUnit{
Cfg: &newCfg, // 复制一份新的cfg
PanClient: panClient.WebapiPanClient(),
PanClient: panClient,
VerbosePrinter: panCommandVerbose,
PrintFormat: downloadPrintFormat(options.Load),
ParentTaskExecutor: &executor,

View File

@ -19,6 +19,7 @@ import (
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/cmder/cmdutil"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/waitgroup"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/cachepool"
@ -62,7 +63,7 @@ type (
loadBalansers []string
writer io.WriterAt
client *requester.HTTPClient
panClient *aliyunpan.PanClient
panClient *config.PanClient
config *Config
monitor *Monitor
instanceState *InstanceState
@ -75,7 +76,7 @@ type (
)
// NewDownloader 初始化Downloader
func NewDownloader(writer io.WriterAt, config *Config, p *aliyunpan.PanClient, globalSpeedsStat *speeds.Speeds) (der *Downloader) {
func NewDownloader(writer io.WriterAt, config *Config, p *config.PanClient, globalSpeedsStat *speeds.Speeds) (der *Downloader) {
der = &Downloader{
config: config,
writer: writer,
@ -379,22 +380,10 @@ func (der *Downloader) Execute() error {
// 获取下载链接
var apierr *apierror.ApiError
durl, apierr := der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
durl, apierr := der.panClient.OpenapiPanClient().GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: der.driveId,
FileId: der.fileInfo.FileId,
})
if apierr != nil && apierr.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
_, e := der.panClient.CreateSession(nil)
if e == nil {
// retry
durl, apierr = der.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: der.driveId,
FileId: der.fileInfo.FileId,
})
} else {
logger.Verboseln("CreateSession failed")
}
}
time.Sleep(time.Duration(200) * time.Millisecond)
if apierr != nil {
logger.Verbosef("ERROR: get download url error: %s\n", der.fileInfo.FileId)

View File

@ -19,11 +19,12 @@ import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/cachepool"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester"
"github.com/tickstep/library-go/requester/rio/speeds"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"io"
"net/http"
"sync"
@ -41,7 +42,7 @@ type (
driveId string
url string // 下载地址
acceptRanges string
panClient *aliyunpan.PanClient
panClient *config.PanClient
client *requester.HTTPClient
writerAt io.WriterAt
writeMu *sync.Mutex
@ -114,7 +115,7 @@ func (wer *Worker) SetClient(c *requester.HTTPClient) {
wer.client = c
}
func (wer *Worker) SetPanClient(p *aliyunpan.PanClient) {
func (wer *Worker) SetPanClient(p *config.PanClient) {
wer.panClient = p
}
@ -212,7 +213,7 @@ func (wer *Worker) Reset() {
func (wer *Worker) RefreshDownloadUrl() {
var apierr *apierror.ApiError
durl, apierr := wer.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{DriveId: wer.driveId, FileId: wer.fileId})
durl, apierr := wer.panClient.OpenapiPanClient().GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{DriveId: wer.driveId, FileId: wer.fileId})
if apierr != nil {
wer.status.statusCode = StatusCodeTooManyConnections
return
@ -296,7 +297,7 @@ func (wer *Worker) Execute() {
var resp *http.Response
apierr := wer.panClient.DownloadFileData(wer.url, aliyunpan.FileDownloadRange{
apierr := wer.panClient.OpenapiPanClient().DownloadFileData(wer.url, aliyunpan.FileDownloadRange{
Offset: wer.wrange.Begin,
End: wer.wrange.End - 1,
}, func(httpMethod, fullUrl string, headers map[string]string) (*http.Response, error) {

View File

@ -50,7 +50,7 @@ type (
taskInfo *taskframework.TaskInfo // 任务信息
Cfg *downloader.Config
PanClient *aliyunpan.PanClient
PanClient *config.PanClient
ParentTaskExecutor *taskframework.TaskExecutor
DownloadStatistic *DownloadStatistic // 下载统计
@ -462,7 +462,7 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
// 没有获取文件信息
// 如果是动态添加的下载任务, 是会写入文件信息的
// 如果该任务重试过, 则应该再获取一次文件信息
dtu.fileInfo, apierr = dtu.PanClient.FileInfoByPath(dtu.DriveId, dtu.FilePanPath)
dtu.fileInfo, apierr = dtu.PanClient.OpenapiPanClient().FileInfoByPath(dtu.DriveId, dtu.FilePanPath)
if apierr != nil {
// 如果不是未登录或文件不存在, 则不重试
result.ResultMessage = "获取下载路径信息错误"
@ -545,7 +545,7 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
}
// 获取该目录下的文件列表
fileList, apierr := dtu.PanClient.FileListGetAll(&aliyunpan.FileListParam{
fileList, apierr := dtu.PanClient.OpenapiPanClient().FileListGetAll(&aliyunpan.FileListParam{
DriveId: dtu.DriveId,
ParentFileId: dtu.fileInfo.FileId,
}, 1000)
@ -553,7 +553,7 @@ func (dtu *DownloadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
// retry one more time
time.Sleep(3 * time.Second)
fileList, apierr = dtu.PanClient.FileListGetAll(&aliyunpan.FileListParam{
fileList, apierr = dtu.PanClient.OpenapiPanClient().FileListGetAll(&aliyunpan.FileListParam{
DriveId: dtu.DriveId,
ParentFileId: dtu.fileInfo.FileId,
}, 1000)

View File

@ -332,7 +332,8 @@ func (f *FileActionTask) downloadFile(ctx context.Context) error {
client.SetKeepAlive(true)
client.SetTimeout(10 * time.Minute)
worker.SetClient(client)
worker.SetPanClient(f.panClient)
// TODO: need fix
//worker.SetPanClient(f.panClient)
writeMu := &sync.Mutex{}
worker.SetWriteMutex(writeMu)