optimize uplaod file speed

internal/command/upload.go

@@ -101,8 +101,8 @@ var UploadFlags = []cli.Flag{
 	},
 	cli.IntFlag{
 		Name:  "bs",
-		Usage: "block size，上传分片大小，单位KB。推荐值：512 ~ 2048",
-		Value: 512,
+		Usage: "block size，上传分片大小，单位KB。推荐值：1024 ~ 10240",
+		Value: 10240,
 	},
 }
 
@@ -416,7 +416,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
 				UseInternalUrl:    opt.UseInternalUrl,
 			}, opt.MaxRetry)
 
-			fmt.Printf("%s [%s] 加入上传队列: %s\n", time.Now().Format("2006-01-02 15:04:05"), taskinfo.Id(), file)
+			fmt.Printf("[%s] 加入上传队列: %s\n", taskinfo.Id(), file)
 			return nil
 		}
 		if err := WalkAllFile(curPath, walkFunc); err != nil {

internal/file/uploader/multiuploader.go

@@ -29,7 +29,7 @@ type (
 	// MultiUpload 支持多线程的上传, 可用于断点续传
 	MultiUpload interface {
 		Precreate() (perr error)
-		UploadFile(ctx context.Context, partseq int, partOffset int64, partEnd int64, readerlen64 rio.ReaderLen64) (uploadDone bool, terr error)
+		UploadFile(ctx context.Context, partseq int, partOffset int64, partEnd int64, readerlen64 rio.ReaderLen64, uploadClient *requester.HTTPClient) (uploadDone bool, terr error)
 		CommitFile() (cerr error)
 	}
 

internal/file/uploader/multiworker.go

@@ -17,6 +17,7 @@ import (
 	"context"
 	"github.com/tickstep/aliyunpan/internal/waitgroup"
 	"github.com/oleiade/lane"
+	"github.com/tickstep/library-go/requester"
 	"os"
 	"strconv"
 )
@@ -59,6 +60,11 @@ func (muer *MultiUploader) upload() (uperr error) {
 		}
 	}
 
+	// 上传客户端
+	uploadClient := requester.NewHTTPClient()
+	uploadClient.SetTimeout(0)
+	uploadClient.SetKeepAlive(true)
+
 	for {
 		// 阿里云盘只支持分片按顺序上传，这里正常应该是parallel = 1
 		wg := waitgroup.NewWaitGroup(muer.config.Parallel)
@@ -82,7 +88,7 @@ func (muer *MultiUploader) upload() (uperr error) {
 				go func() {
 					if !wer.uploadDone {
 						uploaderVerbose.Info("begin to upload part: " + strconv.Itoa(wer.id))
-						uploadDone, terr = muer.multiUpload.UploadFile(ctx, int(wer.id), wer.partOffset, wer.splitUnit.Range().End, wer.splitUnit)
+						uploadDone, terr = muer.multiUpload.UploadFile(ctx, int(wer.id), wer.partOffset, wer.splitUnit.Range().End, wer.splitUnit, uploadClient)
 					} else {
 						uploadDone = true
 					}
@@ -138,6 +144,9 @@ func (muer *MultiUploader) upload() (uperr error) {
 		}
 	}
 
+	// 释放链路
+	uploadClient.CloseIdleConnections()
+
 	select {
 	case <-muer.canceled:
 		if uperr != nil {

internal/functions/panupload/upload.go

@@ -18,6 +18,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"github.com/tickstep/library-go/logger"
+	"github.com/tickstep/library-go/requester"
 	"io"
 	"net/http"
 	"strconv"
@@ -25,7 +26,6 @@ import (
 	"github.com/tickstep/aliyunpan-api/aliyunpan"
 	"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
 	"github.com/tickstep/aliyunpan/internal/file/uploader"
-	"github.com/tickstep/library-go/requester"
 	"github.com/tickstep/library-go/requester/rio"
 )
 
@@ -90,7 +90,7 @@ func (pu *PanUpload) Precreate() (err error) {
 	return nil
 }
 
-func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int64, partEnd int64, r rio.ReaderLen64) (uploadDone bool, uperr error) {
+func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int64, partEnd int64, r rio.ReaderLen64, uploadClient *requester.HTTPClient) (uploadDone bool, uperr error) {
 	pu.lazyInit()
 
 	// check url expired or not
@@ -129,9 +129,12 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
 		respErr = nil
 
 		// do http upload request
-		client := requester.NewHTTPClient()
-		client.SetTimeout(0)
-		resp, _ = client.Req(httpMethod, fullUrl, r, headers)
+		if uploadClient == nil {
+			uploadClient = requester.NewHTTPClient()
+			uploadClient.SetTimeout(0)
+			uploadClient.SetKeepAlive(true)
+		}
+		resp, _ = uploadClient.Req(httpMethod, fullUrl, r, headers)
 
 		if resp != nil {
 			if blen, e := strconv.Atoi(resp.Header.Get("content-length")); e == nil {

internal/functions/panupload/upload_task_unit.go

@@ -315,13 +315,13 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
 	}
 
 StepUploadPrepareUpload:
-
 	if utu.FolderSyncDb != nil {
 		//启用了备份功能，强制使用覆盖同名文件功能
 		utu.IsOverwrite = true
 		testFileMeta = utu.FolderSyncDb.Get(utu.SavePath)
 	}
 	// 创建上传任务
+	fmt.Printf("[%s] 正在计算文件SHA1: %s\n", utu.taskInfo.Id(), utu.LocalFileChecksum.Path)
 	utu.LocalFileChecksum.Sum(localfile.CHECKSUM_SHA1)
 
 	if testFileMeta.SHA1 == utu.LocalFileChecksum.SHA1 {