mirror of
https://github.com/tickstep/aliyunpan.git
synced 2025-01-23 14:32:14 +08:00
optimize upload code
This commit is contained in:
tickstep
parent
5cab8cc6a5
commit
df53c7f44b
@ -441,21 +441,76 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
|
||||
targetPanFilePath := f.syncItem.getPanFileFullPath()
|
||||
|
||||
if f.syncItem.UploadEntity == nil {
|
||||
// 尝试创建文件夹
|
||||
panDirPath := filepath.Dir(targetPanFilePath)
|
||||
panDirFileId := ""
|
||||
if dirFile, er2 := f.panClient.OpenapiPanClient().FileInfoByPath(f.syncItem.DriveId, panDirPath); er2 != nil {
|
||||
if er2.Code == apierror.ApiCodeFileNotFoundCode {
|
||||
logger.Verbosef("创建云盘文件夹: %s\n", panDirPath)
|
||||
f.panFolderCreateMutex.Lock()
|
||||
rs, apierr1 := f.panClient.OpenapiPanClient().MkdirByFullPath(f.syncItem.DriveId, panDirPath)
|
||||
f.panFolderCreateMutex.Unlock()
|
||||
if apierr1 != nil || rs.FileId == "" {
|
||||
return apierr1
|
||||
}
|
||||
panDirFileId = rs.FileId
|
||||
logger.Verbosef("创建云盘文件夹成功: %s\n", panDirPath)
|
||||
} else {
|
||||
logger.Verbosef("创建云盘文件夹错误: %s\n", er2.String())
|
||||
return er2
|
||||
}
|
||||
} else {
|
||||
if dirFile != nil && dirFile.FileId != "" {
|
||||
panDirFileId = dirFile.FileId
|
||||
}
|
||||
}
|
||||
|
||||
// 计算文件SHA1
|
||||
sha1Str := ""
|
||||
|
||||
proofCode := ""
|
||||
contentHashName := "sha1"
|
||||
if f.syncItem.LocalFile.Sha1Hash != "" {
|
||||
sha1Str = f.syncItem.LocalFile.Sha1Hash
|
||||
} else {
|
||||
logger.Verbosef("正在计算文件SHA1: %s\n", localFile.Path)
|
||||
if localFile.Length == 0 {
|
||||
sha1Str = aliyunpan.DefaultZeroSizeFileContentHash
|
||||
} else {
|
||||
localFile.Sum(localfile.CHECKSUM_SHA1)
|
||||
sha1Str = localFile.SHA1
|
||||
// 正常上传流程，检测是否能秒传
|
||||
preHashMatch := true
|
||||
if f.syncItem.LocalFile.FileSize >= panupload.DefaultCheckPreHashFileSize {
|
||||
// 大文件，先计算 PreHash，用于检测是否可能支持秒传
|
||||
preHash := panupload.CalcFilePreHash(f.syncItem.LocalFile.Path)
|
||||
if len(preHash) > 0 {
|
||||
if b, er := f.panClient.OpenapiPanClient().CheckUploadFilePreHash(&aliyunpan.FileUploadCheckPreHashParam{
|
||||
DriveId: f.syncItem.DriveId,
|
||||
Name: f.syncItem.LocalFile.FileName,
|
||||
Size: f.syncItem.LocalFile.FileSize,
|
||||
ParentFileId: panDirFileId,
|
||||
PreHash: preHash,
|
||||
}); er == nil {
|
||||
preHashMatch = b
|
||||
}
|
||||
}
|
||||
}
|
||||
if preHashMatch {
|
||||
// 再计算完整文件SHA1
|
||||
logger.Verbosef("正在计算文件SHA1: %s\n", localFile.Path)
|
||||
if localFile.Length == 0 {
|
||||
sha1Str = aliyunpan.DefaultZeroSizeFileContentHash
|
||||
} else {
|
||||
localFile.Sum(localfile.CHECKSUM_SHA1)
|
||||
sha1Str = localFile.SHA1
|
||||
}
|
||||
f.syncItem.LocalFile.Sha1Hash = sha1Str
|
||||
f.syncFileDb.Update(f.syncItem)
|
||||
|
||||
// 计算proof code
|
||||
localFileEntity, _ := os.Open(localFile.Path.RealPath)
|
||||
localFileInfo, _ := localFileEntity.Stat()
|
||||
proofCode = aliyunpan.CalcProofCode(f.panClient.OpenapiPanClient().GetAccessToken(), rio.NewFileReaderAtLen64(localFileEntity), localFileInfo.Size())
|
||||
} else {
|
||||
// 无需计算 sha1，直接上传
|
||||
logger.Verboseln("PreHash not match, upload file directly")
|
||||
sha1Str = ""
|
||||
contentHashName = ""
|
||||
}
|
||||
f.syncItem.LocalFile.Sha1Hash = sha1Str
|
||||
f.syncFileDb.Update(f.syncItem)
|
||||
}
|
||||
|
||||
// 检查同名文件是否存在
|
||||
@ -489,37 +544,6 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// 创建文件夹
|
||||
panDirPath := filepath.Dir(targetPanFilePath)
|
||||
panDirFileId := ""
|
||||
|
||||
if dirFile, er2 := f.panClient.OpenapiPanClient().FileInfoByPath(f.syncItem.DriveId, panDirPath); er2 != nil {
|
||||
if er2.Code == apierror.ApiCodeFileNotFoundCode {
|
||||
logger.Verbosef("创建云盘文件夹: %s\n", panDirPath)
|
||||
f.panFolderCreateMutex.Lock()
|
||||
rs, apierr1 := f.panClient.OpenapiPanClient().MkdirByFullPath(f.syncItem.DriveId, panDirPath)
|
||||
f.panFolderCreateMutex.Unlock()
|
||||
if apierr1 != nil || rs.FileId == "" {
|
||||
return apierr1
|
||||
}
|
||||
panDirFileId = rs.FileId
|
||||
logger.Verbosef("创建云盘文件夹成功: %s\n", panDirPath)
|
||||
} else {
|
||||
logger.Verbosef("上传文件错误: %s\n", apierr.String())
|
||||
return apierr
|
||||
}
|
||||
} else {
|
||||
if dirFile != nil && dirFile.FileId != "" {
|
||||
panDirFileId = dirFile.FileId
|
||||
}
|
||||
}
|
||||
|
||||
// 计算proof code
|
||||
proofCode := ""
|
||||
localFileEntity, _ := os.Open(localFile.Path.RealPath)
|
||||
localFileInfo, _ := localFileEntity.Stat()
|
||||
proofCode = aliyunpan.CalcProofCode(f.panClient.OpenapiPanClient().GetAccessToken(), rio.NewFileReaderAtLen64(localFileEntity), localFileInfo.Size())
|
||||
|
||||
// 自动调整BlockSize大小
|
||||
newBlockSize := utils.ResizeUploadBlockSize(localFile.Length, f.syncItem.UploadBlockSize)
|
||||
if newBlockSize != f.syncItem.UploadBlockSize {
|
||||
@ -535,7 +559,7 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
|
||||
Name: filepath.Base(targetPanFilePath),
|
||||
Size: localFile.Length,
|
||||
ContentHash: sha1Str,
|
||||
ContentHashName: "sha1",
|
||||
ContentHashName: contentHashName,
|
||||
CheckNameMode: "refuse",
|
||||
ParentFileId: panDirFileId,
|
||||
BlockSize: f.syncItem.UploadBlockSize,
|
||||
|
||||
@ -4,9 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
mapset "github.com/deckarep/golang-set"
|
||||
"github.com/tickstep/aliyunpan-api/aliyunpan"
|
||||
"github.com/tickstep/aliyunpan/internal/config"
|
||||
"github.com/tickstep/aliyunpan/internal/localfile"
|
||||
"github.com/tickstep/aliyunpan/internal/plugins"
|
||||
"github.com/tickstep/aliyunpan/internal/utils"
|
||||
"github.com/tickstep/aliyunpan/internal/waitgroup"
|
||||
@ -256,26 +254,27 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
|
||||
// 本地文件和云盘文件SHA1不一样
|
||||
// 不同模式同步策略不一样
|
||||
if f.task.Mode == UploadOnly {
|
||||
// 计算本地文件SHA1
|
||||
if localFile.Sha1Hash == "" {
|
||||
// calc sha1
|
||||
if localFile.FileSize == 0 {
|
||||
localFile.Sha1Hash = aliyunpan.DefaultZeroSizeFileContentHash
|
||||
} else {
|
||||
fileSum := localfile.NewLocalFileEntity(localFile.Path)
|
||||
err := fileSum.OpenPath()
|
||||
if err != nil {
|
||||
logger.Verbosef("文件不可读, 错误信息: %s, 跳过...\n", err)
|
||||
continue
|
||||
}
|
||||
fileSum.Sum(localfile.CHECKSUM_SHA1) // block operation
|
||||
localFile.Sha1Hash = fileSum.SHA1
|
||||
fileSum.Close()
|
||||
}
|
||||
|
||||
// save sha1 to local DB
|
||||
f.task.localFileDb.Update(localFile)
|
||||
}
|
||||
// 不再这里计算SHA1，待到上传的时候再计算
|
||||
//if localFile.Sha1Hash == "" {
|
||||
// // 计算本地文件SHA1
|
||||
// if localFile.FileSize == 0 {
|
||||
// localFile.Sha1Hash = aliyunpan.DefaultZeroSizeFileContentHash
|
||||
// } else {
|
||||
// fileSum := localfile.NewLocalFileEntity(localFile.Path)
|
||||
// err := fileSum.OpenPath()
|
||||
// if err != nil {
|
||||
// logger.Verbosef("文件不可读, 错误信息: %s, 跳过...\n", err)
|
||||
// continue
|
||||
// }
|
||||
// fileSum.Sum(localfile.CHECKSUM_SHA1) // block operation
|
||||
// localFile.Sha1Hash = fileSum.SHA1
|
||||
// fileSum.Close()
|
||||
// }
|
||||
//
|
||||
// // save sha1 to local DB
|
||||
// f.task.localFileDb.Update(localFile)
|
||||
//}
|
||||
|
||||
// 校验SHA1是否相同
|
||||
if strings.ToLower(panFile.Sha1Hash) == strings.ToLower(localFile.Sha1Hash) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user