Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-01-23 14:32:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

optimize upload code

Browse Source
This commit is contained in:
tickstep 2024-03-16 21:28:21 +08:00
parent 5cab8cc6a5
commit df53c7f44b
2 changed files with 85 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
internal/syncdrive/file_action_task.go
View File

@ -441,12 +441,56 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
targetPanFilePath := f.syncItem.getPanFileFullPath()
if f.syncItem.UploadEntity == nil {
// 尝试创建文件夹
panDirPath := filepath.Dir(targetPanFilePath)
panDirFileId := ""
if dirFile, er2 := f.panClient.OpenapiPanClient().FileInfoByPath(f.syncItem.DriveId, panDirPath); er2 != nil {
if er2.Code == apierror.ApiCodeFileNotFoundCode {
logger.Verbosef("创建云盘文件夹: %s\n", panDirPath)
f.panFolderCreateMutex.Lock()
rs, apierr1 := f.panClient.OpenapiPanClient().MkdirByFullPath(f.syncItem.DriveId, panDirPath)
f.panFolderCreateMutex.Unlock()
if apierr1 != nil || rs.FileId == "" {
return apierr1
}
panDirFileId = rs.FileId
logger.Verbosef("创建云盘文件夹成功: %s\n", panDirPath)
} else {
logger.Verbosef("创建云盘文件夹错误: %s\n", er2.String())
return er2
}
} else {
if dirFile != nil && dirFile.FileId != "" {
panDirFileId = dirFile.FileId
}
}
// 计算文件SHA1
sha1Str := ""
proofCode := ""
contentHashName := "sha1"
if f.syncItem.LocalFile.Sha1Hash != "" {
sha1Str = f.syncItem.LocalFile.Sha1Hash
} else {
// 正常上传流程，检测是否能秒传
preHashMatch := true
if f.syncItem.LocalFile.FileSize >= panupload.DefaultCheckPreHashFileSize {
// 大文件，先计算 PreHash，用于检测是否可能支持秒传
preHash := panupload.CalcFilePreHash(f.syncItem.LocalFile.Path)
if len(preHash) > 0 {
if b, er := f.panClient.OpenapiPanClient().CheckUploadFilePreHash(&aliyunpan.FileUploadCheckPreHashParam{
DriveId: f.syncItem.DriveId,
Name: f.syncItem.LocalFile.FileName,
Size: f.syncItem.LocalFile.FileSize,
ParentFileId: panDirFileId,
PreHash: preHash,
}); er == nil {
preHashMatch = b
}
}
}
if preHashMatch {
// 再计算完整文件SHA1
logger.Verbosef("正在计算文件SHA1: %s\n", localFile.Path)
if localFile.Length == 0 {
sha1Str = aliyunpan.DefaultZeroSizeFileContentHash
@ -456,6 +500,17 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
}
f.syncItem.LocalFile.Sha1Hash = sha1Str
f.syncFileDb.Update(f.syncItem)
// 计算proof code
localFileEntity, _ := os.Open(localFile.Path.RealPath)
localFileInfo, _ := localFileEntity.Stat()
proofCode = aliyunpan.CalcProofCode(f.panClient.OpenapiPanClient().GetAccessToken(), rio.NewFileReaderAtLen64(localFileEntity), localFileInfo.Size())
} else {
// 无需计算 sha1，直接上传
logger.Verboseln("PreHash not match, upload file directly")
sha1Str = ""
contentHashName = ""
}
}
// 检查同名文件是否存在
@ -489,37 +544,6 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
}
}
// 创建文件夹
panDirPath := filepath.Dir(targetPanFilePath)
panDirFileId := ""
if dirFile, er2 := f.panClient.OpenapiPanClient().FileInfoByPath(f.syncItem.DriveId, panDirPath); er2 != nil {
if er2.Code == apierror.ApiCodeFileNotFoundCode {
logger.Verbosef("创建云盘文件夹: %s\n", panDirPath)
f.panFolderCreateMutex.Lock()
rs, apierr1 := f.panClient.OpenapiPanClient().MkdirByFullPath(f.syncItem.DriveId, panDirPath)
f.panFolderCreateMutex.Unlock()
if apierr1 != nil || rs.FileId == "" {
return apierr1
}
panDirFileId = rs.FileId
logger.Verbosef("创建云盘文件夹成功: %s\n", panDirPath)
} else {
logger.Verbosef("上传文件错误: %s\n", apierr.String())
return apierr
}
} else {
if dirFile != nil && dirFile.FileId != "" {
panDirFileId = dirFile.FileId
}
}
// 计算proof code
proofCode := ""
localFileEntity, _ := os.Open(localFile.Path.RealPath)
localFileInfo, _ := localFileEntity.Stat()
proofCode = aliyunpan.CalcProofCode(f.panClient.OpenapiPanClient().GetAccessToken(), rio.NewFileReaderAtLen64(localFileEntity), localFileInfo.Size())
// 自动调整BlockSize大小
newBlockSize := utils.ResizeUploadBlockSize(localFile.Length, f.syncItem.UploadBlockSize)
if newBlockSize != f.syncItem.UploadBlockSize {
@ -535,7 +559,7 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
Name: filepath.Base(targetPanFilePath),
Size: localFile.Length,
ContentHash: sha1Str,
ContentHashName: "sha1",
ContentHashName: contentHashName,
CheckNameMode: "refuse",
ParentFileId: panDirFileId,
BlockSize: f.syncItem.UploadBlockSize,

41
internal/syncdrive/file_action_task_mgr.go
View File

@ -4,9 +4,7 @@ import (
"context"
"fmt"
mapset "github.com/deckarep/golang-set"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/localfile"
"github.com/tickstep/aliyunpan/internal/plugins"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/aliyunpan/internal/waitgroup"
@ -256,26 +254,27 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
// 本地文件和云盘文件SHA1不一样
// 不同模式同步策略不一样
if f.task.Mode == UploadOnly {
// 计算本地文件SHA1
if localFile.Sha1Hash == "" {
// calc sha1
if localFile.FileSize == 0 {
localFile.Sha1Hash = aliyunpan.DefaultZeroSizeFileContentHash
} else {
fileSum := localfile.NewLocalFileEntity(localFile.Path)
err := fileSum.OpenPath()
if err != nil {
logger.Verbosef("文件不可读, 错误信息: %s, 跳过...\n", err)
continue
}
fileSum.Sum(localfile.CHECKSUM_SHA1) // block operation
localFile.Sha1Hash = fileSum.SHA1
fileSum.Close()
}
// save sha1 to local DB
f.task.localFileDb.Update(localFile)
}
// 不再这里计算SHA1，待到上传的时候再计算
//if localFile.Sha1Hash == "" {
// // 计算本地文件SHA1
// if localFile.FileSize == 0 {
// localFile.Sha1Hash = aliyunpan.DefaultZeroSizeFileContentHash
// } else {
// fileSum := localfile.NewLocalFileEntity(localFile.Path)
// err := fileSum.OpenPath()
// if err != nil {
// logger.Verbosef("文件不可读, 错误信息: %s, 跳过...\n", err)
// continue
// }
// fileSum.Sum(localfile.CHECKSUM_SHA1) // block operation
// localFile.Sha1Hash = fileSum.SHA1
// fileSum.Close()
// }
//
// // save sha1 to local DB
// f.task.localFileDb.Update(localFile)
//}
// 校验SHA1是否相同
if strings.ToLower(panFile.Sha1Hash) == strings.ToLower(localFile.Sha1Hash) {