diff --git a/internal/command/backup.go b/internal/command/backup.go deleted file mode 100644 index 5763ed7..0000000 --- a/internal/command/backup.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (c) 2020 tickstep -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package command - -import ( - "fmt" - "github.com/tickstep/aliyunpan-api/aliyunpan" - "github.com/tickstep/aliyunpan-api/aliyunpan/apierror" - "github.com/tickstep/aliyunpan/cmder" - "github.com/tickstep/aliyunpan/internal/config" - "github.com/tickstep/aliyunpan/internal/functions/panupload" - "github.com/tickstep/library-go/logger" - "github.com/urfave/cli" - "os" - "path" - "path/filepath" - "strings" - "sync" -) - -func CmdBackup() cli.Command { - return cli.Command{ - Name: "backup", - Description: `备份指定 <文件/目录> 到云盘 <目标目录> 中 -该功能已过期，下个版本会删除，请使用sync命令替代。 - -和上传的功能一样，只是备份多进行了如下操作 - -1. 增加了数据库，记录已经上传的文件信息。 - 目前只记录 文件位置、大小、修改时间、MD5 。 -2. 上传前先根据数据库记录判断是否需要重新上传。 -3. 强制同名覆盖。 - -注：只备份(上传)新的文件（同名覆盖），不处理删除操作。 - - 示例: - 1. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录 - 注意区别反斜杠 "\" 和 斜杠 "/" !!! - aliyunpan-go backup C:/Users/Administrator/Video /视频 - - 2. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录，但是排除所有的.jpg文件 - aliyunpan-go backup -exn "\.jpg$" C:/Users/Administrator/Video /视频 - - 3. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录，但是排除所有的.jpg文件和.mp3文件，每一个排除项就是一个exn参数 - aliyunpan-go backup -exn "\.jpg$" -exn "\.mp3$" C:/Users/Administrator/Video /视频 - - 4. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录，但是排除所有的 @eadir 文件夹 - aliyunpan-go backup -exn "^@eadir$" C:/Users/Administrator/Video /视频 - - 参考： - 以下是典型的排除特定文件或者文件夹的例子，注意：参数值必须是正则表达式。在正则表达式中，^表示匹配开头，$表示匹配结尾。 - 1)排除@eadir文件或者文件夹：-exn "^@eadir$" - 2)排除.jpg文件：-exn "\.jpg$" - 3)排除.号开头的文件：-exn "^\." - 4)排除~号开头的文件：-exn "^~" - 5)排除 myfile.txt 文件：-exn "^myfile.txt$" -`, - Usage: "备份文件或目录(Deprecated，请使用sync命令替代)", - UsageText: "backup <文件/目录路径1> <文件/目录2> <文件/目录3> ... <目标目录>", - Category: "阿里云盘", - Before: cmder.ReloadConfigFunc, - Action: Backup, - Flags: append(UploadFlags, cli.BoolFlag{ - Name: "delete", - Usage: "通过本地数据库记录同步删除网盘文件", - }, cli.BoolFlag{ - Name: "sync", - Usage: "本地同步到网盘（会同步删除网盘文件）", - }), - } -} - -func OpenSyncDb(path string) (panupload.SyncDb, error) { - return panupload.OpenSyncDb(path, BackupMetaBucketName) -} - -// 删除那些本地不存在而网盘存在的网盘文件 默认使用本地数据库判断，如果 flagSync 为 true 则遍历网盘文件列表进行判断（速度较慢）。 -func DelRemoteFileFromDB(driveId string, localDir string, savePath string, flagSync bool) { - activeUser := config.Config.ActiveUser() - var db panupload.SyncDb - var err error - - dbpath := filepath.Join(localDir, BackupMetaDirName) - db, err = OpenSyncDb(dbpath + string(os.PathSeparator) + "db") - if err != nil { - fmt.Println("同步数据库打开失败！", err) - return - } - defer db.Close() - - savePath = path.Join(savePath, filepath.Base(localDir)) - - //判断本地文件是否存在，如果存在返回 true 否则删除数据库相关记录和网盘上的文件。 - isLocalFileExist := func(ent *panupload.UploadedFileMeta) (isExists bool) { - testPath := strings.TrimPrefix(ent.Path, savePath) - testPath = filepath.Join(localDir, testPath) - logger.Verboseln("同步删除检测:", testPath, ent.Path) - - //为防止误删，只有当 err 是文件不存在的时候才进行删除处理。 - if fi, err := os.Stat(testPath); err == nil || !os.IsNotExist(err) { - //使用sync功能时没有传时间参数进来，为方便对比回写数据库需补上时间。 - if fi != nil { - ent.ModTime = fi.ModTime().Unix() - } - return true - } - - var err *apierror.ApiError - - // 尝试从本地数据库查找 - if ent.ParentId == "" { - if test := db.Get(path.Dir(ent.Path)); test != nil && test.IsFolder && test.FileId != "" { - ent.ParentId = test.FileId - } - } - - // 从网盘查找 - if ent.FileId == "" || ent.ParentId == "" { - efi, err := activeUser.PanClient().FileInfoById(driveId, ent.FileId) - //网盘上不存在这个文件或目录，只需要清理数据库 - if err != nil && err.Code == apierror.ApiCodeFileNotFoundCode { - db.DelWithPrefix(ent.Path) - logger.Verboseln("删除数据库记录", ent.Path) - return - } - if efi != nil { - ent.FileId = efi.FileId - ent.ParentId = efi.ParentFileId - } - } - - if ent.FileId == "" { - return - } - - // 本地文件不存在 - // 删除网盘对应文件 - fileDeleteResult, err := activeUser.PanClient().FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: driveId, FileId: ent.FileId}}) - if err != nil || len(fileDeleteResult) == 0 { - fmt.Println("删除网盘文件或目录失败", ent.Path, err) - } else { - db.DelWithPrefix(ent.Path) - logger.Verboseln("删除网盘文件和数据库记录", ent.Path) - } - return - } - - // 根据数据库记录删除不存在的文件 - if !flagSync { - for ent, err := db.First(savePath); err == nil; ent, err = db.Next(savePath) { - isLocalFileExist(ent) - } - return - } - - parent := db.Get(savePath) - if parent.FileId == "" { - efi, err := activeUser.PanClient().FileInfoByPath(driveId, savePath) - if err != nil { - return - } - parent.FileId = efi.FileId - } - - var syncFunc func(curPath, parentID string) - - syncFunc = func(curPath, parentID string) { - param := &aliyunpan.FileListParam{ - DriveId: driveId, - ParentFileId: parentID, - } - fileResult, err := activeUser.PanClient().FileListGetAll(param) - if err != nil { - return - } - if fileResult == nil || len(fileResult) == 0 { - return - } - for _, fileEntity := range fileResult { - ufm := &panupload.UploadedFileMeta{ - FileId: fileEntity.FileId, - ParentId: fileEntity.ParentFileId, - Size: fileEntity.FileSize, - IsFolder: fileEntity.IsFolder(), - Path: path.Join(curPath, fileEntity.FileName), - SHA1: strings.ToLower(fileEntity.ContentHash), - } - - if !isLocalFileExist(ufm) { - continue - } - - //如果这是一个目录就直接更新数据库，否则判断原始记录的Hash信息，如果一致才更新。 - if ufm.IsFolder { - db.Put(ufm.Path, ufm) - syncFunc(ufm.Path, ufm.FileId) - } else if test := db.Get(ufm.Path); test.SHA1 == ufm.SHA1 { - db.Put(ufm.Path, ufm) - } - } - } - - //开启自动清理功能 - db.AutoClean(parent.Path, true) - db.Put(parent.Path, parent) - - syncFunc(savePath, parent.FileId) -} - -func checkPath(localdir string) (string, error) { - fullPath, err := filepath.Abs(localdir) - if err != nil { - fullPath = localdir - } - - if fi, err := os.Stat(fullPath); err != nil && !fi.IsDir() { - return fullPath, os.ErrInvalid - } - - dbpath := filepath.Join(fullPath, BackupMetaDirName) - //数据库目录判断 - fi, err := os.Stat(dbpath) - - if err != nil { - if os.IsNotExist(err) { - err = os.Mkdir(dbpath, 0755) - } - if err != nil { - return fullPath, fmt.Errorf("数据库目录[%s]创建失败，跳过处理: %s", dbpath, err) - } - } - - if fi != nil && !fi.IsDir() { - return fullPath, os.ErrPermission - } - - return fullPath, nil -} - -func Backup(c *cli.Context) error { - fmt.Println("该功能已过期，下个版本会删除，请使用sync命令替代。") - - if c.NArg() < 2 { - cli.ShowCommandHelp(c, c.Command.Name) - return nil - } - - subArgs := c.Args() - localpaths := make([]string, 0) - flagSync := c.Bool("sync") - flagDelete := c.Bool("delete") - - opt := &UploadOptions{ - AllParallel: c.Int("p"), - Parallel: 1, // 阿里云盘一个文件只支持单线程上传 - MaxRetry: c.Int("retry"), - NoRapidUpload: c.Bool("norapid"), - ShowProgress: !c.Bool("np"), - IsOverwrite: true, - DriveId: parseDriveId(c), - ExcludeNames: c.StringSlice("exn"), - BlockSize: int64(c.Int("bs") * 1024), - } - - localCount := c.NArg() - 1 - savePath := GetActiveUser().PathJoin(opt.DriveId, subArgs[localCount]) - - wg := sync.WaitGroup{} - wg.Add(localCount) - for _, p := range subArgs[:localCount] { - go func(p string) { - defer wg.Done() - fullPath, err := checkPath(p) - switch err { - case nil: - if flagSync || flagDelete { - DelRemoteFileFromDB(opt.DriveId, fullPath, savePath, flagSync) - } - case os.ErrInvalid: - default: - return - } - localpaths = append(localpaths, fullPath) - }(p) - } - - wg.Wait() - - if len(localpaths) == 0 { - return nil - } - - RunUpload(localpaths, savePath, opt) - return nil -} diff --git a/internal/command/upload.go b/internal/command/upload.go index 23908e6..7940584 100644 --- a/internal/command/upload.go +++ b/internal/command/upload.go @@ -312,7 +312,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { // 遍历指定的文件并创建上传任务 for _, curPath := range localPaths { var walkFunc filepath.WalkFunc - var db panupload.SyncDb curPath = filepath.Clean(curPath) localPathDir := filepath.Dir(curPath) @@ -327,26 +326,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { localPathDir = "" } - if fi, err := os.Stat(curPath); err == nil && fi.IsDir() { - //使用绝对路径避免异常 - dbpath, err := filepath.Abs(curPath) - if err != nil { - dbpath = curPath - } - dbpath += string(os.PathSeparator) + BackupMetaDirName - if di, err := os.Stat(dbpath); err == nil && di.IsDir() { - db, err = panupload.OpenSyncDb(dbpath+string(os.PathSeparator)+"db", BackupMetaBucketName) - if db != nil { - defer func(syncDb panupload.SyncDb) { - db.Close() - }(db) - } else { - fmt.Println(curPath, "同步数据库打开失败,跳过该目录的备份", err) - continue - } - } - } - walkFunc = func(file string, fi os.FileInfo, err error) error { if err != nil { return err @@ -374,41 +353,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { } subSavePath = path.Clean(savePath + aliyunpan.PathSeparator + subSavePath) - var ufm *panupload.UploadedFileMeta - - if db != nil { - if ufm = db.Get(subSavePath); ufm.Size == fi.Size() && ufm.ModTime == fi.ModTime().Unix() { - logger.Verbosef("文件未修改跳过:%s\n", file) - return nil - } - } - - if fi.IsDir() { // 备份目录处理 - if strings.HasPrefix(fi.Name(), BackupMetaDirName) { - return filepath.SkipDir - } - //不存在同步数据库时跳过 - if db == nil || ufm.FileId != "" { - return nil - } - panClient := activeUser.PanClient() - fmt.Println(subSavePath, "云盘文件夹预创建") - //首先尝试直接创建文件夹 - if ufm = db.Get(path.Dir(subSavePath)); ufm.IsFolder == true && ufm.FileId != "" { - rs, err := panClient.Mkdir(opt.DriveId, ufm.FileId, fi.Name()) - if err == nil && rs != nil && rs.FileId != "" { - db.Put(subSavePath, &panupload.UploadedFileMeta{FileId: rs.FileId, IsFolder: true, ModTime: fi.ModTime().Unix(), ParentId: rs.ParentFileId}) - return nil - } - } - rs, err := panClient.MkdirRecursive(opt.DriveId, "", "", 0, strings.Split(path.Clean(subSavePath), "/")) - if err == nil && rs != nil && rs.FileId != "" { - db.Put(subSavePath, &panupload.UploadedFileMeta{FileId: rs.FileId, IsFolder: true, ModTime: fi.ModTime().Unix(), ParentId: rs.ParentFileId}) - return nil - } - fmt.Println(subSavePath, "创建云盘文件夹失败", err) - return filepath.SkipDir - } // 插件回调 if !fi.IsDir() { // 针对文件上传前进行回调 @@ -433,27 +377,25 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { fmt.Printf("插件修改文件网盘保存路径为: %s\n", subSavePath) } } + + taskinfo := executor.Append(&panupload.UploadTaskUnit{ + LocalFileChecksum: localfile.NewLocalFileEntity(file), + SavePath: subSavePath, + DriveId: opt.DriveId, + PanClient: activeUser.PanClient(), + UploadingDatabase: uploadDatabase, + FolderCreateMutex: folderCreateMutex, + Parallel: opt.Parallel, + NoRapidUpload: opt.NoRapidUpload, + BlockSize: opt.BlockSize, + UploadStatistic: statistic, + ShowProgress: opt.ShowProgress, + IsOverwrite: opt.IsOverwrite, + UseInternalUrl: opt.UseInternalUrl, + GlobalSpeedsStat: globalSpeedsStat, + }, opt.MaxRetry) + fmt.Printf("[%s] 加入上传队列: %s\n", taskinfo.Id(), file) } - - taskinfo := executor.Append(&panupload.UploadTaskUnit{ - LocalFileChecksum: localfile.NewLocalFileEntity(file), - SavePath: subSavePath, - DriveId: opt.DriveId, - PanClient: activeUser.PanClient(), - UploadingDatabase: uploadDatabase, - FolderCreateMutex: folderCreateMutex, - Parallel: opt.Parallel, - NoRapidUpload: opt.NoRapidUpload, - BlockSize: opt.BlockSize, - UploadStatistic: statistic, - ShowProgress: opt.ShowProgress, - IsOverwrite: opt.IsOverwrite, - FolderSyncDb: db, - UseInternalUrl: opt.UseInternalUrl, - GlobalSpeedsStat: globalSpeedsStat, - }, opt.MaxRetry) - - fmt.Printf("[%s] 加入上传队列: %s\n", taskinfo.Id(), file) return nil } if err := WalkAllFile(curPath, walkFunc); err != nil { @@ -494,8 +436,7 @@ func isExcludeFile(filePath string, opt *UploadOptions) bool { } for _, pattern := range opt.ExcludeNames { - fileName := path.Base(filePath) - + fileName := path.Base(strings.ReplaceAll(filePath, "\\", "/")) m, _ := regexp.MatchString(pattern, fileName) if m { return true @@ -519,13 +460,13 @@ func walkAllFile(dirPath string, info os.FileInfo, walkFn filepath.WalkFunc) err return walkFn(dirPath, info, nil) } - files, err := ioutil.ReadDir(dirPath) - if err != nil { - return walkFn(dirPath, nil, err) + files, err1 := ioutil.ReadDir(dirPath) + if err1 != nil { + return walkFn(dirPath, nil, err1) } for _, fi := range files { subFilePath := dirPath + "/" + fi.Name() - err := walkFn(subFilePath, fi, err) + err := walkFn(subFilePath, fi, err1) if err != nil && err != filepath.SkipDir { return err } diff --git a/internal/command/utils.go b/internal/command/utils.go index 962d23d..c6846cc 100644 --- a/internal/command/utils.go +++ b/internal/command/utils.go @@ -29,14 +29,6 @@ var ( panCommandVerbose = logger.New("PANCOMMAND", config.EnvVerbose) ) -const( - // 备份数据库桶分区标志 - BackupMetaBucketName = "adrive" - - // 备份数据文件夹目录名称，隐藏目录 - BackupMetaDirName = ".adrive" -) - // GetFileInfoByPaths 获取指定文件路径的文件详情信息 func GetAppFileInfoByPaths(driveId string, paths ...string) (fileInfoList []*aliyunpan.FileEntity, failedPaths []string, error error) { if len(paths) <= 0 { @@ -89,7 +81,7 @@ func RandomStr(count int) string { STR_SET := "abcdefjhijklmnopqrstuvwxyz1234567890" rand.Seed(time.Now().UnixNano()) str := strings.Builder{} - for i := 0; i < count; i++ { + for i := 0; i < count; i++ { str.WriteByte(byte(STR_SET[rand.Intn(len(STR_SET))])) } return str.String() @@ -100,7 +92,7 @@ func GetAllPathFolderByPath(pathStr string) []string { dirs := []string{} p := "/" dirs = append(dirs, p) - for _,s := range dirNames { + for _, s := range dirNames { p = path.Join(p, s) dirs = append(dirs, p) } @@ -114,7 +106,7 @@ func EscapeStr(s string) string { // UnescapeStr 反转义字符串 func UnescapeStr(s string) string { - r,_ := url.PathUnescape(s) + r, _ := url.PathUnescape(s) return r } @@ -143,4 +135,4 @@ func RefreshTokenInNeed(activeUser *config.PanUser) bool { } } return false -} \ No newline at end of file +} diff --git a/internal/functions/panupload/sync_database.go b/internal/functions/panupload/sync_database.go deleted file mode 100644 index 93ac6a0..0000000 --- a/internal/functions/panupload/sync_database.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2020 tickstep & chenall -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package panupload - -type SyncDb interface { - //读取记录,返回值不会是nil - Get(key string) (ufm *UploadedFileMeta) - //删除单条记录 - Del(key string) error - //根据前辍删除数据库记录，比如删除一个目录时可以连同子目录一起删除 - DelWithPrefix(prefix string) error - Put(key string, value *UploadedFileMeta) error - Close() error - //读取数据库指定路径前辍的第一条记录（也作为循环获取的初始化，配置Next函数使用) - First(prefix string) (*UploadedFileMeta, error) - //获取指定路径前辍的的下一条记录 - Next(prefix string) (*UploadedFileMeta, error) - //是否进行自动数据库清理 - //注: 清理规则，所有以 prefix 前辍开头并且未更新的记录都将被清理，只有在必要的时候才开启这个功能。 - AutoClean(prefix string, cleanFlag bool) -} - -type autoCleanInfo struct { - PreFix string - SyncTime int64 -} - -func OpenSyncDb(file string, bucket string) (SyncDb, error) { - return openBoltDb(file, bucket) -} - -type dbTableField struct { - Path string - Data []byte -} diff --git a/internal/functions/panupload/sync_database_bolt.go b/internal/functions/panupload/sync_database_bolt.go deleted file mode 100644 index d82175d..0000000 --- a/internal/functions/panupload/sync_database_bolt.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) 2020 tickstep & chenall -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package panupload - -import ( - "bytes" - "fmt" - jsoniter "github.com/json-iterator/go" - "github.com/tickstep/bolt" - "github.com/tickstep/library-go/logger" - "time" -) - -type boltDB struct { - db *bolt.DB - bucket string - next map[string]*boltDBScan - cleanInfo *autoCleanInfo -} - -type boltDBScan struct { - entries []*boltKV - off int - size int -} - -type boltKV struct { - k []byte - v []byte -} - -func openBoltDb(file string, bucket string) (SyncDb, error) { - db, err := bolt.Open(file+"_bolt.db", 0755, &bolt.Options{Timeout: 5 * time.Second}) - - if err != nil { - return nil, err - } - logger.Verboseln("open boltDB ok") - return &boltDB{db: db, bucket: bucket, next: make(map[string]*boltDBScan)}, nil -} - -func (db *boltDB) Get(key string) (data *UploadedFileMeta) { - data = &UploadedFileMeta{Path: key} - db.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(db.bucket)) - if b == nil { - return nil - } - v := b.Get([]byte(key)) - return jsoniter.Unmarshal(v, data) - }) - - return data -} - -func (db *boltDB) Del(key string) error { - return db.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(db.bucket)) - if b == nil { - return nil - } - return b.Delete([]byte(key)) - }) -} - -func (db *boltDB) AutoClean(prefix string, cleanFlag bool) { - if !cleanFlag { - db.cleanInfo = nil - } else if db.cleanInfo == nil { - db.cleanInfo = &autoCleanInfo{ - PreFix: prefix, - SyncTime: time.Now().Unix(), - } - } -} - -func (db *boltDB) clean() (count uint) { - for ufm, err := db.First(db.cleanInfo.PreFix); err == nil; ufm, err = db.Next(db.cleanInfo.PreFix) { - if ufm.LastSyncTime != db.cleanInfo.SyncTime { - db.DelWithPrefix(ufm.Path) - } - } - return -} - -func (db *boltDB) DelWithPrefix(prefix string) error { - return db.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(db.bucket)) - if b == nil { - return nil - } - c := b.Cursor() - for k, _ := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, _ = c.Next() { - b.Delete(k) - } - return nil - }) -} - -func (db *boltDB) First(prefix string) (*UploadedFileMeta, error) { - db.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(db.bucket)) - if b == nil { - return nil - } - c := b.Cursor() - db.next[prefix] = &boltDBScan{ - entries: []*boltKV{}, - off: 0, - size: 0, - } - for k, v := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() { - //fmt.Printf("key=%s, value=%s\n", k, v) - if len(k) > 0 { - db.next[prefix].entries = append(db.next[prefix].entries, &boltKV{ - k: k, - v: v, - }) - } - } - db.next[prefix].off = 0 - db.next[prefix].size = len(db.next[prefix].entries) - return nil - }) - return db.Next(prefix) -} - -func (db *boltDB) Next(prefix string) (*UploadedFileMeta, error) { - data := &UploadedFileMeta{} - if _, ok := db.next[prefix]; ok { - if db.next[prefix].off >= db.next[prefix].size { - return nil, fmt.Errorf("no any more record") - } - kv := db.next[prefix].entries[db.next[prefix].off] - db.next[prefix].off++ - if kv != nil { - jsoniter.Unmarshal(kv.v, &data) - data.Path = string(kv.k) - return data, nil - } - } - return nil, fmt.Errorf("no any more record") -} - -func (db *boltDB) Put(key string, value *UploadedFileMeta) error { - if db.cleanInfo != nil { - value.LastSyncTime = db.cleanInfo.SyncTime - } - - return db.db.Update(func(tx *bolt.Tx) error { - data, err := jsoniter.Marshal(value) - if err != nil { - return err - } - b := tx.Bucket([]byte(db.bucket)) - if b == nil { - b, err = tx.CreateBucket([]byte(db.bucket)) - if err != nil { - return err - } - } - return b.Put([]byte(key), data) - }) -} - -func (db *boltDB) Close() error { - if db.cleanInfo != nil { - db.clean() - } - if db.db != nil { - return db.db.Close() - } - return nil -} diff --git a/internal/functions/panupload/upload.go b/internal/functions/panupload/upload.go index 5c86586..96e0046 100644 --- a/internal/functions/panupload/upload.go +++ b/internal/functions/panupload/upload.go @@ -40,17 +40,6 @@ type ( useInternalUrl bool } - UploadedFileMeta struct { - IsFolder bool `json:"isFolder,omitempty"` // 是否目录 - Path string `json:"-"` // 本地路径，不记录到数据库 - SHA1 string `json:"sha1,omitempty"` // 文件的 SHA1 - FileId string `json:"id,omitempty"` //文件、目录ID - ParentId string `json:"parentId,omitempty"` //父文件夹ID - Size int64 `json:"length,omitempty"` // 文件大小 - ModTime int64 `json:"modtime,omitempty"` // 修改日期 - LastSyncTime int64 `json:"synctime,omitempty"` //最后同步时间 - } - EmptyReaderLen64 struct { } ) diff --git a/internal/functions/panupload/upload_task_unit.go b/internal/functions/panupload/upload_task_unit.go index 8f0f2a4..82f9a0d 100644 --- a/internal/functions/panupload/upload_task_unit.go +++ b/internal/functions/panupload/upload_task_unit.go @@ -49,7 +49,6 @@ type ( SavePath string // 保存路径 DriveId string // 网盘ID，例如：文件网盘，相册网盘 FolderCreateMutex *sync.Mutex - FolderSyncDb SyncDb //文件备份状态数据库 PanClient *aliyunpan.PanClient UploadingDatabase *UploadingDatabase // 数据库 @@ -240,29 +239,6 @@ func (utu *UploadTaskUnit) OnRetry(lastRunResult *taskframework.TaskUnitRunResul func (utu *UploadTaskUnit) OnSuccess(lastRunResult *taskframework.TaskUnitRunResult) { // 执行插件 utu.pluginCallback("success") - - //文件上传成功 - if utu.FolderSyncDb == nil || lastRunResult == ResultLocalFileNotUpdated { //不需要更新数据库 - return - } - ufm := &UploadedFileMeta{ - IsFolder: false, - SHA1: utu.LocalFileChecksum.SHA1, - ModTime: utu.LocalFileChecksum.ModTime, - Size: utu.LocalFileChecksum.Length, - } - - if utu.LocalFileChecksum.UploadOpEntity != nil { - ufm.FileId = utu.LocalFileChecksum.UploadOpEntity.FileId - ufm.ParentId = utu.LocalFileChecksum.UploadOpEntity.ParentFileId - } else { - efi, _ := utu.PanClient.FileInfoByPath(utu.DriveId, utu.SavePath) - if efi != nil { - ufm.FileId = efi.FileId - ufm.ParentId = efi.ParentFileId - } - } - utu.FolderSyncDb.Put(utu.SavePath, ufm) } func (utu *UploadTaskUnit) OnFailed(lastRunResult *taskframework.TaskUnitRunResult) { @@ -295,9 +271,6 @@ func (utu *UploadTaskUnit) pluginCallback(result string) { } } -var ResultLocalFileNotUpdated = &taskframework.TaskUnitRunResult{ResultCode: 1, Succeed: true, ResultMessage: "本地文件未更新，无需上传！"} -var ResultUpdateLocalDatabase = &taskframework.TaskUnitRunResult{ResultCode: 2, Succeed: true, ResultMessage: "本地文件和云端文件MD5一致，无需上传！"} - func (utu *UploadTaskUnit) OnComplete(lastRunResult *taskframework.TaskUnitRunResult) { // 任务结束，可能成功也可能失败 } @@ -343,14 +316,10 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { var contentHashName string var checkNameMode string var saveFilePath string - var testFileMeta = &UploadedFileMeta{} var uploadOpEntity *aliyunpan.CreateFileUploadResult var proofCode = "" var localFileInfo os.FileInfo var localFile *os.File - timeStart2 := time.Now() - timeStart3 := time.Now() - timeStart4 := time.Now() switch utu.Step { case StepUploadPrepareUpload: @@ -363,50 +332,32 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) { StepUploadPrepareUpload: // 创建上传任务 - if utu.FolderSyncDb != nil { - //启用了备份功能，强制使用覆盖同名文件功能 - utu.IsOverwrite = true - testFileMeta = utu.FolderSyncDb.Get(utu.SavePath) - } - // 创建云盘文件夹 - timeStart2 = time.Now() - // utu.FolderCreateMutex.Lock() saveFilePath = path.Dir(utu.SavePath) if saveFilePath != "/" { fmt.Printf("[%s] %s 正在检测和创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath) - //同步功能先尝试从数据库获取 - if utu.FolderSyncDb != nil { - timeStart3 = time.Now() - utu.FolderCreateMutex.Lock() - if test := utu.FolderSyncDb.Get(saveFilePath); test.FileId != "" && test.IsFolder { - rs = &aliyunpan.MkdirResult{FileId: test.FileId} - } - utu.FolderCreateMutex.Unlock() - logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕[from db], 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart3))) - } - if rs == nil { - timeStart4 = time.Now() + fe, apierr1 := utu.PanClient.FileInfoByPath(utu.DriveId, saveFilePath) + time.Sleep(1 * time.Second) + if apierr1 != nil && apierr1.Code == apierror.ApiCodeFileNotFoundCode { logger.Verbosef("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath) - utu.FolderCreateMutex.Lock() // rs, apierr = utu.PanClient.MkdirRecursive(utu.DriveId, "", "", 0, strings.Split(path.Clean(saveFilePath), "/")) // 可以直接创建的，不用循环创建 rs, apierr = utu.PanClient.Mkdir(utu.DriveId, "root", saveFilePath) - utu.FolderCreateMutex.Unlock() if apierr != nil || rs.FileId == "" { result.Err = apierr result.ResultMessage = "创建云盘文件夹失败" return } - logger.Verbosef("[%s] %s 创建云盘文件夹, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart4))) + logger.Verbosef("[%s] %s 创建云盘文件夹成功\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06")) + } else { + rs = &aliyunpan.MkdirResult{} + rs.FileId = fe.FileId } } else { rs = &aliyunpan.MkdirResult{} rs.FileId = "" } - // time.Sleep(time.Duration(2) * time.Second) - // utu.FolderCreateMutex.Unlock() - logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart2))) + time.Sleep(time.Duration(2) * time.Second) sha1Str = "" proofCode = "" @@ -416,9 +367,6 @@ StepUploadPrepareUpload: // 计算文件SHA1 fmt.Printf("[%s] %s 正在计算文件SHA1: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utu.LocalFileChecksum.Path) utu.LocalFileChecksum.Sum(localfile.CHECKSUM_SHA1) - if testFileMeta.SHA1 == utu.LocalFileChecksum.SHA1 { - return ResultUpdateLocalDatabase - } sha1Str = utu.LocalFileChecksum.SHA1 if utu.LocalFileChecksum.Length == 0 { sha1Str = aliyunpan.DefaultZeroSizeFileContentHash diff --git a/main.go b/main.go index b6517e5..fd35d30 100644 --- a/main.go +++ b/main.go @@ -420,9 +420,6 @@ func main() { //// 拷贝文件/目录 cp //command.CmdCp(), - // - //// 拷贝文件/目录到个人云/家庭云 xcp - //command.CmdXcp(), // 移动文件/目录 mv command.CmdMv(), @@ -433,9 +430,6 @@ func main() { // 分享文件/目录 share command.CmdShare(), - // 备份 backup - command.CmdBackup(), - // 同步备份 sync command.CmdSync(), @@ -592,32 +586,32 @@ func main() { }, // 调试用 debug - { - Name: "debug", - Aliases: []string{"dg"}, - Usage: "开发调试用", - Description: "", - Category: "debug", - Before: cmder.ReloadConfigFunc, - Action: func(c *cli.Context) error { - os.Setenv(config.EnvVerbose, "1") - logger.IsVerbose = true - fmt.Println("显示调试日志", logger.IsVerbose) - return nil - }, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "param", - Usage: "参数", - }, - cli.BoolFlag{ - Name: "verbose", - Destination: &logger.IsVerbose, - EnvVar: config.EnvVerbose, - Usage: "显示调试信息", - }, - }, - }, + //{ + // Name: "debug", + // Aliases: []string{"dg"}, + // Usage: "开发调试用", + // Description: "", + // Category: "debug", + // Before: cmder.ReloadConfigFunc, + // Action: func(c *cli.Context) error { + // os.Setenv(config.EnvVerbose, "1") + // logger.IsVerbose = true + // fmt.Println("显示调试日志", logger.IsVerbose) + // return nil + // }, + // Flags: []cli.Flag{ + // cli.StringFlag{ + // Name: "param", + // Usage: "参数", + // }, + // cli.BoolFlag{ + // Name: "verbose", + // Destination: &logger.IsVerbose, + // EnvVar: config.EnvVerbose, + // Usage: "显示调试信息", + // }, + // }, + //}, } sort.Sort(cli.FlagsByName(app.Flags))