mirror of
https://github.com/tickstep/aliyunpan.git
synced 2025-01-23 14:32:14 +08:00
remove backup command
This commit is contained in:
xiaoyaofenfen
parent
ac93c7e184
commit
b64c07f079
@ -1,306 +0,0 @@
|
||||
// Copyright (c) 2020 tickstep
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/tickstep/aliyunpan-api/aliyunpan"
|
||||
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
|
||||
"github.com/tickstep/aliyunpan/cmder"
|
||||
"github.com/tickstep/aliyunpan/internal/config"
|
||||
"github.com/tickstep/aliyunpan/internal/functions/panupload"
|
||||
"github.com/tickstep/library-go/logger"
|
||||
"github.com/urfave/cli"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func CmdBackup() cli.Command {
|
||||
return cli.Command{
|
||||
Name: "backup",
|
||||
Description: `备份指定 <文件/目录> 到云盘 <目标目录> 中
|
||||
该功能已过期，下个版本会删除，请使用sync命令替代。
|
||||
|
||||
和上传的功能一样，只是备份多进行了如下操作
|
||||
|
||||
1. 增加了数据库，记录已经上传的文件信息。
|
||||
目前只记录 文件位置、大小、修改时间、MD5 。
|
||||
2. 上传前先根据数据库记录判断是否需要重新上传。
|
||||
3. 强制同名覆盖。
|
||||
|
||||
注：只备份(上传)新的文件（同名覆盖），不处理删除操作。
|
||||
|
||||
示例:
|
||||
1. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录
|
||||
注意区别反斜杠 "\" 和 斜杠 "/" !!!
|
||||
aliyunpan-go backup C:/Users/Administrator/Video /视频
|
||||
|
||||
2. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录，但是排除所有的.jpg文件
|
||||
aliyunpan-go backup -exn "\.jpg$" C:/Users/Administrator/Video /视频
|
||||
|
||||
3. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录，但是排除所有的.jpg文件和.mp3文件，每一个排除项就是一个exn参数
|
||||
aliyunpan-go backup -exn "\.jpg$" -exn "\.mp3$" C:/Users/Administrator/Video /视频
|
||||
|
||||
4. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录，但是排除所有的 @eadir 文件夹
|
||||
aliyunpan-go backup -exn "^@eadir$" C:/Users/Administrator/Video /视频
|
||||
|
||||
参考：
|
||||
以下是典型的排除特定文件或者文件夹的例子，注意：参数值必须是正则表达式。在正则表达式中，^表示匹配开头，$表示匹配结尾。
|
||||
1)排除@eadir文件或者文件夹：-exn "^@eadir$"
|
||||
2)排除.jpg文件：-exn "\.jpg$"
|
||||
3)排除.号开头的文件：-exn "^\."
|
||||
4)排除~号开头的文件：-exn "^~"
|
||||
5)排除 myfile.txt 文件：-exn "^myfile.txt$"
|
||||
`,
|
||||
Usage: "备份文件或目录(Deprecated，请使用sync命令替代)",
|
||||
UsageText: "backup <文件/目录路径1> <文件/目录2> <文件/目录3> ... <目标目录>",
|
||||
Category: "阿里云盘",
|
||||
Before: cmder.ReloadConfigFunc,
|
||||
Action: Backup,
|
||||
Flags: append(UploadFlags, cli.BoolFlag{
|
||||
Name: "delete",
|
||||
Usage: "通过本地数据库记录同步删除网盘文件",
|
||||
}, cli.BoolFlag{
|
||||
Name: "sync",
|
||||
Usage: "本地同步到网盘（会同步删除网盘文件）",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func OpenSyncDb(path string) (panupload.SyncDb, error) {
|
||||
return panupload.OpenSyncDb(path, BackupMetaBucketName)
|
||||
}
|
||||
|
||||
// 删除那些本地不存在而网盘存在的网盘文件 默认使用本地数据库判断，如果 flagSync 为 true 则遍历网盘文件列表进行判断（速度较慢）。
|
||||
func DelRemoteFileFromDB(driveId string, localDir string, savePath string, flagSync bool) {
|
||||
activeUser := config.Config.ActiveUser()
|
||||
var db panupload.SyncDb
|
||||
var err error
|
||||
|
||||
dbpath := filepath.Join(localDir, BackupMetaDirName)
|
||||
db, err = OpenSyncDb(dbpath + string(os.PathSeparator) + "db")
|
||||
if err != nil {
|
||||
fmt.Println("同步数据库打开失败！", err)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
savePath = path.Join(savePath, filepath.Base(localDir))
|
||||
|
||||
//判断本地文件是否存在，如果存在返回 true 否则删除数据库相关记录和网盘上的文件。
|
||||
isLocalFileExist := func(ent *panupload.UploadedFileMeta) (isExists bool) {
|
||||
testPath := strings.TrimPrefix(ent.Path, savePath)
|
||||
testPath = filepath.Join(localDir, testPath)
|
||||
logger.Verboseln("同步删除检测:", testPath, ent.Path)
|
||||
|
||||
//为防止误删，只有当 err 是文件不存在的时候才进行删除处理。
|
||||
if fi, err := os.Stat(testPath); err == nil || !os.IsNotExist(err) {
|
||||
//使用sync功能时没有传时间参数进来，为方便对比回写数据库需补上时间。
|
||||
if fi != nil {
|
||||
ent.ModTime = fi.ModTime().Unix()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var err *apierror.ApiError
|
||||
|
||||
// 尝试从本地数据库查找
|
||||
if ent.ParentId == "" {
|
||||
if test := db.Get(path.Dir(ent.Path)); test != nil && test.IsFolder && test.FileId != "" {
|
||||
ent.ParentId = test.FileId
|
||||
}
|
||||
}
|
||||
|
||||
// 从网盘查找
|
||||
if ent.FileId == "" || ent.ParentId == "" {
|
||||
efi, err := activeUser.PanClient().FileInfoById(driveId, ent.FileId)
|
||||
//网盘上不存在这个文件或目录，只需要清理数据库
|
||||
if err != nil && err.Code == apierror.ApiCodeFileNotFoundCode {
|
||||
db.DelWithPrefix(ent.Path)
|
||||
logger.Verboseln("删除数据库记录", ent.Path)
|
||||
return
|
||||
}
|
||||
if efi != nil {
|
||||
ent.FileId = efi.FileId
|
||||
ent.ParentId = efi.ParentFileId
|
||||
}
|
||||
}
|
||||
|
||||
if ent.FileId == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// 本地文件不存在
|
||||
// 删除网盘对应文件
|
||||
fileDeleteResult, err := activeUser.PanClient().FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: driveId, FileId: ent.FileId}})
|
||||
if err != nil || len(fileDeleteResult) == 0 {
|
||||
fmt.Println("删除网盘文件或目录失败", ent.Path, err)
|
||||
} else {
|
||||
db.DelWithPrefix(ent.Path)
|
||||
logger.Verboseln("删除网盘文件和数据库记录", ent.Path)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 根据数据库记录删除不存在的文件
|
||||
if !flagSync {
|
||||
for ent, err := db.First(savePath); err == nil; ent, err = db.Next(savePath) {
|
||||
isLocalFileExist(ent)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
parent := db.Get(savePath)
|
||||
if parent.FileId == "" {
|
||||
efi, err := activeUser.PanClient().FileInfoByPath(driveId, savePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
parent.FileId = efi.FileId
|
||||
}
|
||||
|
||||
var syncFunc func(curPath, parentID string)
|
||||
|
||||
syncFunc = func(curPath, parentID string) {
|
||||
param := &aliyunpan.FileListParam{
|
||||
DriveId: driveId,
|
||||
ParentFileId: parentID,
|
||||
}
|
||||
fileResult, err := activeUser.PanClient().FileListGetAll(param)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if fileResult == nil || len(fileResult) == 0 {
|
||||
return
|
||||
}
|
||||
for _, fileEntity := range fileResult {
|
||||
ufm := &panupload.UploadedFileMeta{
|
||||
FileId: fileEntity.FileId,
|
||||
ParentId: fileEntity.ParentFileId,
|
||||
Size: fileEntity.FileSize,
|
||||
IsFolder: fileEntity.IsFolder(),
|
||||
Path: path.Join(curPath, fileEntity.FileName),
|
||||
SHA1: strings.ToLower(fileEntity.ContentHash),
|
||||
}
|
||||
|
||||
if !isLocalFileExist(ufm) {
|
||||
continue
|
||||
}
|
||||
|
||||
//如果这是一个目录就直接更新数据库，否则判断原始记录的Hash信息，如果一致才更新。
|
||||
if ufm.IsFolder {
|
||||
db.Put(ufm.Path, ufm)
|
||||
syncFunc(ufm.Path, ufm.FileId)
|
||||
} else if test := db.Get(ufm.Path); test.SHA1 == ufm.SHA1 {
|
||||
db.Put(ufm.Path, ufm)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//开启自动清理功能
|
||||
db.AutoClean(parent.Path, true)
|
||||
db.Put(parent.Path, parent)
|
||||
|
||||
syncFunc(savePath, parent.FileId)
|
||||
}
|
||||
|
||||
func checkPath(localdir string) (string, error) {
|
||||
fullPath, err := filepath.Abs(localdir)
|
||||
if err != nil {
|
||||
fullPath = localdir
|
||||
}
|
||||
|
||||
if fi, err := os.Stat(fullPath); err != nil && !fi.IsDir() {
|
||||
return fullPath, os.ErrInvalid
|
||||
}
|
||||
|
||||
dbpath := filepath.Join(fullPath, BackupMetaDirName)
|
||||
//数据库目录判断
|
||||
fi, err := os.Stat(dbpath)
|
||||
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
err = os.Mkdir(dbpath, 0755)
|
||||
}
|
||||
if err != nil {
|
||||
return fullPath, fmt.Errorf("数据库目录[%s]创建失败，跳过处理: %s", dbpath, err)
|
||||
}
|
||||
}
|
||||
|
||||
if fi != nil && !fi.IsDir() {
|
||||
return fullPath, os.ErrPermission
|
||||
}
|
||||
|
||||
return fullPath, nil
|
||||
}
|
||||
|
||||
func Backup(c *cli.Context) error {
|
||||
fmt.Println("该功能已过期，下个版本会删除，请使用sync命令替代。")
|
||||
|
||||
if c.NArg() < 2 {
|
||||
cli.ShowCommandHelp(c, c.Command.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
subArgs := c.Args()
|
||||
localpaths := make([]string, 0)
|
||||
flagSync := c.Bool("sync")
|
||||
flagDelete := c.Bool("delete")
|
||||
|
||||
opt := &UploadOptions{
|
||||
AllParallel: c.Int("p"),
|
||||
Parallel: 1, // 阿里云盘一个文件只支持单线程上传
|
||||
MaxRetry: c.Int("retry"),
|
||||
NoRapidUpload: c.Bool("norapid"),
|
||||
ShowProgress: !c.Bool("np"),
|
||||
IsOverwrite: true,
|
||||
DriveId: parseDriveId(c),
|
||||
ExcludeNames: c.StringSlice("exn"),
|
||||
BlockSize: int64(c.Int("bs") * 1024),
|
||||
}
|
||||
|
||||
localCount := c.NArg() - 1
|
||||
savePath := GetActiveUser().PathJoin(opt.DriveId, subArgs[localCount])
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(localCount)
|
||||
for _, p := range subArgs[:localCount] {
|
||||
go func(p string) {
|
||||
defer wg.Done()
|
||||
fullPath, err := checkPath(p)
|
||||
switch err {
|
||||
case nil:
|
||||
if flagSync || flagDelete {
|
||||
DelRemoteFileFromDB(opt.DriveId, fullPath, savePath, flagSync)
|
||||
}
|
||||
case os.ErrInvalid:
|
||||
default:
|
||||
return
|
||||
}
|
||||
localpaths = append(localpaths, fullPath)
|
||||
}(p)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if len(localpaths) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
RunUpload(localpaths, savePath, opt)
|
||||
return nil
|
||||
}
|
||||
@ -312,7 +312,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
// 遍历指定的文件并创建上传任务
|
||||
for _, curPath := range localPaths {
|
||||
var walkFunc filepath.WalkFunc
|
||||
var db panupload.SyncDb
|
||||
curPath = filepath.Clean(curPath)
|
||||
localPathDir := filepath.Dir(curPath)
|
||||
|
||||
@ -327,26 +326,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
localPathDir = ""
|
||||
}
|
||||
|
||||
if fi, err := os.Stat(curPath); err == nil && fi.IsDir() {
|
||||
//使用绝对路径避免异常
|
||||
dbpath, err := filepath.Abs(curPath)
|
||||
if err != nil {
|
||||
dbpath = curPath
|
||||
}
|
||||
dbpath += string(os.PathSeparator) + BackupMetaDirName
|
||||
if di, err := os.Stat(dbpath); err == nil && di.IsDir() {
|
||||
db, err = panupload.OpenSyncDb(dbpath+string(os.PathSeparator)+"db", BackupMetaBucketName)
|
||||
if db != nil {
|
||||
defer func(syncDb panupload.SyncDb) {
|
||||
db.Close()
|
||||
}(db)
|
||||
} else {
|
||||
fmt.Println(curPath, "同步数据库打开失败,跳过该目录的备份", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walkFunc = func(file string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
@ -374,41 +353,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
}
|
||||
|
||||
subSavePath = path.Clean(savePath + aliyunpan.PathSeparator + subSavePath)
|
||||
var ufm *panupload.UploadedFileMeta
|
||||
|
||||
if db != nil {
|
||||
if ufm = db.Get(subSavePath); ufm.Size == fi.Size() && ufm.ModTime == fi.ModTime().Unix() {
|
||||
logger.Verbosef("文件未修改跳过:%s\n", file)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if fi.IsDir() { // 备份目录处理
|
||||
if strings.HasPrefix(fi.Name(), BackupMetaDirName) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
//不存在同步数据库时跳过
|
||||
if db == nil || ufm.FileId != "" {
|
||||
return nil
|
||||
}
|
||||
panClient := activeUser.PanClient()
|
||||
fmt.Println(subSavePath, "云盘文件夹预创建")
|
||||
//首先尝试直接创建文件夹
|
||||
if ufm = db.Get(path.Dir(subSavePath)); ufm.IsFolder == true && ufm.FileId != "" {
|
||||
rs, err := panClient.Mkdir(opt.DriveId, ufm.FileId, fi.Name())
|
||||
if err == nil && rs != nil && rs.FileId != "" {
|
||||
db.Put(subSavePath, &panupload.UploadedFileMeta{FileId: rs.FileId, IsFolder: true, ModTime: fi.ModTime().Unix(), ParentId: rs.ParentFileId})
|
||||
return nil
|
||||
}
|
||||
}
|
||||
rs, err := panClient.MkdirRecursive(opt.DriveId, "", "", 0, strings.Split(path.Clean(subSavePath), "/"))
|
||||
if err == nil && rs != nil && rs.FileId != "" {
|
||||
db.Put(subSavePath, &panupload.UploadedFileMeta{FileId: rs.FileId, IsFolder: true, ModTime: fi.ModTime().Unix(), ParentId: rs.ParentFileId})
|
||||
return nil
|
||||
}
|
||||
fmt.Println(subSavePath, "创建云盘文件夹失败", err)
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
// 插件回调
|
||||
if !fi.IsDir() { // 针对文件上传前进行回调
|
||||
@ -433,7 +377,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
fmt.Printf("插件修改文件网盘保存路径为: %s\n", subSavePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
taskinfo := executor.Append(&panupload.UploadTaskUnit{
|
||||
LocalFileChecksum: localfile.NewLocalFileEntity(file),
|
||||
@ -448,12 +391,11 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
UploadStatistic: statistic,
|
||||
ShowProgress: opt.ShowProgress,
|
||||
IsOverwrite: opt.IsOverwrite,
|
||||
FolderSyncDb: db,
|
||||
UseInternalUrl: opt.UseInternalUrl,
|
||||
GlobalSpeedsStat: globalSpeedsStat,
|
||||
}, opt.MaxRetry)
|
||||
|
||||
fmt.Printf("[%s] 加入上传队列: %s\n", taskinfo.Id(), file)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := WalkAllFile(curPath, walkFunc); err != nil {
|
||||
@ -494,8 +436,7 @@ func isExcludeFile(filePath string, opt *UploadOptions) bool {
|
||||
}
|
||||
|
||||
for _, pattern := range opt.ExcludeNames {
|
||||
fileName := path.Base(filePath)
|
||||
|
||||
fileName := path.Base(strings.ReplaceAll(filePath, "\\", "/"))
|
||||
m, _ := regexp.MatchString(pattern, fileName)
|
||||
if m {
|
||||
return true
|
||||
@ -519,13 +460,13 @@ func walkAllFile(dirPath string, info os.FileInfo, walkFn filepath.WalkFunc) err
|
||||
return walkFn(dirPath, info, nil)
|
||||
}
|
||||
|
||||
files, err := ioutil.ReadDir(dirPath)
|
||||
if err != nil {
|
||||
return walkFn(dirPath, nil, err)
|
||||
files, err1 := ioutil.ReadDir(dirPath)
|
||||
if err1 != nil {
|
||||
return walkFn(dirPath, nil, err1)
|
||||
}
|
||||
for _, fi := range files {
|
||||
subFilePath := dirPath + "/" + fi.Name()
|
||||
err := walkFn(subFilePath, fi, err)
|
||||
err := walkFn(subFilePath, fi, err1)
|
||||
if err != nil && err != filepath.SkipDir {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -29,14 +29,6 @@ var (
|
||||
panCommandVerbose = logger.New("PANCOMMAND", config.EnvVerbose)
|
||||
)
|
||||
|
||||
const(
|
||||
// 备份数据库桶分区标志
|
||||
BackupMetaBucketName = "adrive"
|
||||
|
||||
// 备份数据文件夹目录名称，隐藏目录
|
||||
BackupMetaDirName = ".adrive"
|
||||
)
|
||||
|
||||
// GetFileInfoByPaths 获取指定文件路径的文件详情信息
|
||||
func GetAppFileInfoByPaths(driveId string, paths ...string) (fileInfoList []*aliyunpan.FileEntity, failedPaths []string, error error) {
|
||||
if len(paths) <= 0 {
|
||||
|
||||
@ -1,46 +0,0 @@
|
||||
// Copyright (c) 2020 tickstep & chenall
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package panupload
|
||||
|
||||
type SyncDb interface {
|
||||
//读取记录,返回值不会是nil
|
||||
Get(key string) (ufm *UploadedFileMeta)
|
||||
//删除单条记录
|
||||
Del(key string) error
|
||||
//根据前辍删除数据库记录，比如删除一个目录时可以连同子目录一起删除
|
||||
DelWithPrefix(prefix string) error
|
||||
Put(key string, value *UploadedFileMeta) error
|
||||
Close() error
|
||||
//读取数据库指定路径前辍的第一条记录（也作为循环获取的初始化，配置Next函数使用)
|
||||
First(prefix string) (*UploadedFileMeta, error)
|
||||
//获取指定路径前辍的的下一条记录
|
||||
Next(prefix string) (*UploadedFileMeta, error)
|
||||
//是否进行自动数据库清理
|
||||
//注: 清理规则，所有以 prefix 前辍开头并且未更新的记录都将被清理，只有在必要的时候才开启这个功能。
|
||||
AutoClean(prefix string, cleanFlag bool)
|
||||
}
|
||||
|
||||
type autoCleanInfo struct {
|
||||
PreFix string
|
||||
SyncTime int64
|
||||
}
|
||||
|
||||
func OpenSyncDb(file string, bucket string) (SyncDb, error) {
|
||||
return openBoltDb(file, bucket)
|
||||
}
|
||||
|
||||
type dbTableField struct {
|
||||
Path string
|
||||
Data []byte
|
||||
}
|
||||
@ -1,185 +0,0 @@
|
||||
// Copyright (c) 2020 tickstep & chenall
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package panupload
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/tickstep/bolt"
|
||||
"github.com/tickstep/library-go/logger"
|
||||
"time"
|
||||
)
|
||||
|
||||
type boltDB struct {
|
||||
db *bolt.DB
|
||||
bucket string
|
||||
next map[string]*boltDBScan
|
||||
cleanInfo *autoCleanInfo
|
||||
}
|
||||
|
||||
type boltDBScan struct {
|
||||
entries []*boltKV
|
||||
off int
|
||||
size int
|
||||
}
|
||||
|
||||
type boltKV struct {
|
||||
k []byte
|
||||
v []byte
|
||||
}
|
||||
|
||||
func openBoltDb(file string, bucket string) (SyncDb, error) {
|
||||
db, err := bolt.Open(file+"_bolt.db", 0755, &bolt.Options{Timeout: 5 * time.Second})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Verboseln("open boltDB ok")
|
||||
return &boltDB{db: db, bucket: bucket, next: make(map[string]*boltDBScan)}, nil
|
||||
}
|
||||
|
||||
func (db *boltDB) Get(key string) (data *UploadedFileMeta) {
|
||||
data = &UploadedFileMeta{Path: key}
|
||||
db.db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte(db.bucket))
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
v := b.Get([]byte(key))
|
||||
return jsoniter.Unmarshal(v, data)
|
||||
})
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func (db *boltDB) Del(key string) error {
|
||||
return db.db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte(db.bucket))
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
return b.Delete([]byte(key))
|
||||
})
|
||||
}
|
||||
|
||||
func (db *boltDB) AutoClean(prefix string, cleanFlag bool) {
|
||||
if !cleanFlag {
|
||||
db.cleanInfo = nil
|
||||
} else if db.cleanInfo == nil {
|
||||
db.cleanInfo = &autoCleanInfo{
|
||||
PreFix: prefix,
|
||||
SyncTime: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (db *boltDB) clean() (count uint) {
|
||||
for ufm, err := db.First(db.cleanInfo.PreFix); err == nil; ufm, err = db.Next(db.cleanInfo.PreFix) {
|
||||
if ufm.LastSyncTime != db.cleanInfo.SyncTime {
|
||||
db.DelWithPrefix(ufm.Path)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (db *boltDB) DelWithPrefix(prefix string) error {
|
||||
return db.db.Update(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte(db.bucket))
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
c := b.Cursor()
|
||||
for k, _ := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, _ = c.Next() {
|
||||
b.Delete(k)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (db *boltDB) First(prefix string) (*UploadedFileMeta, error) {
|
||||
db.db.View(func(tx *bolt.Tx) error {
|
||||
b := tx.Bucket([]byte(db.bucket))
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
c := b.Cursor()
|
||||
db.next[prefix] = &boltDBScan{
|
||||
entries: []*boltKV{},
|
||||
off: 0,
|
||||
size: 0,
|
||||
}
|
||||
for k, v := c.Seek([]byte(prefix)); k != nil && bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() {
|
||||
//fmt.Printf("key=%s, value=%s\n", k, v)
|
||||
if len(k) > 0 {
|
||||
db.next[prefix].entries = append(db.next[prefix].entries, &boltKV{
|
||||
k: k,
|
||||
v: v,
|
||||
})
|
||||
}
|
||||
}
|
||||
db.next[prefix].off = 0
|
||||
db.next[prefix].size = len(db.next[prefix].entries)
|
||||
return nil
|
||||
})
|
||||
return db.Next(prefix)
|
||||
}
|
||||
|
||||
func (db *boltDB) Next(prefix string) (*UploadedFileMeta, error) {
|
||||
data := &UploadedFileMeta{}
|
||||
if _, ok := db.next[prefix]; ok {
|
||||
if db.next[prefix].off >= db.next[prefix].size {
|
||||
return nil, fmt.Errorf("no any more record")
|
||||
}
|
||||
kv := db.next[prefix].entries[db.next[prefix].off]
|
||||
db.next[prefix].off++
|
||||
if kv != nil {
|
||||
jsoniter.Unmarshal(kv.v, &data)
|
||||
data.Path = string(kv.k)
|
||||
return data, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no any more record")
|
||||
}
|
||||
|
||||
func (db *boltDB) Put(key string, value *UploadedFileMeta) error {
|
||||
if db.cleanInfo != nil {
|
||||
value.LastSyncTime = db.cleanInfo.SyncTime
|
||||
}
|
||||
|
||||
return db.db.Update(func(tx *bolt.Tx) error {
|
||||
data, err := jsoniter.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b := tx.Bucket([]byte(db.bucket))
|
||||
if b == nil {
|
||||
b, err = tx.CreateBucket([]byte(db.bucket))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return b.Put([]byte(key), data)
|
||||
})
|
||||
}
|
||||
|
||||
func (db *boltDB) Close() error {
|
||||
if db.cleanInfo != nil {
|
||||
db.clean()
|
||||
}
|
||||
if db.db != nil {
|
||||
return db.db.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -40,17 +40,6 @@ type (
|
||||
useInternalUrl bool
|
||||
}
|
||||
|
||||
UploadedFileMeta struct {
|
||||
IsFolder bool `json:"isFolder,omitempty"` // 是否目录
|
||||
Path string `json:"-"` // 本地路径，不记录到数据库
|
||||
SHA1 string `json:"sha1,omitempty"` // 文件的 SHA1
|
||||
FileId string `json:"id,omitempty"` //文件、目录ID
|
||||
ParentId string `json:"parentId,omitempty"` //父文件夹ID
|
||||
Size int64 `json:"length,omitempty"` // 文件大小
|
||||
ModTime int64 `json:"modtime,omitempty"` // 修改日期
|
||||
LastSyncTime int64 `json:"synctime,omitempty"` //最后同步时间
|
||||
}
|
||||
|
||||
EmptyReaderLen64 struct {
|
||||
}
|
||||
)
|
||||
|
||||
@ -49,7 +49,6 @@ type (
|
||||
SavePath string // 保存路径
|
||||
DriveId string // 网盘ID，例如：文件网盘，相册网盘
|
||||
FolderCreateMutex *sync.Mutex
|
||||
FolderSyncDb SyncDb //文件备份状态数据库
|
||||
|
||||
PanClient *aliyunpan.PanClient
|
||||
UploadingDatabase *UploadingDatabase // 数据库
|
||||
@ -240,29 +239,6 @@ func (utu *UploadTaskUnit) OnRetry(lastRunResult *taskframework.TaskUnitRunResul
|
||||
func (utu *UploadTaskUnit) OnSuccess(lastRunResult *taskframework.TaskUnitRunResult) {
|
||||
// 执行插件
|
||||
utu.pluginCallback("success")
|
||||
|
||||
//文件上传成功
|
||||
if utu.FolderSyncDb == nil || lastRunResult == ResultLocalFileNotUpdated { //不需要更新数据库
|
||||
return
|
||||
}
|
||||
ufm := &UploadedFileMeta{
|
||||
IsFolder: false,
|
||||
SHA1: utu.LocalFileChecksum.SHA1,
|
||||
ModTime: utu.LocalFileChecksum.ModTime,
|
||||
Size: utu.LocalFileChecksum.Length,
|
||||
}
|
||||
|
||||
if utu.LocalFileChecksum.UploadOpEntity != nil {
|
||||
ufm.FileId = utu.LocalFileChecksum.UploadOpEntity.FileId
|
||||
ufm.ParentId = utu.LocalFileChecksum.UploadOpEntity.ParentFileId
|
||||
} else {
|
||||
efi, _ := utu.PanClient.FileInfoByPath(utu.DriveId, utu.SavePath)
|
||||
if efi != nil {
|
||||
ufm.FileId = efi.FileId
|
||||
ufm.ParentId = efi.ParentFileId
|
||||
}
|
||||
}
|
||||
utu.FolderSyncDb.Put(utu.SavePath, ufm)
|
||||
}
|
||||
|
||||
func (utu *UploadTaskUnit) OnFailed(lastRunResult *taskframework.TaskUnitRunResult) {
|
||||
@ -295,9 +271,6 @@ func (utu *UploadTaskUnit) pluginCallback(result string) {
|
||||
}
|
||||
}
|
||||
|
||||
var ResultLocalFileNotUpdated = &taskframework.TaskUnitRunResult{ResultCode: 1, Succeed: true, ResultMessage: "本地文件未更新，无需上传！"}
|
||||
var ResultUpdateLocalDatabase = &taskframework.TaskUnitRunResult{ResultCode: 2, Succeed: true, ResultMessage: "本地文件和云端文件MD5一致，无需上传！"}
|
||||
|
||||
func (utu *UploadTaskUnit) OnComplete(lastRunResult *taskframework.TaskUnitRunResult) {
|
||||
// 任务结束，可能成功也可能失败
|
||||
}
|
||||
@ -343,14 +316,10 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
|
||||
var contentHashName string
|
||||
var checkNameMode string
|
||||
var saveFilePath string
|
||||
var testFileMeta = &UploadedFileMeta{}
|
||||
var uploadOpEntity *aliyunpan.CreateFileUploadResult
|
||||
var proofCode = ""
|
||||
var localFileInfo os.FileInfo
|
||||
var localFile *os.File
|
||||
timeStart2 := time.Now()
|
||||
timeStart3 := time.Now()
|
||||
timeStart4 := time.Now()
|
||||
|
||||
switch utu.Step {
|
||||
case StepUploadPrepareUpload:
|
||||
@ -363,50 +332,32 @@ func (utu *UploadTaskUnit) Run() (result *taskframework.TaskUnitRunResult) {
|
||||
|
||||
StepUploadPrepareUpload:
|
||||
// 创建上传任务
|
||||
if utu.FolderSyncDb != nil {
|
||||
//启用了备份功能，强制使用覆盖同名文件功能
|
||||
utu.IsOverwrite = true
|
||||
testFileMeta = utu.FolderSyncDb.Get(utu.SavePath)
|
||||
}
|
||||
|
||||
// 创建云盘文件夹
|
||||
timeStart2 = time.Now()
|
||||
// utu.FolderCreateMutex.Lock()
|
||||
saveFilePath = path.Dir(utu.SavePath)
|
||||
if saveFilePath != "/" {
|
||||
fmt.Printf("[%s] %s 正在检测和创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
|
||||
//同步功能先尝试从数据库获取
|
||||
if utu.FolderSyncDb != nil {
|
||||
timeStart3 = time.Now()
|
||||
utu.FolderCreateMutex.Lock()
|
||||
if test := utu.FolderSyncDb.Get(saveFilePath); test.FileId != "" && test.IsFolder {
|
||||
rs = &aliyunpan.MkdirResult{FileId: test.FileId}
|
||||
}
|
||||
utu.FolderCreateMutex.Unlock()
|
||||
logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕[from db], 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart3)))
|
||||
}
|
||||
if rs == nil {
|
||||
timeStart4 = time.Now()
|
||||
fe, apierr1 := utu.PanClient.FileInfoByPath(utu.DriveId, saveFilePath)
|
||||
time.Sleep(1 * time.Second)
|
||||
if apierr1 != nil && apierr1.Code == apierror.ApiCodeFileNotFoundCode {
|
||||
logger.Verbosef("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
|
||||
utu.FolderCreateMutex.Lock()
|
||||
// rs, apierr = utu.PanClient.MkdirRecursive(utu.DriveId, "", "", 0, strings.Split(path.Clean(saveFilePath), "/"))
|
||||
// 可以直接创建的，不用循环创建
|
||||
rs, apierr = utu.PanClient.Mkdir(utu.DriveId, "root", saveFilePath)
|
||||
utu.FolderCreateMutex.Unlock()
|
||||
if apierr != nil || rs.FileId == "" {
|
||||
result.Err = apierr
|
||||
result.ResultMessage = "创建云盘文件夹失败"
|
||||
return
|
||||
}
|
||||
logger.Verbosef("[%s] %s 创建云盘文件夹, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart4)))
|
||||
logger.Verbosef("[%s] %s 创建云盘文件夹成功\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"))
|
||||
} else {
|
||||
rs = &aliyunpan.MkdirResult{}
|
||||
rs.FileId = fe.FileId
|
||||
}
|
||||
} else {
|
||||
rs = &aliyunpan.MkdirResult{}
|
||||
rs.FileId = ""
|
||||
}
|
||||
// time.Sleep(time.Duration(2) * time.Second)
|
||||
// utu.FolderCreateMutex.Unlock()
|
||||
logger.Verbosef("[%s] %s 检测和创建云盘文件夹完毕, 耗时 %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utils.ConvertTime(time.Now().Sub(timeStart2)))
|
||||
time.Sleep(time.Duration(2) * time.Second)
|
||||
|
||||
sha1Str = ""
|
||||
proofCode = ""
|
||||
@ -416,9 +367,6 @@ StepUploadPrepareUpload:
|
||||
// 计算文件SHA1
|
||||
fmt.Printf("[%s] %s 正在计算文件SHA1: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), utu.LocalFileChecksum.Path)
|
||||
utu.LocalFileChecksum.Sum(localfile.CHECKSUM_SHA1)
|
||||
if testFileMeta.SHA1 == utu.LocalFileChecksum.SHA1 {
|
||||
return ResultUpdateLocalDatabase
|
||||
}
|
||||
sha1Str = utu.LocalFileChecksum.SHA1
|
||||
if utu.LocalFileChecksum.Length == 0 {
|
||||
sha1Str = aliyunpan.DefaultZeroSizeFileContentHash
|
||||
|
||||
58
main.go
58
main.go
@ -420,9 +420,6 @@ func main() {
|
||||
|
||||
//// 拷贝文件/目录 cp
|
||||
//command.CmdCp(),
|
||||
//
|
||||
//// 拷贝文件/目录到个人云/家庭云 xcp
|
||||
//command.CmdXcp(),
|
||||
|
||||
// 移动文件/目录 mv
|
||||
command.CmdMv(),
|
||||
@ -433,9 +430,6 @@ func main() {
|
||||
// 分享文件/目录 share
|
||||
command.CmdShare(),
|
||||
|
||||
// 备份 backup
|
||||
command.CmdBackup(),
|
||||
|
||||
// 同步备份 sync
|
||||
command.CmdSync(),
|
||||
|
||||
@ -592,32 +586,32 @@ func main() {
|
||||
},
|
||||
|
||||
// 调试用 debug
|
||||
{
|
||||
Name: "debug",
|
||||
Aliases: []string{"dg"},
|
||||
Usage: "开发调试用",
|
||||
Description: "",
|
||||
Category: "debug",
|
||||
Before: cmder.ReloadConfigFunc,
|
||||
Action: func(c *cli.Context) error {
|
||||
os.Setenv(config.EnvVerbose, "1")
|
||||
logger.IsVerbose = true
|
||||
fmt.Println("显示调试日志", logger.IsVerbose)
|
||||
return nil
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "param",
|
||||
Usage: "参数",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
Destination: &logger.IsVerbose,
|
||||
EnvVar: config.EnvVerbose,
|
||||
Usage: "显示调试信息",
|
||||
},
|
||||
},
|
||||
},
|
||||
//{
|
||||
// Name: "debug",
|
||||
// Aliases: []string{"dg"},
|
||||
// Usage: "开发调试用",
|
||||
// Description: "",
|
||||
// Category: "debug",
|
||||
// Before: cmder.ReloadConfigFunc,
|
||||
// Action: func(c *cli.Context) error {
|
||||
// os.Setenv(config.EnvVerbose, "1")
|
||||
// logger.IsVerbose = true
|
||||
// fmt.Println("显示调试日志", logger.IsVerbose)
|
||||
// return nil
|
||||
// },
|
||||
// Flags: []cli.Flag{
|
||||
// cli.StringFlag{
|
||||
// Name: "param",
|
||||
// Usage: "参数",
|
||||
// },
|
||||
// cli.BoolFlag{
|
||||
// Name: "verbose",
|
||||
// Destination: &logger.IsVerbose,
|
||||
// EnvVar: config.EnvVerbose,
|
||||
// Usage: "显示调试信息",
|
||||
// },
|
||||
// },
|
||||
//},
|
||||
}
|
||||
|
||||
sort.Sort(cli.FlagsByName(app.Flags))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user