Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-01-23 05:52:15 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add upload prepare hook

Browse Source
This commit is contained in:
xiaoyaofenfen 2022-04-20 14:14:18 +08:00
parent 70443455de
commit c1115dfcc7
9 changed files with 104 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmder/cmdutil/file.go
View File

@ -123,3 +123,8 @@ func WalkDir(dirPth, suffix string) (files []string, err error) {
func ConvertToUnixPathSeparator(p string) string {
return strings.Replace(p, "\\", "/", -1)
}
// ConvertToWindowsPathSeparator 将路径中的所有分隔符转换成windows格式
func ConvertToWindowsPathSeparator(p string) string {
return strings.Replace(p, "/", "\\", -1)
}

82
internal/command/upload.go
View File

@ -17,6 +17,7 @@ import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/cmder"
"github.com/tickstep/aliyunpan/internal/plugins"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/library-go/requester/rio/speeds"
"io/ioutil"
@ -54,16 +55,16 @@ const (
type (
// UploadOptions 上传可选项
UploadOptions struct {
AllParallel int // 所有文件并发上传数量，即可以同时并发上传多少个文件
Parallel int // 单个文件并发上传数量
MaxRetry int
NoRapidUpload bool
ShowProgress bool
IsOverwrite bool // 覆盖已存在的文件，如果同名文件已存在则移到回收站里
DriveId string
ExcludeNames []string // 排除的文件名，包括文件夹和文件。即这些文件/文件夹不进行上传，支持正则表达式
BlockSize int64 // 分片大小
UseInternalUrl bool // 是否使用内置链接
AllParallel int // 所有文件并发上传数量，即可以同时并发上传多少个文件
Parallel int // 单个文件并发上传数量
MaxRetry int
NoRapidUpload bool
ShowProgress bool
IsOverwrite bool // 覆盖已存在的文件，如果同名文件已存在则移到回收站里
DriveId string
ExcludeNames []string // 排除的文件名，包括文件夹和文件。即这些文件/文件夹不进行上传，支持正则表达式
BlockSize int64 // 分片大小
UseInternalUrl bool // 是否使用内置链接
}
)
@ -161,14 +162,14 @@ func CmdUpload() cli.Command {
subArgs := c.Args()
RunUpload(subArgs[:c.NArg()-1], subArgs[c.NArg()-1], &UploadOptions{
AllParallel: c.Int("p"), // 多文件上传的时候，允许同时并行上传的文件数量
Parallel: 1, // 一个文件同时多少个线程并发上传的数量。阿里云盘只支持单线程按顺序进行文件part数据上传，所以只能是1
Parallel: 1, // 一个文件同时多少个线程并发上传的数量。阿里云盘只支持单线程按顺序进行文件part数据上传，所以只能是1
MaxRetry: c.Int("retry"),
NoRapidUpload: c.Bool("norapid"),
ShowProgress: !c.Bool("np"),
IsOverwrite: c.Bool("ow"),
DriveId: parseDriveId(c),
ExcludeNames: c.StringSlice("exn"),
BlockSize: int64(c.Int("bs") * 1024),
DriveId: parseDriveId(c),
ExcludeNames: c.StringSlice("exn"),
BlockSize: int64(c.Int("bs") * 1024),
})
return nil
},
@ -296,6 +297,8 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
statistic = &panupload.UploadStatistic{}
folderCreateMutex = &sync.Mutex{}
pluginManger = plugins.NewPluginManager(config.GetConfigDir())
)
executor.SetParallel(opt.AllParallel)
statistic.StartTimer() // 开始计时
@ -303,6 +306,9 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
// 全局速度统计
globalSpeedsStat := &speeds.Speeds{}
// 获取当前插件
plugin, _ := pluginManger.GetPlugin()
// 遍历指定的文件并创建上传任务
for _, curPath := range localPaths {
var walkFunc filepath.WalkFunc
@ -329,7 +335,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
}
dbpath += string(os.PathSeparator) + BackupMetaDirName
if di, err := os.Stat(dbpath); err == nil && di.IsDir() {
db, err = panupload.OpenSyncDb(dbpath+string(os.PathSeparator) + "db", BackupMetaBucketName)
db, err = panupload.OpenSyncDb(dbpath+string(os.PathSeparator)+"db", BackupMetaBucketName)
if db != nil {
defer func(syncDb panupload.SyncDb) {
db.Close()
@ -345,6 +351,9 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
if err != nil {
return err
}
if os.PathSeparator == '\\' {
file = cmdutil.ConvertToWindowsPathSeparator(file)
}
// 是否排除上传
if isExcludeFile(file, opt) {
@ -401,6 +410,31 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
return filepath.SkipDir
}
// 插件回调
if !fi.IsDir() { // 针对文件上传前进行回调
pluginParam := &plugins.UploadFilePrepareParams{
LocalFilePath: file,
LocalFileName: fi.Name(),
LocalFileSize: fi.Size(),
LocalFileType: "file",
LocalFileUpdatedAt: fi.ModTime().Format("2006-01-02 15:04:05"),
DriveId: activeUser.ActiveDriveId,
DriveFilePath: strings.TrimPrefix(strings.TrimPrefix(subSavePath, savePath), "/"),
}
if uploadFilePrepareResult, er := plugin.UploadFilePrepareCallback(plugins.GetContext(activeUser), pluginParam); er == nil && uploadFilePrepareResult != nil {
if strings.Compare("yes", uploadFilePrepareResult.UploadApproved) != 0 {
// skip upload this file
fmt.Printf("插件禁止该文件上传: %s\n", file)
return filepath.SkipDir
}
if uploadFilePrepareResult.DriveFilePath != "" {
targetSavePanRelativePath := strings.TrimPrefix(uploadFilePrepareResult.DriveFilePath, "/")
subSavePath = path.Clean(savePath + aliyunpan.PathSeparator + targetSavePanRelativePath)
fmt.Printf("插件修改文件网盘保存路径为: %s\n", subSavePath)
}
}
}
taskinfo := executor.Append(&panupload.UploadTaskUnit{
LocalFileChecksum: localfile.NewLocalFileEntity(file),
SavePath: subSavePath,
@ -455,14 +489,14 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
// 是否是排除上传的文件
func isExcludeFile(filePath string, opt *UploadOptions) bool {
if opt == nil || len(opt.ExcludeNames) == 0{
if opt == nil || len(opt.ExcludeNames) == 0 {
return false
}
for _,pattern := range opt.ExcludeNames {
for _, pattern := range opt.ExcludeNames {
fileName := path.Base(filePath)
m,_ := regexp.MatchString(pattern, fileName)
m, _ := regexp.MatchString(pattern, fileName)
if m {
return true
}
@ -520,8 +554,8 @@ func RunRapidUpload(driveId string, isOverwrite bool, fileMetaList []string, sav
items := []*RapidUploadItem{}
// parse file meta strings
for _,fileMeta := range fileMetaList {
item,e := newRapidUploadItem(fileMeta)
for _, fileMeta := range fileMetaList {
item, e := newRapidUploadItem(fileMeta)
if e != nil {
fmt.Println(e)
continue
@ -539,7 +573,7 @@ func RunRapidUpload(driveId string, isOverwrite bool, fileMetaList []string, sav
}
// upload one by one
for _,item := range items {
for _, item := range items {
fmt.Println("准备秒传:", item.FilePath)
if ee := doRapidUpload(driveId, isOverwrite, item); ee != nil {
fmt.Println(ee)
@ -580,7 +614,7 @@ func doRapidUpload(driveId string, isOverwrite bool, item *RapidUploadItem) erro
}
if efi != nil && efi.FileId != "" {
// existed, delete it
fileDeleteResult, err1 := panClient.FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId:efi.DriveId, FileId:efi.FileId}})
fileDeleteResult, err1 := panClient.FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: efi.DriveId, FileId: efi.FileId}})
if err1 != nil || len(fileDeleteResult) == 0 {
return fmt.Errorf("无法删除文件，请稍后重试")
}
@ -604,11 +638,11 @@ func doRapidUpload(driveId string, isOverwrite bool, item *RapidUploadItem) erro
if apierr != nil {
return fmt.Errorf("创建秒传任务失败：" + apierr.Error())
}
if uploadOpEntity.RapidUpload {
logger.Verboseln("秒传成功, 保存到网盘路径: ", path.Join(panDir, uploadOpEntity.FileName))
} else {
return fmt.Errorf("失败，文件未曾上传，无法秒传")
}
return nil
}
}

2
internal/plugin/idle_plugin.go → internal/plugins/idle_plugin.go
View File

@ -1,4 +1,4 @@
package plugin
package plugins
type (
IdlePlugin struct {

22
internal/plugin/js_plugin.go → internal/plugins/js_plugin.go
View File

@ -1,4 +1,4 @@
package plugin
package plugins
import (
"fmt"
@ -9,17 +9,15 @@ import (
type (
JsPlugin struct {
Name string
vm *goja.Runtime
logger *logger.CmdVerbose
Name string
vm *goja.Runtime
}
)
func NewJsPlugin(log *logger.CmdVerbose) *JsPlugin {
func NewJsPlugin() *JsPlugin {
return &JsPlugin{
Name: "JsPlugin",
vm: nil,
logger: log,
Name: "JsPlugin",
vm: nil,
}
}
@ -55,22 +53,24 @@ func (js *JsPlugin) Start() error {
func (js *JsPlugin) LoadScript(script string) error {
_, err := js.vm.RunString(script)
if err != nil {
fmt.Println("JS代码有问题！")
logger.Verboseln("JS代码有问题！")
return err
}
return nil
}
// UploadFilePrepareCallback 上传文件前的回调函数
func (js *JsPlugin) UploadFilePrepareCallback(context *Context, params *UploadFilePrepareParams) (*UploadFilePrepareResult, error) {
var fn func(*Context, *UploadFilePrepareParams) (*UploadFilePrepareResult, error)
err := js.vm.ExportTo(js.vm.Get("uploadFilePrepareCallback"), &fn)
if err != nil {
fmt.Println("Js函数映射到 Go 函数失败！")
logger.Verboseln("Js函数映射到 Go 函数失败！")
return nil, nil
}
r, er := fn(context, params)
if er != nil {
fmt.Println(er)
logger.Verboseln(er)
return nil, er
}
return r, nil
}

4
internal/plugin/plugin.go → internal/plugins/plugin.go
View File

@ -11,7 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
package plugins
type (
// Context 插件回调函数上下文信息
@ -28,7 +28,7 @@ type (
UploadFilePrepareParams struct {
LocalFilePath string `json:"localFilePath"`
LocalFileName string `json:"localFileName"`
LocalFileSize int `json:"localFileSize"`
LocalFileSize int64 `json:"localFileSize"`
LocalFileType string `json:"localFileType"`
LocalFileUpdatedAt string `json:"localFileUpdatedAt"`
DriveId string `json:"driveId"`

27
internal/plugin/plugin_manager.go → internal/plugins/plugin_manager.go
View File

@ -1,4 +1,4 @@
package plugin
package plugins
import (
"fmt"
@ -14,14 +14,23 @@ import (
type (
PluginManager struct {
PluginPath string
log *logger.CmdVerbose
}
)
func NewPluginManager() *PluginManager {
func GetContext(user *config.PanUser) *Context {
return &Context{
AppName: "aliyunpan",
Version: config.AppVersion,
UserId: user.UserId,
Nickname: user.Nickname,
FileDriveId: user.DriveList.GetFileDriveId(),
AlbumDriveId: user.DriveList.GetFileDriveId(),
}
}
func NewPluginManager(pluginDir string) *PluginManager {
return &PluginManager{
PluginPath: "",
log: logger.New("PLUGIN", config.EnvVerbose),
PluginPath: pluginDir,
}
}
@ -35,11 +44,11 @@ func (p *PluginManager) SetPluginPath(pluginPath string) error {
}
func (p *PluginManager) GetPlugin() (Plugin, error) {
// js plugin folder
// only support js plugin right now
// js plugins folder
// only support js plugins right now
jsPluginPath := path.Clean(p.PluginPath + string(os.PathSeparator) + "js")
if fi, err := os.Stat(jsPluginPath); err == nil && fi.IsDir() {
jsPlugin := NewJsPlugin(p.log)
jsPlugin := NewJsPlugin()
if jsPlugin.Start() != nil {
logger.Verbosef("初始化JS脚本错误\n")
return interface{}(NewIdlePlugin()).(Plugin), nil
@ -70,6 +79,6 @@ func (p *PluginManager) GetPlugin() (Plugin, error) {
}
}
// default idle plugin
// default idle plugins
return interface{}(NewIdlePlugin()).(Plugin), nil
}

5
internal/plugin/plugin_manager_test.go → internal/plugins/plugin_manager_test.go
View File

@ -1,4 +1,4 @@
package plugin
package plugins
import (
"fmt"
@ -6,8 +6,7 @@ import (
)
func TestPlugin(t *testing.T) {
pluginManager := NewPluginManager()
pluginManager.SetPluginPath("D:\\smb\\feny\\goprojects\\dev")
pluginManager := NewPluginManager("D:\\smb\\feny\\goprojects\\dev")
plugin, err := pluginManager.GetPlugin()
if err != nil {
fmt.Println(err)

2
internal/plugin/plugin_util.go → internal/plugins/plugin_util.go
View File

@ -1,4 +1,4 @@
package plugin
package plugins
import (
"github.com/tickstep/library-go/logger"

12
internal/webdav/webdav.go
View File

@ -28,7 +28,7 @@ type Config struct {
LogFormat string
}
// ServeHTTP determines if the request is for this plugin, and if all prerequisites are met.
// ServeHTTP determines if the request is for this plugins, and if all prerequisites are met.
func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
u := c.User
requestOrigin := r.Header.Get("Origin")
@ -74,7 +74,7 @@ func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Gets the correct user for this request.
username, password, ok := r.BasicAuth()
logger.Verboseln("login attempt", "username = " + username, "remote_address = " + r.RemoteAddr)
logger.Verboseln("login attempt", "username = "+username, "remote_address = "+r.RemoteAddr)
if !ok {
http.Error(w, "Not authorized", 401)
return
@ -87,17 +87,17 @@ func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if !checkPassword(user.Password, password) {
logger.Verboseln("invalid password", "username = " + username, "remote_address = " + r.RemoteAddr)
logger.Verboseln("invalid password", "username = "+username, "remote_address = "+r.RemoteAddr)
http.Error(w, "Not authorized", 401)
return
}
u = user
logger.Verboseln("user authorized", "username = " + username)
logger.Verboseln("user authorized", "username = "+username)
} else {
// Even if Auth is disabled, we might want to get
// the user from the Basic Auth header. Useful for Caddy
// plugin implementation.
// plugins implementation.
username, _, ok := r.BasicAuth()
if ok {
if user, ok := c.Users[username]; ok {
@ -167,7 +167,7 @@ func addContextValue(r *http.Request) *http.Request {
contentLength = "0"
}
}
if cl,ok := strconv.ParseInt(contentLength, 10, 64);ok == nil{
if cl, ok := strconv.ParseInt(contentLength, 10, 64); ok == nil {
length = cl
}
}