Compare commits

...
50 Commits
Author SHA1 Message Date
HFO4 14f5982b47 Modify: set node_js version to node_js in travis.yml 2020-05-24 11:31:00 +08:00
HFO4 e607311268 Update: version number for 3.1.0 2020-05-24 10:47:30 +08:00
HFO4 acc5d53bab Update: submodules 2020-05-24 10:36:48 +08:00
HFO4 aa3e8913ab Feat: add ".zip" suffix if not specified while creating compressed file 2020-05-24 10:36:26 +08:00
HFO4 ee0f8e964d Feat: eject internal static files 2020-05-23 13:57:02 +08:00
HFO4 60745ac8ba Merge remote-tracking branch 'origin/master' 2020-05-23 13:19:10 +08:00
HFO4 bfb5b34edc Fix: concurrent logging lock / Feat: listen SSL (#287) 2020-05-23 13:17:48 +08:00
hiCasperandGitHub a5000c0621 Modify: update actions (#398) 2020-05-20 11:48:00 +08:00
HFO4 e038350cf0 Merge branch 'master' of https://github.com/cloudreve/Cloudreve 2020-05-19 11:25:12 +08:00
HFO4 5af3c4e244 Fix: directory renaming should not be limited by file extensions (#395) 2020-05-19 11:25:01 +08:00
AaronLiuandGitHub 7ed14c4d81 Update FUNDING.yml 2020-05-19 11:12:45 +08:00
AaronLiuandGitHub 869c0006c5 Create FUNDING.yml 2020-05-19 11:11:56 +08:00
HFO4 4c458df666 Fix: failed tests while static files not loaded 2020-05-19 10:34:16 +08:00
HFO4 ed684420a2 Modify: actions should not perform clean before build 2020-05-19 10:26:24 +08:00
HFO4 2076d56f0f Feat: dynamic writing site title, favicon, description, custom html (#286) 2020-05-19 10:24:15 +08:00
HFO4 280308bc05 Modify: increase OneDrive client uploading chunk size to 100MB 2020-05-13 09:09:11 +08:00
HFO4 1172765c58 Feat: option for using SSL connection in mail queue 2020-05-11 09:29:19 +08:00
HFO4 58856612e2 Feat: create empty file in web panel (#305) 2020-05-09 10:35:18 +08:00
HFO4 ee0f224cbb Feat: use Monaco code editor 2020-05-03 13:16:40 +08:00
HFO4 e8a6df9a86 Feat: import file from existing outer folder 2020-05-02 10:22:28 +08:00
HFO4 b02d27ca0a Merge branch 'master' of https://github.com/cloudreve/Cloudreve 2020-05-02 10:15:57 +08:00
TSandGitHub 51f66eb06b Feat: ICP 备案信息 (#348) 2020-05-01 13:50:16 +08:00
HFO4 0df8a9ba65 Modify: search shares without login required 2020-05-01 09:33:59 +08:00
HFO4 79daf92896 Test: List files in OneDrive 2020-05-01 09:22:27 +08:00
HFO4 8a2be58ef3 Feat: List files in OneDrive 2020-04-30 11:45:35 +08:00
HFO4 ce7784090f Feat: List files in upyun bucket 2020-04-30 10:37:48 +08:00
HFO4 dfb663a6e0 Feat: List files in Qiniu bucket 2020-04-29 10:37:15 +08:00
HFO4 cfaf20926f Feat: List files in COS bucket 2020-04-29 10:36:52 +08:00
HFO4 305497e7cb Test: oss.List 2020-04-29 09:18:26 +08:00
HFO4 52c2422be9 Feat: list oss files 2020-04-28 11:43:32 +08:00
HFO4 1afc750dae Modify: create folder if not exist while importing files 2020-04-28 10:47:40 +08:00
HFO4 960c886496 Test: slave side list file 2020-04-28 10:16:04 +08:00
HFO4 5d579cdadc Feat: slave side list file 2020-04-28 10:02:53 +08:00
HFO4 d5fc5745b4 Fix: failed test due to missing AdminCreateImportTask controller 2020-04-27 14:45:27 +08:00
HFO4 a732025d5a Update: submodules 2020-04-27 14:40:09 +08:00
HFO4 030fd4ac57 Test: task.ImportTask 2020-04-27 14:39:49 +08:00
HFO4 9eeb4b6d19 Feat: task for importing file from existing filesystem 2020-04-27 14:39:16 +08:00
HFO4 36e5b31f73 Modify: add recursive options for list method
Test: local.List
2020-04-27 10:31:34 +08:00
cha0sCatandGitHub 26d4d34837 Feat: webdav 适配 rclone (#330)
rclone在传输时会反复创建父目录, 导致cloudreve返回409, 最终导致传输失败.
因此检测如果是rclone则忽略文目录已存在错误
2020-04-27 10:01:00 +08:00
HFO4 8c547a05fd Fix: mismatched pointer receiver in oss.Driver 2020-04-24 11:14:32 +08:00
HFO4 f7311f906b Feat: add List method to Handler interface / implement local.List 2020-04-24 11:08:07 +08:00
HFO4 6006ff4d22 Feat: regenerate when thumbnail files not exist 2020-04-24 10:18:01 +08:00
ihipopandGitHub 034ed956a3 Bad syntax for struct tag value authn (#314) 2020-04-23 09:30:19 +08:00
hiCasperandGitHub 700c5795f5 Support Github Actions (#310) 2020-04-22 11:15:26 +08:00
HFO4 8c3287d380 Test: Policy.IsThumbExist / filesystem.AddFile 2020-04-22 11:10:43 +08:00
HFO4 304e7b502c Feat: thumbnail support for COS policy 2020-04-22 11:09:33 +08:00
HFO4 514e069113 Refactor: decide if there is a thumbnail based on the file extension 2020-04-22 11:04:35 +08:00
HFO4 7b571499a7 Feat: forcibly delete files in dashboard (#277) 2020-04-21 09:47:54 +08:00
HFO4 8b68d46bdf Update: submodules 2020-04-19 10:12:26 +08:00
HFO4 ab3b59e63d Feat: add option to force delete file record in database 2020-04-19 10:09:16 +08:00
58 changed files with 2156 additions and 49 deletions
+1
View File
@@ -0,0 +1 @@
custom: ["https://cloudreve.org/buy.php"]
+88
View File
@@ -0,0 +1,88 @@
name: Build
on:
push:
branches: [ master ]
jobs:
test:
name: Test
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Get dependencies
run: |
go get github.com/rakyll/statik
export PATH=$PATH:~/go/bin/
statik -src=models -f
- name: Test
run: go test -coverprofile=coverage.txt -covermode=atomic ./...
build:
name: Build
needs: test
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
with:
clean: false
submodules: 'recursive'
- run: |
git fetch --prune --unshallow --tags
- name: Get dependencies and build
run: |
go get github.com/rakyll/statik
export PATH=$PATH:~/go/bin/
statik -src=models -f
sudo apt-get update
sudo apt-get -y install gcc-mingw-w64-x86-64
sudo apt-get -y install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
sudo apt-get -y install gcc-aarch64-linux-gnu libc6-dev-arm64-cross
chmod +x ./build.sh
./build.sh -r b
- name: Upload binary files (windows_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_windows_amd64
path: release/cloudreve*windows_amd64.*
- name: Upload binary files (linux_amd64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_amd64
path: release/cloudreve*linux_amd64.*
- name: Upload binary files (linux_arm)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm
path: release/cloudreve*linux_arm.*
- name: Upload binary files (linux_arm64)
uses: actions/upload-artifact@v2
with:
name: cloudreve_linux_arm64
path: release/cloudreve*linux_arm64.*
+2
View File
@@ -1,6 +1,8 @@
language: go
go:
- 1.13.x
node_js:
- 12.16.3
git:
depth: 1
install:
+1 -1
Submodule assets updated: cfb1206e13...54f4e608bd
+2
View File
@@ -37,11 +37,13 @@ func CheckUpdate() {
res, err := client.Request("GET", "https://api.github.com/repos/cloudreve/cloudreve/releases", nil).GetResponse()
if err != nil {
util.Log().Warning("更新检查失败, %s", err)
return
}
var list []GitHubRelease
if err := json.Unmarshal([]byte(res), &list); err != nil {
util.Log().Warning("更新检查失败, %s", err)
return
}
if len(list) > 0 {
+67 -1
View File
@@ -7,10 +7,14 @@ import (
_ "github.com/HFO4/cloudreve/statik"
"github.com/gin-contrib/static"
"github.com/rakyll/statik/fs"
"io"
"io/ioutil"
"net/http"
"path"
)
const StaticFolder = "statics"
type GinFS struct {
FS http.FileSystem
}
@@ -42,7 +46,7 @@ func (b *GinFS) Exists(prefix string, filepath string) bool {
func InitStatic() {
var err error
if util.Exists(util.RelativePath("statics")) {
if util.Exists(util.RelativePath(StaticFolder)) {
util.Log().Info("检测到 statics 目录存在,将使用此目录下的静态资源文件")
StaticFS = static.LocalFile(util.RelativePath("statics"), false)
@@ -89,3 +93,65 @@ func InitStatic() {
}
}
// Eject 抽离内置静态资源
func Eject() {
staticFS, err := fs.New()
if err != nil {
util.Log().Panic("无法初始化静态资源, %s", err)
}
root, err := staticFS.Open("/")
if err != nil {
util.Log().Panic("根目录不存在, %s", err)
}
var walk func(relPath string, object http.File)
walk = func(relPath string, object http.File) {
stat, err := object.Stat()
if err != nil {
util.Log().Error("无法获取[%s]的信息, %s, 跳过...", relPath, err)
return
}
if !stat.IsDir() {
// 写入文件
out, err := util.CreatNestedFile(util.RelativePath(StaticFolder + relPath))
defer out.Close()
if err != nil {
util.Log().Error("无法创建文件[%s], %s, 跳过...", relPath, err)
return
}
util.Log().Info("导出 [%s]...", relPath)
if _, err := io.Copy(out, object); err != nil {
util.Log().Error("无法写入文件[%s], %s, 跳过...", relPath, err)
return
}
} else {
// 列出目录
objects, err := object.Readdir(0)
if err != nil {
util.Log().Error("无法步入子目录[%s], %s, 跳过...", relPath, err)
return
}
// 递归遍历子目录
for _, newObject := range objects {
newPath := path.Join(relPath, newObject.Name())
newRoot, err := staticFS.Open(newPath)
if err != nil {
util.Log().Error("无法打开对象[%s], %s, 跳过...", newPath, err)
continue
}
walk(newPath, newRoot)
}
}
}
util.Log().Info("开始导出内置静态资源...")
walk("/", root)
util.Log().Info("内置静态资源导出完成")
}
+7 -2
View File
@@ -34,7 +34,7 @@ buildAssets () {
fi
cd $REPO
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico -f
statik -src=assets/build/ -include=*.html,*.js,*.json,*.css,*.png,*.svg,*.ico,*.ttf -f
}
buildBinary () {
@@ -55,7 +55,12 @@ _build() {
export CC=$gcc
export CGO_ENABLED=1
out="release/cloudreve_${VERSION}_${os}_${arch}"
if [ -n "$VERSION" ]; then
out="release/cloudreve_${VERSION}_${os}_${arch}"
else
out="release/cloudreve_${COMMIT_SHA}_${os}_${arch}"
fi
go build -a -o "${out}" -ldflags " -X 'github.com/HFO4/cloudreve/pkg/conf.BackendVersion=$VERSION' -X 'github.com/HFO4/cloudreve/pkg/conf.LastCommit=$COMMIT_SHA'"
if [ "$os" = "windows" ]; then
+23 -1
View File
@@ -8,16 +8,38 @@ import (
"github.com/HFO4/cloudreve/routers"
)
var confPath string
var (
isEject bool
confPath string
)
func init() {
flag.StringVar(&confPath, "c", util.RelativePath("conf.ini"), "配置文件路径")
flag.BoolVar(&isEject, "eject", false, "导出内置静态资源")
flag.Parse()
bootstrap.Init(confPath)
}
func main() {
if isEject {
// 开始导出内置静态资源文件
bootstrap.Eject()
return
}
api := routers.InitRouter()
// 如果启用了SSL
if conf.SSLConfig.CertPath != "" {
go func() {
util.Log().Info("开始监听 %s", conf.SSLConfig.Listen)
if err := api.RunTLS(conf.SSLConfig.Listen,
conf.SSLConfig.CertPath, conf.SSLConfig.KeyPath); err != nil {
util.Log().Error("无法监听[%s]%s", conf.SSLConfig.Listen, err)
}
}()
}
util.Log().Info("开始监听 %s", conf.SystemConfig.Listen)
if err := api.Run(conf.SystemConfig.Listen); err != nil {
util.Log().Error("无法监听[%s]%s", conf.SystemConfig.Listen, err)
+47
View File
@@ -1,10 +1,13 @@
package middleware
import (
"github.com/HFO4/cloudreve/bootstrap"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/hashid"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/gin-gonic/gin"
"io/ioutil"
)
// HashID 将给定对象的HashID转换为真实ID
@@ -38,3 +41,47 @@ func IsFunctionEnabled(key string) gin.HandlerFunc {
c.Next()
}
}
// InjectSiteInfo 向首页html中插入站点信息
func InjectSiteInfo() gin.HandlerFunc {
ignoreFunc := func(c *gin.Context) {
c.Next()
}
if bootstrap.StaticFS == nil {
return ignoreFunc
}
// 读取index.html
file, err := bootstrap.StaticFS.Open("/index.html")
if err != nil {
util.Log().Warning("静态文件[index.html]不存在,可能会影响首页展示")
return ignoreFunc
}
fileContentBytes, err := ioutil.ReadAll(file)
if err != nil {
util.Log().Warning("静态文件[index.html]读取失败,可能会影响首页展示")
return ignoreFunc
}
fileContent := string(fileContentBytes)
return func(c *gin.Context) {
if c.Request.URL.Path == "/" || c.Request.URL.Path == "/index.html" {
// 读取、替换站点设置
options := model.GetSettingByNames("siteName", "siteKeywords", "siteScript",
"pwa_small_icon")
finalHTML := util.Replace(map[string]string{
"{siteName}": options["siteName"],
"{siteDes}": options["siteDes"],
"{siteScript}": options["siteScript"],
"{pwa_small_icon}": options["pwa_small_icon"],
}, fileContent)
c.Header("Content-Type", "text/html")
c.String(200, finalHTML)
c.Abort()
return
}
c.Next()
}
}
+109
View File
@@ -1,12 +1,17 @@
package middleware
import (
"errors"
"github.com/HFO4/cloudreve/bootstrap"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/hashid"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"net/http"
"net/http/httptest"
"os"
"testing"
)
@@ -75,3 +80,107 @@ func TestIsFunctionEnabled(t *testing.T) {
}
}
type StaticMock struct {
testMock.Mock
}
func (m StaticMock) Open(name string) (http.File, error) {
args := m.Called(name)
return args.Get(0).(http.File), args.Error(1)
}
func (m StaticMock) Exists(prefix string, filepath string) bool {
args := m.Called(prefix, filepath)
return args.Bool(0)
}
func TestInjectSiteInfo(t *testing.T) {
asserts := assert.New(t)
rec := httptest.NewRecorder()
// 静态资源未加载
{
TestFunc := InjectSiteInfo()
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("GET", "/", nil)
TestFunc(c)
asserts.False(c.IsAborted())
}
// index.html 不存在
{
testStatic := &StaticMock{}
bootstrap.StaticFS = testStatic
testStatic.On("Open", "/index.html").
Return(&os.File{}, errors.New("error"))
TestFunc := InjectSiteInfo()
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("GET", "/", nil)
TestFunc(c)
asserts.False(c.IsAborted())
}
// index.html 读取失败
{
file, _ := util.CreatNestedFile("tests/index.html")
file.Close()
testStatic := &StaticMock{}
bootstrap.StaticFS = testStatic
testStatic.On("Open", "/index.html").
Return(file, nil)
TestFunc := InjectSiteInfo()
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("GET", "/", nil)
TestFunc(c)
asserts.False(c.IsAborted())
}
// 成功且命中
{
file, _ := util.CreatNestedFile("tests/index.html")
defer file.Close()
testStatic := &StaticMock{}
bootstrap.StaticFS = testStatic
testStatic.On("Open", "/index.html").
Return(file, nil)
TestFunc := InjectSiteInfo()
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("GET", "/", nil)
cache.Set("setting_siteName", "cloudreve", 0)
cache.Set("setting_siteKeywords", "cloudreve", 0)
cache.Set("setting_siteScript", "cloudreve", 0)
cache.Set("setting_pwa_small_icon", "cloudreve", 0)
TestFunc(c)
asserts.True(c.IsAborted())
}
// 成功且未命中
{
file, _ := util.CreatNestedFile("tests/index.html")
defer file.Close()
testStatic := &StaticMock{}
bootstrap.StaticFS = testStatic
testStatic.On("Open", "/index.html").
Return(file, nil)
TestFunc := InjectSiteInfo()
c, _ := gin.CreateTestContext(rec)
c.Params = []gin.Param{}
c.Request, _ = http.NewRequest("GET", "/2", nil)
TestFunc(c)
asserts.False(c.IsAborted())
}
}
+3
View File
@@ -76,11 +76,13 @@ func addDefaultSettings() {
defaultSettings := []Setting{
{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
{Name: "siteICPId", Value: ``, Type: "basic"},
{Name: "register_enabled", Value: `1`, Type: "register"},
{Name: "default_group", Value: `2`, Type: "register"},
{Name: "siteKeywords", Value: `网盘,网盘`, Type: "basic"},
{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
{Name: "siteScript", Value: ``, Type: "basic"},
{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
{Name: "mail_keepalive", Value: `30`, Type: "mail"},
{Name: "fromAdress", Value: `no-reply@acg.blue`, Type: "mail"},
@@ -89,6 +91,7 @@ func addDefaultSettings() {
{Name: "replyTo", Value: `abslant@126.com`, Type: "mail"},
{Name: "smtpUser", Value: `no-reply@acg.blue`, Type: "mail"},
{Name: "smtpPass", Value: ``, Type: "mail"},
{Name: "smtpEncryption", Value: `0`, Type: "mail"},
{Name: "maxEditSize", Value: `4194304`, Type: "file_edit"},
{Name: "archive_timeout", Value: `60`, Type: "timeout"},
{Name: "download_timeout", Value: `60`, Type: "timeout"},
+26 -3
View File
@@ -8,7 +8,9 @@ import (
"github.com/jinzhu/gorm"
"net/url"
"path"
"path/filepath"
"strconv"
"strings"
"time"
)
@@ -48,6 +50,16 @@ type PolicyOption struct {
OdRedirect string `json:"od_redirect,omitempty"`
}
var thumbSuffix = map[string][]string{
"local": {},
"qiniu": {".psd", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"oss": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"cos": {".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"upyun": {".svg", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".tiff", ".bmp"},
"remote": {},
"onedrive": {"*"},
}
func init() {
// 注册缓存用到的复杂结构
gob.Register(Policy{})
@@ -178,6 +190,17 @@ func (policy *Policy) IsDirectlyPreview() bool {
return policy.Type == "local"
}
// IsThumbExist 给定文件名,返回此存储策略下是否可能存在缩略图
func (policy *Policy) IsThumbExist(name string) bool {
if list, ok := thumbSuffix[policy.Type]; ok {
if len(list) == 1 && list[0] == "*" {
return true
}
return util.ContainsString(list, strings.ToLower(filepath.Ext(name)))
}
return false
}
// IsTransitUpload 返回此策略上传给定size文件时是否需要服务端中转
func (policy *Policy) IsTransitUpload(size uint64) bool {
if policy.Type == "local" {
@@ -199,9 +222,9 @@ func (policy *Policy) IsThumbGenerateNeeded() bool {
return policy.Type == "local"
}
// IsMockThumbNeeded 返回策略是否需要在上传后默认当图像文件
func (policy *Policy) IsMockThumbNeeded() bool {
return policy.Type == "onedrive"
// CanStructureBeListed 返回存储策略是否能被前台列物理目录
func (policy *Policy) CanStructureBeListed() bool {
return policy.Type != "local" && policy.Type != "remote"
}
// GetUploadURL 获取文件上传服务API地址
+53 -2
View File
@@ -242,13 +242,64 @@ func TestPolicy_UpdateAccessKey(t *testing.T) {
func TestPolicy_Props(t *testing.T) {
asserts := assert.New(t)
policy := Policy{Type: "onedrive"}
asserts.True(policy.IsMockThumbNeeded())
asserts.False(policy.IsThumbGenerateNeeded())
asserts.True(policy.IsPathGenerateNeeded())
asserts.True(policy.IsTransitUpload(4))
asserts.False(policy.IsTransitUpload(5 * 1024 * 1024))
asserts.True(policy.CanStructureBeListed())
policy.Type = "local"
asserts.False(policy.IsMockThumbNeeded())
asserts.True(policy.IsThumbGenerateNeeded())
asserts.True(policy.IsPathGenerateNeeded())
asserts.False(policy.CanStructureBeListed())
}
func TestPolicy_IsThumbExist(t *testing.T) {
asserts := assert.New(t)
testCases := []struct {
name string
expect bool
policy string
}{
{
"1.png",
false,
"unknown",
},
{
"1.png",
false,
"local",
},
{
"1.png",
true,
"cos",
},
{
"1",
false,
"cos",
},
{
"1.txt.png",
true,
"cos",
},
{
"1.png.txt",
false,
"cos",
},
{
"1",
true,
"onedrive",
},
}
for _, testCase := range testCases {
policy := Policy{Type: testCase.policy}
asserts.Equal(testCase.expect, policy.IsThumbExist(testCase.name))
}
}
+7
View File
@@ -27,6 +27,12 @@ type system struct {
HashIDSalt string
}
type ssl struct {
CertPath string `validate:"omitempty,required"`
KeyPath string `validate:"omitempty,required"`
Listen string `validate:"required"`
}
// slave 作为slave存储端配置
type slave struct {
Secret string `validate:"omitempty,gte=64"`
@@ -113,6 +119,7 @@ func Init(path string) {
sections := map[string]interface{}{
"Database": DatabaseConfig,
"System": SystemConfig,
"SSL": SSLConfig,
"Captcha": CaptchaConfig,
"Redis": RedisConfig,
"Thumbnail": ThumbConfig,
+6
View File
@@ -59,3 +59,9 @@ var SlaveConfig = &slave{
CallbackTimeout: 20,
SignatureTTL: 60,
}
var SSLConfig = &ssl{
Listen: ":443",
CertPath: "",
KeyPath: "",
}
+3 -3
View File
@@ -1,13 +1,13 @@
package conf
// BackendVersion 当前后端版本号
var BackendVersion = "3.0.0-beta1"
var BackendVersion = "3.1.0"
// RequiredDBVersion 与当前版本匹配的数据库版本
var RequiredDBVersion = "3.0.0"
var RequiredDBVersion = "3.1.0"
// RequiredStaticVersion 与当前版本匹配的静态资源版本
var RequiredStaticVersion = "3.0.0"
var RequiredStaticVersion = "3.1.0"
// IsPro 是否为Pro版本
var IsPro = "false"
+10 -8
View File
@@ -30,19 +30,21 @@ func Init() {
"replyTo",
"smtpUser",
"smtpPass",
"smtpEncryption",
)
port := model.GetIntSetting("smtpPort", 25)
keepAlive := model.GetIntSetting("mail_keepalive", 30)
client := NewSMTPClient(SMTPConfig{
Name: options["fromName"],
Address: options["fromAdress"],
ReplyTo: options["replyTo"],
Host: options["smtpHost"],
Port: port,
User: options["smtpUser"],
Password: options["smtpPass"],
Keepalive: keepAlive,
Name: options["fromName"],
Address: options["fromAdress"],
ReplyTo: options["replyTo"],
Host: options["smtpHost"],
Port: port,
User: options["smtpUser"],
Password: options["smtpPass"],
Keepalive: keepAlive,
Encryption: model.IsTrueVal(options["smtpEncryption"]),
})
Client = client
+6 -1
View File
@@ -22,7 +22,7 @@ type SMTPConfig struct {
Port int // 服务器端口
User string // 用户名
Password string // 密码
Encryption string // 是否启用加密
Encryption bool // 是否启用加密
Keepalive int // SMTP 连接保留时长
}
@@ -77,6 +77,11 @@ func (client *SMTP) Init() {
d.Timeout = time.Duration(client.Config.Keepalive+5) * time.Second
client.chOpen = true
// 是否启用 SSL
if client.Config.Encryption {
d.SSL = true
}
var s mail.SendCloser
var err error
open := false
+105 -1
View File
@@ -18,6 +18,9 @@ import (
"io"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
@@ -46,6 +49,80 @@ type Driver struct {
HTTPClient request.Client
}
// List 列出COS文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化列目录参数
opt := &cossdk.BucketGetOptions{
Prefix: strings.TrimPrefix(base, "/"),
EncodingType: "",
MaxKeys: 1000,
}
// 是否为递归列出
if !recursive {
opt.Delimiter = "/"
}
// 手动补齐结尾的slash
if opt.Prefix != "" {
opt.Prefix += "/"
}
var (
marker string
objects []cossdk.Object
commons []string
)
for {
res, _, err := handler.Client.Bucket.Get(ctx, opt)
if err != nil {
return nil, err
}
objects = append(objects, res.Contents...)
commons = append(commons, res.CommonPrefixes...)
// 如果本次未列取完,则继续使用marker获取结果
marker = res.NextMarker
// marker 为空时结果列取完毕,跳出
if marker == "" {
break
}
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(opt.Prefix, object)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(opt.Prefix, object.Key)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(object.Size),
IsDir: false,
LastModify: time.Now(),
})
}
return res, nil
}
// CORS 创建跨域策略
func (handler Driver) CORS() error {
_, err := handler.Client.Bucket.PutCORS(context.Background(), &cossdk.BucketPutCORSOptions{
@@ -143,7 +220,34 @@ func (handler Driver) Delete(ctx context.Context, files []string) ([]string, err
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, path string) (*response.ContentResponse, error) {
return nil, errors.New("未实现")
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("无法获取缩略图尺寸设置")
}
thumbParam := fmt.Sprintf("imageMogr2/thumbnail/%dx%d", thumbSize[0], thumbSize[1])
source, err := handler.signSourceURL(
ctx,
path,
int64(model.GetIntSetting("preview_timeout", 60)),
&urlOption{},
)
if err != nil {
return nil, err
}
thumbURL, _ := url.Parse(source)
thumbQuery := thumbURL.Query()
thumbQuery.Add(thumbParam, "")
thumbURL.RawQuery = thumbQuery.Encode()
return &response.ContentResponse{
Redirect: true,
URL: thumbURL.String(),
}, nil
}
// Source 获取外链URL
+46
View File
@@ -23,6 +23,52 @@ type Driver struct {
Policy *model.Policy
}
// List 递归列取给定物理路径下所有文件
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object
// 取得起始路径
root := util.RelativePath(filepath.FromSlash(path))
// 开始遍历路径下的文件、目录
err := filepath.Walk(root,
func(path string, info os.FileInfo, err error) error {
// 跳过根目录
if path == root {
return nil
}
if err != nil {
util.Log().Warning("无法遍历目录 %s, %s", path, err)
return filepath.SkipDir
}
// 将遍历对象的绝对路径转换为相对路径
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
res = append(res, response.Object{
Name: info.Name(),
RelativePath: filepath.ToSlash(rel),
Source: path,
Size: uint64(info.Size()),
IsDir: info.IsDir(),
LastModify: info.ModTime(),
})
// 如果非递归,则不步入目录
if !recursive && info.IsDir() {
return filepath.SkipDir
}
return nil
})
return res, err
}
// Get 获取文件内容
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 打开文件
@@ -230,3 +230,34 @@ func TestHandler_Token(t *testing.T) {
_, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{}
ctx := context.Background()
// 创建测试目录结构
for _, path := range []string{
"test/TestDriver_List/parent.txt",
"test/TestDriver_List/parent_folder2/sub2.txt",
"test/TestDriver_List/parent_folder1/sub_folder/sub1.txt",
"test/TestDriver_List/parent_folder1/sub_folder/sub2.txt",
} {
f, _ := util.CreatNestedFile(util.RelativePath(path))
f.Close()
}
// 非递归列出
{
res, err := handler.List(ctx, "test/TestDriver_List", false)
asserts.NoError(err)
asserts.Len(res, 3)
}
// 递归列出
{
res, err := handler.List(ctx, "test/TestDriver_List", true)
asserts.NoError(err)
asserts.Len(res, 7)
}
}
+40 -1
View File
@@ -24,8 +24,10 @@ import (
const (
// SmallFileSize 单文件上传接口最大尺寸
SmallFileSize uint64 = 4 * 1024 * 1024
// ChunkSize 分片上传分片大小
// ChunkSize 服务端中转分片上传分片大小
ChunkSize uint64 = 10 * 1024 * 1024
// ListRetry 列取请求重试次数
ListRetry = 1
)
// GetSourcePath 获取文件的绝对路径
@@ -59,6 +61,43 @@ func (client *Client) getRequestURL(api string) string {
return base.String()
}
// ListChildren 根据路径列取子对象
func (client *Client) ListChildren(ctx context.Context, path string) ([]FileInfo, error) {
var requestURL string
dst := strings.TrimPrefix(path, "/")
if dst == "" {
requestURL = client.getRequestURL("me/drive/root/children")
} else {
requestURL = client.getRequestURL("me/drive/root:/" + dst + ":/children")
}
res, err := client.requestWithStr(ctx, "GET", requestURL+"?$top=999999999", "", 200)
if err != nil {
retried := 0
if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok {
retried = v
}
if retried < ListRetry {
retried++
util.Log().Debug("路径[%s]列取请求失败[%s]5秒钟后重试", path, err)
time.Sleep(time.Duration(5) * time.Second)
return client.ListChildren(context.WithValue(ctx, fsctx.RetryCtx, retried), path)
}
return nil, err
}
var (
decodeErr error
fileInfo ListResponse
)
decodeErr = json.Unmarshal([]byte(res), &fileInfo)
if decodeErr != nil {
return nil, decodeErr
}
return fileInfo.Value, nil
}
// Meta 根据资源ID或文件路径获取文件元信息
func (client *Client) Meta(ctx context.Context, id string, path string) (*FileInfo, error) {
var requestURL string
@@ -752,6 +752,66 @@ func TestClient_Delete(t *testing.T) {
}
}
func TestClient_ListChildren(t *testing.T) {
asserts := assert.New(t)
client, _ := NewClient(&model.Policy{})
client.Credential.AccessToken = "AccessToken"
// 根目录,请求失败,重测试
{
client.Credential.ExpiresIn = 0
res, err := client.ListChildren(context.Background(), "/")
asserts.Error(err)
asserts.Empty(res)
}
// 非根目录,未知响应
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`???`)),
},
})
client.Request = clientMock
res, err := client.ListChildren(context.Background(), "/uploads")
asserts.Error(err)
asserts.Empty(res)
}
// 非根目录,成功
{
client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"value":[{}]}`)),
},
})
client.Request = clientMock
res, err := client.ListChildren(context.Background(), "/uploads")
asserts.NoError(err)
asserts.Len(res, 1)
}
}
func TestClient_GetThumbURL(t *testing.T) {
asserts := assert.New(t)
client, _ := NewClient(&model.Policy{})
+48
View File
@@ -12,6 +12,9 @@ import (
"github.com/HFO4/cloudreve/pkg/serializer"
"io"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
@@ -22,6 +25,51 @@ type Driver struct {
HTTPClient request.Client
}
// List 列取项目
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
// 列取子项目
objects, _ := handler.Client.ListChildren(ctx, base)
// 获取真实的列取起始根目录
rootPath := base
if realBase, ok := ctx.Value(fsctx.PathCtx).(string); ok {
rootPath = realBase
} else {
ctx = context.WithValue(ctx, fsctx.PathCtx, base)
}
// 整理结果
res := make([]response.Object, 0, len(objects))
for _, object := range objects {
source := path.Join(base, object.Name)
rel, err := filepath.Rel(rootPath, source)
if err != nil {
continue
}
res = append(res, response.Object{
Name: object.Name,
RelativePath: filepath.ToSlash(rel),
Source: source,
Size: object.Size,
IsDir: object.Folder != nil,
LastModify: time.Now(),
})
}
// 递归列取子目录
if recursive {
for _, object := range objects {
if object.Folder != nil {
sub, _ := handler.List(ctx, path.Join(base, object.Name), recursive)
res = append(res, sub...)
}
}
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
@@ -174,6 +174,78 @@ func TestDriver_Source(t *testing.T) {
}
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
},
}
handler.Client, _ = NewClient(&model.Policy{})
handler.Client.Credential.AccessToken = "AccessToken"
handler.Client.Credential.ExpiresIn = time.Now().Add(time.Duration(100) * time.Hour).Unix()
// 非递归
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
testMock.Anything,
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"value":[{}]}`)),
},
})
handler.Client.Request = clientMock
res, err := handler.List(context.Background(), "/", false)
asserts.NoError(err)
asserts.Len(res, 1)
}
// 递归一次
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"GET",
"me/drive/root/children?$top=999999999",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"value":[{"name":"1","folder":{}}]}`)),
},
})
clientMock.On(
"Request",
"GET",
"me/drive/root:/1:/children?$top=999999999",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"value":[{"name":"2"}]}`)),
},
})
handler.Client.Request = clientMock
res, err := handler.List(context.Background(), "/", true)
asserts.NoError(err)
asserts.Len(res, 2)
}
}
func TestDriver_Thumb(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
+16
View File
@@ -32,6 +32,16 @@ type FileInfo struct {
Image imageInfo `json:"image"`
ParentReference parentReference `json:"parentReference"`
DownloadURL string `json:"@microsoft.graph.downloadUrl"`
File *file `json:"file"`
Folder *folder `json:"folder"`
}
type file struct {
MimeType string `json:"mimeType"`
}
type folder struct {
ChildCount int `json:"childCount"`
}
type imageInfo struct {
@@ -83,6 +93,12 @@ type ThumbResponse struct {
URL string `json:"url"`
}
// ListResponse 列取子项目响应
type ListResponse struct {
Value []FileInfo `json:"value"`
Context string `json:"@odata.context"`
}
// Chunk 文件分片
type Chunk struct {
Offset int
+74
View File
@@ -18,6 +18,8 @@ import (
"io"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
@@ -99,6 +101,78 @@ func (handler *Driver) InitOSSClient() error {
return nil
}
// List 列出OSS上的文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化客户端
if err := handler.InitOSSClient(); err != nil {
return nil, err
}
// 列取文件
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
var (
delimiter string
marker string
objects []oss.ObjectProperties
commons []string
)
if !recursive {
delimiter = "/"
}
for {
subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base),
oss.MaxKeys(1000), oss.Delimiter(delimiter))
if err != nil {
return nil, err
}
objects = append(objects, subRes.Objects...)
commons = append(commons, subRes.CommonPrefixes...)
marker = subRes.NextMarker
if marker == "" {
break
}
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(base, object)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(base, object.Key)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(object.Size),
IsDir: false,
LastModify: object.LastModified,
})
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 通过VersionID禁止缓存
+20
View File
@@ -311,3 +311,23 @@ func TestDriver_Get(t *testing.T) {
asserts.Equal("123", string(content))
}
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
AccessKey: "ak",
SecretKey: "sk",
BucketName: "test",
Server: "test.com",
IsPrivate: true,
},
}
// 连接失败
{
res, err := handler.List(context.Background(), "/", true)
asserts.Error(err)
asserts.Empty(res)
}
}
+76
View File
@@ -14,6 +14,9 @@ import (
"io"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
"time"
)
@@ -22,6 +25,79 @@ type Driver struct {
Policy *model.Policy
}
// List 列出给定路径下的文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
var (
delimiter string
marker string
objects []storage.ListItem
commons []string
)
if !recursive {
delimiter = "/"
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
for {
entries, folders, nextMarker, hashNext, err := bucketManager.ListFiles(
handler.Policy.BucketName,
base, delimiter, marker, 1000)
if err != nil {
return nil, err
}
objects = append(objects, entries...)
commons = append(commons, folders...)
if !hashNext {
break
}
marker = nextMarker
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(base, object)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(base, object.Key)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(object.Fsize),
IsDir: false,
LastModify: time.Now(),
})
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 给文件名加上随机参数以强制拉取
+43
View File
@@ -27,6 +27,47 @@ type Driver struct {
AuthInstance auth.Auth
}
// List 列取文件
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object
reqBody := serializer.ListRequest{
Path: path,
Recursive: recursive,
}
reqBodyEncoded, err := json.Marshal(reqBody)
if err != nil {
return res, err
}
// 发送列表请求
bodyReader := strings.NewReader(string(reqBodyEncoded))
signTTL := model.GetIntSetting("slave_api_timeout", 60)
resp, err := handler.Client.Request(
"POST",
handler.getAPIUrl("list"),
bodyReader,
request.WithCredential(handler.AuthInstance, int64(signTTL)),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return res, err
}
// 处理列取结果
if resp.Code != 0 {
return res, errors.New(resp.Error)
}
if resStr, ok := resp.Data.(string); ok {
err = json.Unmarshal([]byte(resStr), &res)
if err != nil {
return res, err
}
}
return res, nil
}
// getAPIUrl 获取接口请求地址
func (handler Driver) getAPIUrl(scope string, routes ...string) string {
serverURL, err := url.Parse(handler.Policy.Server)
@@ -40,6 +81,8 @@ func (handler Driver) getAPIUrl(scope string, routes ...string) string {
controller, _ = url.Parse("/api/v3/slave/delete")
case "thumb":
controller, _ = url.Parse("/api/v3/slave/thumb")
case "list":
controller, _ = url.Parse("/api/v3/slave/list")
default:
controller = serverURL
}
@@ -223,6 +223,89 @@ func TestHandler_Delete(t *testing.T) {
}
}
func TestDriver_List(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
Policy: &model.Policy{
SecretKey: "test",
Server: "http://test.com",
},
AuthInstance: auth.HMACAuth{},
}
ctx := context.Background()
cache.Set("setting_slave_api_timeout", "60", 0)
// 成功
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/list",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0,"data":"[{}]"}`)),
},
})
handler.Client = clientMock
res, err := handler.List(ctx, "/", true)
clientMock.AssertExpectations(t)
asserts.NoError(err)
asserts.Len(res, 1)
}
// 响应解析失败
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/list",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":0,"data":"233"}`)),
},
})
handler.Client = clientMock
res, err := handler.List(ctx, "/", true)
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Len(res, 0)
}
// 从机返回错误
{
clientMock := ClientMock{}
clientMock.On(
"Request",
"POST",
"http://test.com/api/v3/slave/list",
testMock.Anything,
testMock.Anything,
).Return(&request.Response{
Err: nil,
Response: &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(strings.NewReader(`{"code":203}`)),
},
})
handler.Client = clientMock
res, err := handler.List(ctx, "/", true)
clientMock.AssertExpectations(t)
asserts.Error(err)
asserts.Len(res, 0)
}
}
func TestHandler_Get(t *testing.T) {
asserts := assert.New(t)
handler := Driver{
+62
View File
@@ -18,6 +18,7 @@ import (
"io"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
@@ -40,6 +41,67 @@ type Driver struct {
Policy *model.Policy
}
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
// 用于接受SDK返回对象的chan
objChan := make(chan *upyun.FileInfo)
objects := []*upyun.FileInfo{}
// 列取配置
listConf := &upyun.GetObjectsConfig{
Path: "/" + base,
ObjectsChan: objChan,
MaxListTries: 1,
}
// 递归列取时不限制递归次数
if recursive {
listConf.MaxListLevel = -1
}
// 启动一个goroutine收集列取的对象信
wg := &sync.WaitGroup{}
wg.Add(1)
go func(input chan *upyun.FileInfo, output *[]*upyun.FileInfo, wg *sync.WaitGroup) {
defer wg.Done()
for {
file, ok := <-input
if !ok {
return
}
*output = append(*output, file)
}
}(objChan, &objects, wg)
up := upyun.NewUpYun(&upyun.UpYunConfig{
Bucket: handler.Policy.BucketName,
Operator: handler.Policy.AccessKey,
Password: handler.Policy.SecretKey,
})
err := up.List(listConf)
if err != nil {
return nil, err
}
wg.Wait()
// 汇总处理列取结果
res := make([]response.Object, 0, len(objects))
for _, object := range objects {
res = append(res, response.Object{
Name: path.Base(object.Name),
RelativePath: object.Name,
Source: path.Join(base, object.Name),
Size: uint64(object.Size),
IsDir: object.IsDir,
LastModify: object.Time,
})
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 给文件名加上随机参数以强制拉取
+5
View File
@@ -63,6 +63,11 @@ func (fs *FileSystem) AddFile(ctx context.Context, parent *model.Folder) (*model
FolderID: parent.ID,
PolicyID: fs.User.Policy.ID,
}
if fs.User.Policy.IsThumbExist(file.GetFileName()) {
newFile.PicInfo = "1,1"
}
_, err = newFile.Create()
if err != nil {
+5 -3
View File
@@ -20,7 +20,7 @@ func TestFileSystem_AddFile(t *testing.T) {
asserts := assert.New(t)
file := local.FileStream{
Size: 5,
Name: "1.txt",
Name: "1.png",
}
folder := model.Folder{
Model: gorm.Model{
@@ -33,6 +33,7 @@ func TestFileSystem_AddFile(t *testing.T) {
ID: 1,
},
Policy: model.Policy{
Type: "cos",
Model: gorm.Model{
ID: 1,
},
@@ -40,7 +41,7 @@ func TestFileSystem_AddFile(t *testing.T) {
},
}
ctx := context.WithValue(context.Background(), fsctx.FileHeaderCtx, file)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.txt")
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/Uploads/1_sad.png")
_, err := fs.AddFile(ctx, &folder)
@@ -54,7 +55,8 @@ func TestFileSystem_AddFile(t *testing.T) {
asserts.NoError(err)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("/Uploads/1_sad.txt", f.SourceName)
asserts.Equal("/Uploads/1_sad.png", f.SourceName)
asserts.NotEmpty(f.PicInfo)
}
func TestFileSystem_GetContent(t *testing.T) {
+6 -1
View File
@@ -41,7 +41,7 @@ type FileHeader interface {
GetVirtualPath() string
}
// Driver 存储策略适配器
// Handler 存储策略适配器
type Handler interface {
// 上传文件, dst为文件存储路径,size 为文件大小。上下文关闭
// 时,应取消上传并清理临时文件
@@ -64,6 +64,11 @@ type Handler interface {
// Token 获取有效期为ttl的上传凭证和签名,同时回调会话有效期为sessionTTL
Token(ctx context.Context, ttl int64, callbackKey string) (serializer.UploadCredential, error)
// List 递归列取远程端path路径下文件、目录,不包含path本身,
// 返回的对象路径以path作为起始根目录.
// recursive - 是否递归列出
List(ctx context.Context, path string, recursive bool) ([]response.Object, error)
}
// FileSystem 管理文件的文件系统
-2
View File
@@ -297,8 +297,6 @@ func GenericAfterUpload(ctx context.Context, fs *FileSystem) error {
// 异步尝试生成缩略图
if fs.User.Policy.IsThumbGenerateNeeded() {
go fs.GenerateThumbnail(ctx, file)
} else if fs.User.Policy.IsMockThumbNeeded() {
file.UpdatePicInfo("1,1")
}
return nil
+5 -1
View File
@@ -38,7 +38,11 @@ func (fs *FileSystem) GetThumb(ctx context.Context, id uint) (*response.ContentR
res.MaxAge = model.GetIntSetting("preview_timeout", 60)
}
// TODO 出错时重新生成缩略图
// 出错时重新生成缩略图
if err != nil {
fs.GenerateThumbnail(ctx, &fs.FileTarget[0])
}
return res, err
}
+41 -4
View File
@@ -32,7 +32,7 @@ type Object struct {
// Rename 重命名对象
func (fs *FileSystem) Rename(ctx context.Context, dir, file []uint, new string) (err error) {
// 验证新名字
if !fs.ValidateLegalName(ctx, new) || !fs.ValidateExtension(ctx, new) {
if !fs.ValidateLegalName(ctx, new) || (len(file) > 0 && !fs.ValidateExtension(ctx, new)) {
return ErrIllegalObjectName
}
@@ -131,8 +131,8 @@ func (fs *FileSystem) Move(ctx context.Context, dirs, files []uint, src, dst str
return err
}
// Delete 递归删除对象
func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
// Delete 递归删除对象, force 为 true 时强制删除文件记录,忽略物理删除是否成功
func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint, force bool) error {
// 已删除的总容量,map用于去重
var deletedStorage = make(map[uint]uint64)
var totalStorage = make(map[uint]uint64)
@@ -182,7 +182,12 @@ func (fs *FileSystem) Delete(ctx context.Context, dirs, files []uint) error {
totalStorage[fs.FileTarget[i].ID] = fs.FileTarget[i].Size
allFileIDs = append(allFileIDs, fs.FileTarget[i].ID)
}
// TODO 用户自主选择是否强制删除
// 如果强制删除,则将全部文件视为删除成功
if force {
deletedFileIDs = allFileIDs
deletedStorage = totalStorage
}
// 删除文件记录
err = model.DeleteFileByIDs(deletedFileIDs)
@@ -280,6 +285,38 @@ func (fs *FileSystem) List(ctx context.Context, dirPath string, pathProcessor fu
return fs.listObjects(ctx, parentPath, childFiles, childFolders, pathProcessor), nil
}
// ListPhysical 列出存储策略中的外部目录
// TODO:测试
func (fs *FileSystem) ListPhysical(ctx context.Context, dirPath string) ([]Object, error) {
if err := fs.DispatchHandler(); fs.Policy == nil || err != nil {
return nil, ErrUnknownPolicyType
}
// 存储策略不支持列取时,返回空结果
if !fs.Policy.CanStructureBeListed() {
return nil, nil
}
// 列取路径
objects, err := fs.Handler.List(ctx, dirPath, false)
if err != nil {
return nil, err
}
var (
folders []model.Folder
)
for _, object := range objects {
if object.IsDir {
folders = append(folders, model.Folder{
Name: object.Name,
})
}
}
return fs.listObjects(ctx, dirPath, nil, folders, nil), nil
}
func (fs *FileSystem) listObjects(ctx context.Context, parent string, files []model.File, folders []model.Folder, pathProcessor func(string) string) []Object {
// 分享文件的ID
shareKey := ""
+131 -3
View File
@@ -8,14 +8,62 @@ import (
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/conf"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
testMock "github.com/stretchr/testify/mock"
"os"
"testing"
)
func TestFileSystem_ListPhysical(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{
User: &model.User{
Model: gorm.Model{
ID: 1,
},
},
Policy: &model.Policy{Type: "mock"},
}
ctx := context.Background()
// 未知存储策略
{
fs.Policy.Type = "unknown"
res, err := fs.ListPhysical(ctx, "/")
asserts.Equal(ErrUnknownPolicyType, err)
asserts.Empty(res)
fs.Policy.Type = "mock"
}
// 无法列取目录
{
testHandler := new(FileHeaderMock)
testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return([]response.Object{}, errors.New("error"))
fs.Handler = testHandler
res, err := fs.ListPhysical(ctx, "/")
asserts.EqualError(err, "error")
asserts.Empty(res)
}
// 成功
{
testHandler := new(FileHeaderMock)
testHandler.On("List", testMock.Anything, "/", testMock.Anything).Return(
[]response.Object{{IsDir: true, Name: "1"}, {IsDir: false, Name: "2"}},
nil,
)
fs.Handler = testHandler
res, err := fs.ListPhysical(ctx, "/")
asserts.NoError(err)
asserts.Len(res, 1)
asserts.Equal("1", res[0].Name)
}
}
func TestFileSystem_List(t *testing.T) {
asserts := assert.New(t)
fs := &FileSystem{User: &model.User{
@@ -342,12 +390,65 @@ func TestFileSystem_Delete(t *testing.T) {
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
err := fs.Delete(ctx, []uint{1}, []uint{1})
err := fs.Delete(ctx, []uint{1}, []uint{1}, false)
asserts.Error(err)
asserts.Equal(203, err.(serializer.AppError).Code)
asserts.Equal(uint64(3), fs.User.Storage)
asserts.NoError(mock.ExpectationsWereMet())
}
//全部未成功,强制
{
fs.CleanTargets()
mock.ExpectQuery("SELECT(.+)").
WillReturnRows(
sqlmock.NewRows([]string{"id"}).
AddRow(1).
AddRow(2).
AddRow(3),
)
mock.ExpectQuery("SELECT(.+)").
WithArgs(1, 2, 3).
WillReturnRows(
sqlmock.NewRows([]string{"id", "name", "source_name", "policy_id", "size"}).
AddRow(4, "1.txt", "1.txt", 365, 1),
)
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "name", "source_name", "policy_id", "size"}).AddRow(1, "2.txt", "2.txt", 365, 2))
mock.ExpectQuery("SELECT(.+)files(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "policy_id", "source_name"}))
// 查询上传策略
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(365, "local"))
// 删除文件记录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 归还容量
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除目录
mock.ExpectBegin()
mock.ExpectExec("DELETE(.+)").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
// 删除对应分享
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)shares").
WillReturnResult(sqlmock.NewResult(0, 3))
mock.ExpectCommit()
fs.FileTarget = []model.File{}
fs.DirTarget = []model.Folder{}
err := fs.Delete(ctx, []uint{1}, []uint{1}, true)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
//全部成功
{
fs.CleanTargets()
@@ -402,7 +503,7 @@ func TestFileSystem_Delete(t *testing.T) {
fs.FileTarget = []model.File{}
fs.DirTarget = []model.Folder{}
err = fs.Delete(ctx, []uint{1}, []uint{1})
err = fs.Delete(ctx, []uint{1}, []uint{1}, false)
asserts.NoError(err)
asserts.Equal(uint64(0), fs.User.Storage)
}
@@ -606,12 +707,39 @@ func TestFileSystem_Rename(t *testing.T) {
asserts.Equal(ErrPathNotExist, err)
}
// 新名字不合法
// 新名字是目录,不合法
{
err := fs.Rename(ctx, []uint{10}, []uint{}, "ne/w")
asserts.Error(err)
asserts.Equal(ErrIllegalObjectName, err)
}
// 新名字是文件,不合法
{
err := fs.Rename(ctx, []uint{}, []uint{10}, "ne/w")
asserts.Error(err)
asserts.Equal(ErrIllegalObjectName, err)
}
// 新名字是文件,扩展名不合法
{
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
err := fs.Rename(ctx, []uint{}, []uint{10}, "1.jpg")
asserts.Error(err)
asserts.Equal(ErrIllegalObjectName, err)
}
// 新名字是目录,不应该检测扩展名
{
fs.User.Policy.OptionsSerialized.FileType = []string{"txt"}
mock.ExpectQuery("SELECT(.+)folders(.+)").
WithArgs(10, 1).
WillReturnRows(sqlmock.NewRows([]string{"id", "name"}))
err := fs.Rename(ctx, []uint{10}, []uint{}, "new")
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.Equal(ErrPathNotExist, err)
}
}
func TestFileSystem_SaveTo(t *testing.T) {
+14 -1
View File
@@ -1,6 +1,9 @@
package response
import "io"
import (
"io"
"time"
)
// ContentResponse 获取文件内容类方法的通用返回值。
// 有些上传策略需要重定向,
@@ -17,3 +20,13 @@ type RSCloser interface {
io.ReadSeeker
io.Closer
}
// Object 列出文件、目录时返回的对象
type Object struct {
Name string `json:"name"`
RelativePath string `json:"relative_path"`
Source string `json:"source"`
Size uint64 `json:"size"`
IsDir bool `json:"is_dir"`
LastModify time.Time `json:"last_modify"`
}
+5
View File
@@ -26,6 +26,11 @@ type FileHeaderMock struct {
testMock.Mock
}
func (m FileHeaderMock) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
args := m.Called(ctx, path, recursive)
return args.Get(0).([]response.Object), args.Error(1)
}
func (m FileHeaderMock) Get(ctx context.Context, path string) (response.RSCloser, error) {
args := m.Called(ctx, path)
return args.Get(0).(response.RSCloser), args.Error(1)
+3 -1
View File
@@ -5,6 +5,7 @@ import model "github.com/HFO4/cloudreve/models"
// SiteConfig 站点全局设置序列
type SiteConfig struct {
SiteName string `json:"title"`
SiteICPId string `json:"siteICPId"`
LoginCaptcha bool `json:"loginCaptcha"`
RegCaptcha bool `json:"regCaptcha"`
ForgetCaptcha bool `json:"forgetCaptcha"`
@@ -13,7 +14,7 @@ type SiteConfig struct {
DefaultTheme string `json:"defaultTheme"`
HomepageViewMethod string `json:"home_view_method"`
ShareViewMethod string `json:"share_view_method"`
Authn bool `json:"authn"'`
Authn bool `json:"authn"`
User User `json:"user"`
UseReCaptcha bool `json:"captcha_IsUseReCaptcha"`
ReCaptchaKey string `json:"captcha_ReCaptchaKey"`
@@ -64,6 +65,7 @@ func BuildSiteConfig(settings map[string]string, user *model.User) Response {
res := Response{
Data: SiteConfig{
SiteName: checkSettingValue(settings, "siteName"),
SiteICPId: checkSettingValue(settings, "siteICPId"),
LoginCaptcha: model.IsTrueVal(checkSettingValue(settings, "login_captcha")),
RegCaptcha: model.IsTrueVal(checkSettingValue(settings, "reg_captcha")),
ForgetCaptcha: model.IsTrueVal(checkSettingValue(settings, "forget_captcha")),
+6
View File
@@ -4,3 +4,9 @@ package serializer
type RemoteDeleteRequest struct {
Files []string `json:"files"`
}
// ListRequest 远程策略列文件请求正文
type ListRequest struct {
Path string `json:"path"`
Recursive bool `json:"recursive"`
}
+221
View File
@@ -0,0 +1,221 @@
package task
import (
"context"
"encoding/json"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem"
"github.com/HFO4/cloudreve/pkg/filesystem/driver/local"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/util"
"path"
)
// ImportTask 导入务
type ImportTask struct {
User *model.User
TaskModel *model.Task
TaskProps ImportProps
Err *JobError
}
// ImportProps 导入任务属性
type ImportProps struct {
PolicyID uint `json:"policy_id"` // 存储策略ID
Src string `json:"src"` // 原始路径
Recursive bool `json:"is_recursive"` // 是否递归导入
Dst string `json:"dst"` // 目的目录
}
// Props 获取任务属性
func (job *ImportTask) Props() string {
res, _ := json.Marshal(job.TaskProps)
return string(res)
}
// Type 获取任务状态
func (job *ImportTask) Type() int {
return ImportTaskType
}
// Creator 获取创建者ID
func (job *ImportTask) Creator() uint {
return job.User.ID
}
// Model 获取任务的数据库模型
func (job *ImportTask) Model() *model.Task {
return job.TaskModel
}
// SetStatus 设定状态
func (job *ImportTask) SetStatus(status int) {
job.TaskModel.SetStatus(status)
}
// SetError 设定任务失败信息
func (job *ImportTask) SetError(err *JobError) {
job.Err = err
res, _ := json.Marshal(job.Err)
job.TaskModel.SetError(string(res))
}
// SetErrorMsg 设定任务失败信息
func (job *ImportTask) SetErrorMsg(msg string, err error) {
jobErr := &JobError{Msg: msg}
if err != nil {
jobErr.Error = err.Error()
}
job.SetError(jobErr)
}
// GetError 返回任务失败信息
func (job *ImportTask) GetError() *JobError {
return job.Err
}
// Do 开始执行任务
func (job *ImportTask) Do() {
ctx := context.Background()
// 查找存储策略
policy, err := model.GetPolicyByID(job.TaskProps.PolicyID)
if err != nil {
job.SetErrorMsg("找不到存储策略", err)
return
}
// 创建文件系统
job.User.Policy = policy
fs, err := filesystem.NewFileSystem(job.User)
if err != nil {
job.SetErrorMsg(err.Error(), nil)
return
}
defer fs.Recycle()
// 注册钩子
fs.Use("BeforeAddFile", filesystem.HookValidateFile)
fs.Use("BeforeAddFile", filesystem.HookValidateCapacity)
fs.Use("AfterValidateFailed", filesystem.HookGiveBackCapacity)
// 列取目录、对象
job.TaskModel.SetProgress(ListingProgress)
coxIgnoreConflict := context.WithValue(context.Background(), fsctx.IgnoreConflictCtx,
true)
objects, err := fs.Handler.List(ctx, job.TaskProps.Src, job.TaskProps.Recursive)
if err != nil {
job.SetErrorMsg("无法列取文件", err)
return
}
job.TaskModel.SetProgress(InsertingProgress)
// 虚拟目录路径与folder对象ID的对应
pathCache := make(map[string]*model.Folder, len(objects))
// 插入目录记录到用户文件系统
for _, object := range objects {
if object.IsDir {
// 创建目录
virtualPath := path.Join(job.TaskProps.Dst, object.RelativePath)
folder, err := fs.CreateDirectory(coxIgnoreConflict, virtualPath)
if err != nil {
util.Log().Warning("导入任务无法创建用户目录[%s], %s", virtualPath, err)
} else if folder.ID > 0 {
pathCache[virtualPath] = folder
}
}
}
// 插入文件记录到用户文件系统
for _, object := range objects {
if !object.IsDir {
// 创建文件信息
virtualPath := path.Dir(path.Join(job.TaskProps.Dst, object.RelativePath))
fileHeader := local.FileStream{
Size: object.Size,
VirtualPath: virtualPath,
Name: object.Name,
}
addFileCtx := context.WithValue(ctx, fsctx.FileHeaderCtx, fileHeader)
addFileCtx = context.WithValue(addFileCtx, fsctx.SavePathCtx, object.Source)
// 查找父目录
parentFolder := &model.Folder{}
if parent, ok := pathCache[virtualPath]; ok {
parentFolder = parent
} else {
exist, folder := fs.IsPathExist(virtualPath)
if exist {
parentFolder = folder
} else {
folder, err := fs.CreateDirectory(context.Background(), virtualPath)
if err != nil {
util.Log().Warning("导入任务无法创建用户目录[%s], %s",
virtualPath, err)
continue
}
parentFolder = folder
}
}
// 插入文件记录
_, err := fs.AddFile(addFileCtx, parentFolder)
if err != nil {
util.Log().Warning("导入任务无法创插入文件[%s], %s",
object.RelativePath, err)
if err == filesystem.ErrInsufficientCapacity {
job.SetErrorMsg("容量不足", err)
return
}
}
}
}
}
// NewImportTask 新建导入任务
func NewImportTask(user, policy uint, src, dst string, recursive bool) (Job, error) {
creator, err := model.GetActiveUserByID(user)
if err != nil {
return nil, err
}
newTask := &ImportTask{
User: &creator,
TaskProps: ImportProps{
PolicyID: policy,
Recursive: recursive,
Src: src,
Dst: dst,
},
}
record, err := Record(newTask)
if err != nil {
return nil, err
}
newTask.TaskModel = record
return newTask, nil
}
// NewImportTaskFromModel 从数据库记录中恢复导入任务
func NewImportTaskFromModel(task *model.Task) (Job, error) {
user, err := model.GetActiveUserByID(task.UserID)
if err != nil {
return nil, err
}
newTask := &ImportTask{
User: &user,
TaskModel: task,
}
err = json.Unmarshal([]byte(task.Props), &newTask.TaskProps)
if err != nil {
return nil, err
}
return newTask, nil
}
+242
View File
@@ -0,0 +1,242 @@
package task
import (
"errors"
"github.com/DATA-DOG/go-sqlmock"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/util"
"github.com/jinzhu/gorm"
"github.com/stretchr/testify/assert"
"testing"
)
func TestImportTask_Props(t *testing.T) {
asserts := assert.New(t)
task := &ImportTask{
User: &model.User{},
}
asserts.NotEmpty(task.Props())
asserts.Equal(ImportTaskType, task.Type())
asserts.EqualValues(0, task.Creator())
asserts.Nil(task.Model())
}
func TestImportTask_SetStatus(t *testing.T) {
asserts := assert.New(t)
task := &ImportTask{
User: &model.User{},
TaskModel: &model.Task{
Model: gorm.Model{ID: 1},
},
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
task.SetStatus(3)
asserts.NoError(mock.ExpectationsWereMet())
}
func TestImportTask_SetError(t *testing.T) {
asserts := assert.New(t)
task := &ImportTask{
User: &model.User{},
TaskModel: &model.Task{
Model: gorm.Model{ID: 1},
},
}
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
task.SetErrorMsg("error", nil)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Equal("error", task.GetError().Msg)
}
func TestImportTask_Do(t *testing.T) {
asserts := assert.New(t)
task := &ImportTask{
User: &model.User{},
TaskModel: &model.Task{
Model: gorm.Model{ID: 1},
},
TaskProps: ImportProps{
PolicyID: 63,
Src: "",
Recursive: false,
Dst: "",
},
}
// 存储策略不存在
{
cache.Deletes([]string{"63"}, "policy_")
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id"}))
// 设定失败状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
task.Do()
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotEmpty(task.Err.Error)
task.Err = nil
}
// 无法分配 Filesystem
{
cache.Deletes([]string{"63"}, "policy_")
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(63, "unknown"))
// 设定失败状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
task.Do()
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotEmpty(task.Err.Msg)
task.Err = nil
}
// 成功列取,但是文件为空
{
cache.Deletes([]string{"63"}, "policy_")
task.TaskProps.Src = "TestImportTask_Do/empty"
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(63, "local"))
// 设定listing状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 设定inserting状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
task.Do()
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(task.Err)
task.Err = nil
}
// 创建测试文件
f, _ := util.CreatNestedFile(util.RelativePath("tests/TestImportTask_Do/test.txt"))
f.Close()
// 成功列取,包含一个文件一个目录,父目录创建失败
{
cache.Deletes([]string{"63"}, "policy_")
task.TaskProps.Src = "tests"
task.TaskProps.Dst = "/"
task.TaskProps.Recursive = true
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(63, "local"))
// 设定listing状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 设定inserting状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 查找父目录,但是不存在
mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}))
// 仍然不存在
mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}))
// 创建文件时查找父目录,仍然不存在
mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}))
task.Do()
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(task.Err)
task.Err = nil
}
// 成功列取,包含一个文件一个目录, 全部操作成功
{
cache.Deletes([]string{"63"}, "policy_")
task.TaskProps.Src = "tests"
task.TaskProps.Dst = "/"
task.TaskProps.Recursive = true
mock.ExpectQuery("SELECT(.+)policies(.+)").
WillReturnRows(sqlmock.NewRows([]string{"id", "type"}).AddRow(63, "local"))
// 设定listing状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 设定inserting状态
mock.ExpectBegin()
mock.ExpectExec("UPDATE(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
// 查找父目录,存在
mock.ExpectQuery("SELECT(.+)folders").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
// 查找同名文件,不存在
mock.ExpectQuery("SELECT(.+)files").WillReturnRows(sqlmock.NewRows([]string{"id"}))
// 创建目录
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)folders(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
// 插入文件记录
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)files(.+)").WillReturnResult(sqlmock.NewResult(2, 1))
mock.ExpectCommit()
task.Do()
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(task.Err)
task.Err = nil
}
}
func TestNewImportTask(t *testing.T) {
asserts := assert.New(t)
// 成功
{
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectCommit()
job, err := NewImportTask(1, 1, "/", "/", false)
asserts.NoError(mock.ExpectationsWereMet())
asserts.NotNil(job)
asserts.NoError(err)
}
// 失败
{
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
mock.ExpectBegin()
mock.ExpectExec("INSERT(.+)").WillReturnError(errors.New("error"))
mock.ExpectRollback()
job, err := NewImportTask(1, 1, "/", "/", false)
asserts.NoError(mock.ExpectationsWereMet())
asserts.Nil(job)
asserts.Error(err)
}
}
func TestNewImportTaskFromModel(t *testing.T) {
asserts := assert.New(t)
// 成功
{
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
job, err := NewImportTaskFromModel(&model.Task{Props: "{}"})
asserts.NoError(mock.ExpectationsWereMet())
asserts.NoError(err)
asserts.NotNil(job)
}
// JSON解析失败
{
mock.ExpectQuery("SELECT(.+)").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1))
job, err := NewImportTaskFromModel(&model.Task{Props: "?"})
asserts.NoError(mock.ExpectationsWereMet())
asserts.Error(err)
asserts.Nil(job)
}
}
+8
View File
@@ -13,6 +13,8 @@ const (
DecompressTaskType
// TransferTaskType 中转任务
TransferTaskType
// ImportTaskType 导入任务
ImportTaskType
)
// 任务状态
@@ -41,6 +43,10 @@ const (
DownloadingProgress
// Transferring 转存中
TransferringProgress
// ListingProgress 索引中
ListingProgress
// InsertingProgress 插入中
InsertingProgress
)
// Job 任务接口
@@ -103,6 +109,8 @@ func GetJobFromModel(task *model.Task) (Job, error) {
return NewDecompressTaskFromModel(task)
case TransferTaskType:
return NewTransferTaskFromModel(task)
case ImportTaskType:
return NewImportTaskFromModel(task)
default:
return nil, ErrUnknownTaskType
}
+6
View File
@@ -3,6 +3,7 @@ package util
import (
"fmt"
"github.com/fatih/color"
"sync"
"time"
)
@@ -23,6 +24,7 @@ var Level = LevelDebug
// Logger 日志
type Logger struct {
level int
mu sync.Mutex
}
// 日志颜色
@@ -49,6 +51,10 @@ func (ll *Logger) Println(prefix string, msg string) {
// color.NoColor = false
c := color.New()
ll.mu.Lock()
defer ll.mu.Unlock()
_, _ = c.Printf(
"%s%s %s %s\n",
colors[prefix]("["+prefix+"]"),
+7 -2
View File
@@ -281,7 +281,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request, fs *files
// 尝试作为文件删除
if ok, file := fs.IsFileExist(reqPath); ok {
if err := fs.Delete(ctx, []uint{}, []uint{file.ID}); err != nil {
if err := fs.Delete(ctx, []uint{}, []uint{file.ID}, false); err != nil {
return http.StatusMethodNotAllowed, err
}
return http.StatusNoContent, nil
@@ -289,7 +289,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request, fs *files
// 尝试作为目录删除
if ok, folder := fs.IsPathExist(reqPath); ok {
if err := fs.Delete(ctx, []uint{folder.ID}, []uint{}); err != nil {
if err := fs.Delete(ctx, []uint{folder.ID}, []uint{}, false); err != nil {
return http.StatusMethodNotAllowed, err
}
return http.StatusNoContent, nil
@@ -400,6 +400,11 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request, fs *filesy
if r.ContentLength > 0 {
return http.StatusUnsupportedMediaType, nil
}
if strings.Contains(r.UserAgent(), "rclone") {
if _, ok := ctx.Value(fsctx.IgnoreConflictCtx).(bool); !ok {
ctx = context.WithValue(ctx, fsctx.IgnoreConflictCtx, true)
}
}
if _, err := fs.CreateDirectory(ctx, reqPath); err != nil {
return http.StatusConflict, err
}
+22
View File
@@ -402,3 +402,25 @@ func AdminDeleteTask(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// AdminCreateImportTask 新建文件导入任务
func AdminCreateImportTask(c *gin.Context) {
var service admin.ImportTaskService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Create(c, CurrentUser(c))
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
// AdminListFolders 列出用户或外部文件系统目录
func AdminListFolders(c *gin.Context) {
var service admin.ListFolderService
if err := c.ShouldBindUri(&service); err == nil {
res := service.List(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
+11
View File
@@ -353,3 +353,14 @@ func SearchFile(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// CreateFile 创建空白文件
func CreateFile(c *gin.Context) {
var service explorer.SingleFileService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.Create(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
+1
View File
@@ -13,6 +13,7 @@ import (
func SiteConfig(c *gin.Context) {
siteConfig := model.GetSettingByNames(
"siteName",
"siteICPId",
"login_captcha",
"reg_captcha",
"email_active",
+11
View File
@@ -158,3 +158,14 @@ func SlavePing(c *gin.Context) {
c.JSON(200, ErrorResponse(err))
}
}
// SlaveList 从机列出文件
func SlaveList(c *gin.Context) {
var service explorer.SlaveListService
if err := c.ShouldBindJSON(&service); err == nil {
res := service.List(c)
c.JSON(200, res)
} else {
c.JSON(200, ErrorResponse(err))
}
}
+12 -2
View File
@@ -49,6 +49,8 @@ func InitSlaveRouter() *gin.Engine {
v3.GET("thumb/:path", controllers.SlaveThumb)
// 删除文件
v3.POST("delete", controllers.SlaveDelete)
// 列出文件
v3.POST("list", controllers.SlaveList)
}
return r
}
@@ -80,6 +82,7 @@ func InitMasterRouter() *gin.Engine {
静态资源
*/
r.Use(gzip.Gzip(gzip.DefaultCompression, gzip.WithExcludedPaths([]string{"/api/"})))
r.Use(middleware.InjectSiteInfo())
r.Use(static.Serve("/", bootstrap.StaticFS))
r.GET("manifest.json", controllers.Manifest)
@@ -274,6 +277,8 @@ func InitMasterRouter() *gin.Engine {
middleware.ShareCanPreview(),
controllers.ShareThumb,
)
// 搜索公共分享
v3.Group("share").GET("search", controllers.SearchShare)
}
// 需要登录保护的
@@ -363,6 +368,9 @@ func InitMasterRouter() *gin.Engine {
file.GET("preview/:id", controllers.AdminGetFile)
// 删除
file.POST("delete", controllers.AdminDeleteFile)
// 列出用户或外部文件系统目录
file.GET("folders/:type/:id/*path",
controllers.AdminListFolders)
}
share := admin.Group("share")
@@ -387,6 +395,8 @@ func InitMasterRouter() *gin.Engine {
task.POST("list", controllers.AdminListTask)
// 删除
task.POST("delete", controllers.AdminDeleteTask)
// 新建文件导入任务
task.POST("import", controllers.AdminCreateImportTask)
}
}
@@ -436,6 +446,8 @@ func InitMasterRouter() *gin.Engine {
file.GET("upload/credential", controllers.GetUploadCredential)
// 更新文件
file.PUT("update/:id", controllers.PutContent)
// 创建空白文件
file.POST("create", controllers.CreateFile)
// 创建文件下载会话
file.PUT("download/:id", controllers.CreateDownloadSession)
// 预览文件
@@ -504,8 +516,6 @@ func InitMasterRouter() *gin.Engine {
share.POST("", controllers.CreateShare)
// 列出我的分享
share.GET("", controllers.ListShare)
// 搜索公共分享
share.GET("search", controllers.SearchShare)
// 更新分享属性
share.PATCH(":id",
middleware.ShareAvailable(),
+68 -2
View File
@@ -18,7 +18,73 @@ type FileService struct {
// FileBatchService 文件批量操作服务
type FileBatchService struct {
ID []uint `json:"id" binding:"min=1"`
ID []uint `json:"id" binding:"min=1"`
Force bool `json:"force"`
}
// ListFolderService 列目录结构
type ListFolderService struct {
Path string `uri:"path" binding:"required,max=65535"`
ID uint `uri:"id" binding:"required"`
Type string `uri:"type" binding:"eq=policy|eq=user"`
}
// List 列出指定路径下的目录
func (service *ListFolderService) List(c *gin.Context) serializer.Response {
if service.Type == "policy" {
// 列取存储策略中的目录
policy, err := model.GetPolicyByID(service.ID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "存储策略不存在", err)
}
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "无法创建文件系统", err)
}
defer fs.Recycle()
// 列取存储策略中的文件
fs.Policy = &policy
res, err := fs.ListPhysical(c.Request.Context(), service.Path)
if err != nil {
return serializer.Err(serializer.CodeIOFailed, "无法列取目录", err)
}
return serializer.Response{
Data: map[string]interface{}{
"objects": res,
},
}
}
// 列取用户空间目录
// 查找用户
user, err := model.GetUserByID(service.ID)
if err != nil {
return serializer.Err(serializer.CodeNotFound, "用户不存在", err)
}
// 创建文件系统
fs, err := filesystem.NewFileSystem(&user)
if err != nil {
return serializer.Err(serializer.CodeInternalSetting, "无法创建文件系统", err)
}
defer fs.Recycle()
// 列取目录
res, err := fs.List(c.Request.Context(), service.Path, nil)
if err != nil {
return serializer.Err(serializer.CodeIOFailed, "无法列取目录", err)
}
return serializer.Response{
Data: map[string]interface{}{
"objects": res,
},
}
}
// Delete 删除文件
@@ -58,7 +124,7 @@ func (service *FileBatchService) Delete(c *gin.Context) serializer.Response {
}
// 执行删除
fs.Delete(context.Background(), []uint{}, ids)
fs.Delete(context.Background(), []uint{}, ids, service.Force)
fs.Recycle()
}
}(userFile)
+21
View File
@@ -3,6 +3,7 @@ package admin
import (
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/HFO4/cloudreve/pkg/task"
"github.com/gin-gonic/gin"
"strings"
)
@@ -12,6 +13,26 @@ type TaskBatchService struct {
ID []uint `json:"id" binding:"min=1"`
}
// ImportTaskService 导入任务
type ImportTaskService struct {
UID uint `json:"uid" binding:"required"`
PolicyID uint `json:"policy_id" binding:"required"`
Src string `json:"src" binding:"required,min=1,max=65535"`
Dst string `json:"dst" binding:"required,min=1,max=65535"`
Recursive bool `json:"recursive"`
}
// Create 新建导入任务
func (service *ImportTaskService) Create(c *gin.Context, user *model.User) serializer.Response {
// 创建任务
job, err := task.NewImportTask(service.UID, service.PolicyID, service.Src, service.Dst, service.Recursive)
if err != nil {
return serializer.Err(serializer.CodeNotSet, "任务创建失败", err)
}
task.TaskPoll.Submit(job)
return serializer.Response{}
}
// Delete 删除任务
func (service *TaskBatchService) Delete(c *gin.Context) serializer.Response {
if err := model.DB.Where("id in (?)", service.ID).Delete(&model.Download{}).Error; err != nil {
+1 -1
View File
@@ -65,7 +65,7 @@ func (service *UserBatchService) Delete() serializer.Response {
if err != nil {
return serializer.Err(serializer.CodeNotFound, "无法找到用户根目录", err)
}
fs.Delete(context.Background(), []uint{root.ID}, []uint{})
fs.Delete(context.Background(), []uint{root.ID}, []uint{}, false)
// 删除相关任务
model.DB.Where("user_id = ?", uid).Delete(&model.Download{})
+61 -1
View File
@@ -13,15 +13,18 @@ import (
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/gin-gonic/gin"
"github.com/jinzhu/gorm"
"io/ioutil"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"time"
)
// SingleFileService 对单文件进行操作的五福,path为文件完整路径
type SingleFileService struct {
Path string `uri:"path" binding:"required,min=1,max=65535"`
Path string `uri:"path" json:"path" binding:"required,min=1,max=65535"`
}
// FileIDService 通过文件ID对文件进行操作的服务
@@ -56,6 +59,63 @@ type SlaveFilesService struct {
Files []string `json:"files" binding:"required,gt=0"`
}
// SlaveListService 从机列表服务
type SlaveListService struct {
Path string `json:"path" binding:"required,min=1,max=65535"`
Recursive bool `json:"recursive"`
}
// New 创建新文件
func (service *SingleFileService) Create(c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewFileSystemFromContext(c)
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
// 上下文
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// 给文件系统分配钩子
fs.Use("BeforeUpload", filesystem.HookValidateFile)
fs.Use("AfterUpload", filesystem.GenericAfterUpload)
// 上传空文件
err = fs.Upload(ctx, local.FileStream{
File: ioutil.NopCloser(strings.NewReader("")),
Size: 0,
VirtualPath: path.Dir(service.Path),
Name: path.Base(service.Path),
})
if err != nil {
return serializer.Err(serializer.CodeUploadFailed, err.Error(), err)
}
return serializer.Response{
Code: 0,
}
}
// List 列出从机上的文件
func (service *SlaveListService) List(c *gin.Context) serializer.Response {
// 创建文件系统
fs, err := filesystem.NewAnonymousFileSystem()
if err != nil {
return serializer.Err(serializer.CodePolicyNotAllowed, err.Error(), err)
}
defer fs.Recycle()
objects, err := fs.Handler.List(context.Background(), service.Path, service.Recursive)
if err != nil {
return serializer.Err(serializer.CodeIOFailed, "无法列取文件", err)
}
res, _ := json.Marshal(objects)
return serializer.Response{Data: string(res)}
}
// DownloadArchived 下載已打包的多文件
func (service *DownloadService) DownloadArchived(ctx context.Context, c *gin.Context) serializer.Response {
// 创建文件系统
+6 -1
View File
@@ -146,6 +146,11 @@ func (service *ItemCompressService) CreateCompressTask(c *gin.Context) serialize
return serializer.Err(serializer.CodeGroupNotAllowed, "当前用户组无法进行此操作", nil)
}
// 补齐压缩文件扩展名(如果没有)
if !strings.HasSuffix(service.Name, ".zip") {
service.Name += ".zip"
}
// 存放目录是否存在,是否重名
if exist, _ := fs.IsPathExist(service.Dst); !exist {
return serializer.Err(serializer.CodeNotFound, "存放路径不存在", nil)
@@ -264,7 +269,7 @@ func (service *ItemIDService) Delete(ctx context.Context, c *gin.Context) serial
// 删除对象
items := service.Raw()
err = fs.Delete(ctx, items.Dirs, items.Items)
err = fs.Delete(ctx, items.Dirs, items.Items, false)
if err != nil {
return serializer.Err(serializer.CodeNotSet, err.Error(), err)
}