mirror of
https://github.com/ouqiang/gocron.git
synced 2024-04-21 12:31:58 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
21028ec6f0 | ||
|
|
c4af1653ff | ||
|
|
250cbdde7c | ||
|
|
a966f3aeda | ||
|
|
3c02d55ebf | ||
|
|
4009e845cb | ||
|
|
37f2405ab3 | ||
|
|
8ec80a02b6 |
@@ -33,7 +33,7 @@
|
||||
|
||||
|
||||
## 下载
|
||||
[v1.2](https://github.com/ouqiang/gocron/releases/tag/v1.2)
|
||||
[v1.2.2](https://github.com/ouqiang/gocron/releases/tag/v1.2.2)
|
||||
|
||||
[版本升级](https://github.com/ouqiang/gocron/wiki/版本升级)
|
||||
|
||||
@@ -71,8 +71,10 @@
|
||||
* -h 查看帮助
|
||||
* gocron-node
|
||||
* -allow-root *nix平台允许以root用户运行
|
||||
* -s ip:port 监听地址
|
||||
* -cert-file 证书文件
|
||||
* -s ip:port 监听地址
|
||||
* -enable-tls 开启TLS
|
||||
* -ca-file CA证书文件
|
||||
* -cert-file 证书文件
|
||||
* -key-file 私钥文件
|
||||
* -h 查看帮助
|
||||
* -v 查看版本
|
||||
@@ -81,7 +83,7 @@
|
||||
- [x] 版本升级
|
||||
- [x] 批量开启、关闭、删除任务
|
||||
- [x] 调度器与任务节点通信支持https
|
||||
- [ ] 任务分组
|
||||
- [x] 任务分组
|
||||
- [ ] 多用户
|
||||
- [ ] 权限控制
|
||||
- [ ] 新增任务API接口
|
||||
@@ -99,12 +101,12 @@
|
||||
|
||||
## ChangeLog
|
||||
|
||||
v1.2
|
||||
v1.2.2
|
||||
--------
|
||||
* 用户登录页增加图形验证码
|
||||
* 支持从旧版本升级
|
||||
* 任务批量开启、关闭、删除
|
||||
* 调度器与任务节点支持HTTPS通信
|
||||
* 调度器与任务节点支持HTTPS双向认证
|
||||
* 修复任务列表页总记录数显示错误
|
||||
|
||||
|
||||
|
||||
+31
-17
@@ -10,47 +10,61 @@ import (
|
||||
"os"
|
||||
"fmt"
|
||||
"strings"
|
||||
"github.com/ouqiang/gocron/modules/rpc/auth"
|
||||
"github.com/ouqiang/gocron/modules/utils"
|
||||
)
|
||||
|
||||
const AppVersion = "1.2"
|
||||
const AppVersion = "1.2.2"
|
||||
|
||||
func main() {
|
||||
var serverAddr string
|
||||
var allowRoot bool
|
||||
var version bool
|
||||
var keyFile string
|
||||
var CAFile string
|
||||
var certFile string
|
||||
var keyFile string
|
||||
var enableTLS bool
|
||||
flag.BoolVar(&allowRoot, "allow-root", false, "./gocron-node -allow-root")
|
||||
flag.StringVar(&serverAddr, "s", "0.0.0.0:5921", "./gocron-node -s ip:port")
|
||||
flag.BoolVar(&version, "v", false, "./gocron-node -v")
|
||||
flag.BoolVar(&enableTLS, "enable-tls", false, "./gocron-node -enable-tls")
|
||||
flag.StringVar(&CAFile, "ca-file", "", "./gocron-node -ca-file path")
|
||||
flag.StringVar(&certFile, "cert-file", "", "./gocron-node -cert-file path")
|
||||
flag.StringVar(&keyFile, "key-file", "", "./gocron-node -key-file path")
|
||||
flag.BoolVar(&version, "v", false, "./gocron-node -v")
|
||||
flag.Parse()
|
||||
|
||||
if version {
|
||||
fmt.Println(AppVersion)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
certFile = strings.TrimSpace(certFile)
|
||||
keyFile = strings.TrimSpace(keyFile)
|
||||
|
||||
if certFile != "" && keyFile == "" {
|
||||
fmt.Println("missing argument key-file")
|
||||
return
|
||||
}
|
||||
|
||||
if keyFile != "" && certFile == "" {
|
||||
fmt.Println("missing argument cert-file")
|
||||
return
|
||||
if (enableTLS) {
|
||||
if !utils.FileExist(CAFile) {
|
||||
fmt.Printf("failed to read ca cert file: %s", CAFile)
|
||||
return
|
||||
}
|
||||
if !utils.FileExist(certFile) {
|
||||
fmt.Printf("failed to read server cert file: %s", certFile)
|
||||
return
|
||||
}
|
||||
if !utils.FileExist(keyFile) {
|
||||
fmt.Printf("failed to read server key file: %s", keyFile)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
certificate := auth.Certificate{
|
||||
CAFile: strings.TrimSpace(CAFile),
|
||||
CertFile: strings.TrimSpace(certFile),
|
||||
KeyFile: strings.TrimSpace(keyFile),
|
||||
}
|
||||
|
||||
|
||||
if runtime.GOOS != "windows" && os.Getuid() == 0 && !allowRoot {
|
||||
fmt.Println("Do not run gocron-node as root user")
|
||||
os.Exit(1)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
|
||||
server.Start(serverAddr, certFile, keyFile)
|
||||
server.Start(serverAddr, enableTLS, certificate)
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/ouqiang/gocron/cmd"
|
||||
)
|
||||
|
||||
const AppVersion = "1.2"
|
||||
const AppVersion = "1.2.2"
|
||||
|
||||
func main() {
|
||||
app := cli.NewApp()
|
||||
|
||||
+1
-2
@@ -10,7 +10,6 @@ type Host struct {
|
||||
Name string `xorm:"varchar(64) notnull"` // 主机名称
|
||||
Alias string `xorm:"varchar(32) notnull default '' "` // 主机别名
|
||||
Port int `xorm:"notnull default 22"` // 主机端口
|
||||
CertFile string `xorm:"varchar(64) notnull default '' "`
|
||||
Remark string `xorm:"varchar(100) notnull default '' "` // 备注
|
||||
BaseModel `xorm:"-"`
|
||||
Selected bool `xorm:"-"`
|
||||
@@ -27,7 +26,7 @@ func (host *Host) Create() (insertId int16, err error) {
|
||||
}
|
||||
|
||||
func (host *Host) UpdateBean(id int16) (int64, error) {
|
||||
return Db.ID(id).Cols("name,alias,port,cert_file,remark").Update(host)
|
||||
return Db.ID(id).Cols("name,alias,port,remark").Update(host)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+24
-13
@@ -49,22 +49,30 @@ func isDatabaseExist(name string) bool {
|
||||
|
||||
// 迭代升级数据库, 新建表、新增字段等
|
||||
func (migration *Migration) Upgrade(oldVersionId int) {
|
||||
versionIds := []int{110, 120}
|
||||
upgradeFuncs := []func(*xorm.Session) error {
|
||||
migration.upgradeFor110,
|
||||
migration.upgradeFor120,
|
||||
// v1.2版本不支持升级
|
||||
if oldVersionId == 120 {
|
||||
return
|
||||
}
|
||||
|
||||
// 默认当前版本为v1.0
|
||||
startIndex := 0
|
||||
versionIds := []int{110, 122}
|
||||
upgradeFuncs := []func(*xorm.Session) error {
|
||||
migration.upgradeFor110,
|
||||
migration.upgradeFor122,
|
||||
}
|
||||
|
||||
startIndex := -1
|
||||
// 从当前版本的下一版本开始升级
|
||||
for i, value := range versionIds {
|
||||
if oldVersionId == value {
|
||||
startIndex = i + 1
|
||||
if value > oldVersionId {
|
||||
startIndex = i
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if startIndex == -1 {
|
||||
return
|
||||
}
|
||||
|
||||
length := len(versionIds)
|
||||
if startIndex >= length {
|
||||
return
|
||||
@@ -137,12 +145,15 @@ func (migration *Migration) upgradeFor110(session *xorm.Session) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// 升级到1.2.2版本
|
||||
func (migration *Migration) upgradeFor122(session *xorm.Session) error {
|
||||
logger.Info("开始升级到v1.2.2")
|
||||
|
||||
// 升级到v1.2版本
|
||||
func (migration *Migration) upgradeFor120(session *xorm.Session) error {
|
||||
// host表增加cert_file字段
|
||||
tableName := TablePrefix + "host"
|
||||
_, err := session.Exec(fmt.Sprintf("ALTER TABLE %s Add COLUMN cert_file VARCHAR(64) NOT NULL DEFAULT ''", tableName))
|
||||
tableName := TablePrefix + "task"
|
||||
// task表增加tag字段
|
||||
_, err := session.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN tag VARCHAR(32) NOT NULL DEFAULT '' ", tableName))
|
||||
|
||||
logger.Info("已升级到v1.2.2\n")
|
||||
|
||||
return err
|
||||
}
|
||||
+20
-47
@@ -9,8 +9,8 @@ import (
|
||||
"strings"
|
||||
"github.com/ouqiang/gocron/modules/logger"
|
||||
"github.com/ouqiang/gocron/modules/app"
|
||||
"strconv"
|
||||
"time"
|
||||
"github.com/ouqiang/gocron/modules/setting"
|
||||
)
|
||||
|
||||
type Status int8
|
||||
@@ -65,27 +65,18 @@ func (model *BaseModel) pageLimitOffset() int {
|
||||
|
||||
// 创建Db
|
||||
func CreateDb() *xorm.Engine {
|
||||
config := getDbConfig()
|
||||
dsn := getDbEngineDSN(config["engine"], config)
|
||||
engine, err := xorm.NewEngine(config["engine"], dsn)
|
||||
dsn := getDbEngineDSN(app.Setting)
|
||||
engine, err := xorm.NewEngine(app.Setting.Db.Engine, dsn)
|
||||
if err != nil {
|
||||
logger.Fatal("创建xorm引擎失败", err)
|
||||
}
|
||||
maxIdleConns, err := strconv.Atoi(config["max_idle_conns"])
|
||||
maxOpenConns, err := strconv.Atoi(config["max_open_conns"])
|
||||
if maxIdleConns <= 0 {
|
||||
maxIdleConns = 30
|
||||
}
|
||||
if maxOpenConns <= 0 {
|
||||
maxOpenConns = 100
|
||||
}
|
||||
engine.SetMaxIdleConns(maxIdleConns)
|
||||
engine.SetMaxOpenConns(maxOpenConns)
|
||||
engine.SetMaxIdleConns(app.Setting.Db.MaxIdleConns)
|
||||
engine.SetMaxOpenConns(app.Setting.Db.MaxOpenConns)
|
||||
|
||||
if config["prefix"] != "" {
|
||||
if app.Setting.Db.Prefix != "" {
|
||||
// 设置表前缀
|
||||
TablePrefix = config["prefix"]
|
||||
mapper := core.NewPrefixMapper(core.SnakeMapper{}, config["prefix"])
|
||||
TablePrefix = app.Setting.Db.Prefix
|
||||
mapper := core.NewPrefixMapper(core.SnakeMapper{}, app.Setting.Db.Prefix)
|
||||
engine.SetTableMapper(mapper)
|
||||
}
|
||||
// 本地环境开启日志
|
||||
@@ -100,48 +91,30 @@ func CreateDb() *xorm.Engine {
|
||||
}
|
||||
|
||||
// 创建临时数据库连接
|
||||
func CreateTmpDb(config map[string]string) (*xorm.Engine, error) {
|
||||
dsn := getDbEngineDSN(config["engine"], config)
|
||||
func CreateTmpDb(setting *setting.Setting) (*xorm.Engine, error) {
|
||||
dsn := getDbEngineDSN(setting)
|
||||
|
||||
return xorm.NewEngine(config["engine"], dsn)
|
||||
return xorm.NewEngine(setting.Db.Engine, dsn)
|
||||
}
|
||||
|
||||
// 获取数据库引擎DSN mysql,sqlite
|
||||
func getDbEngineDSN(engine string, config map[string]string) string {
|
||||
engine = strings.ToLower(engine)
|
||||
func getDbEngineDSN(setting *setting.Setting) string {
|
||||
engine := strings.ToLower(setting.Db.Engine)
|
||||
var dsn string = ""
|
||||
switch engine {
|
||||
case "mysql":
|
||||
dsn = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=%s",
|
||||
config["user"],
|
||||
config["password"],
|
||||
config["host"],
|
||||
config["port"],
|
||||
config["database"],
|
||||
config["charset"])
|
||||
dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s",
|
||||
setting.Db.User,
|
||||
setting.Db.Password,
|
||||
setting.Db.Host,
|
||||
setting.Db.Port ,
|
||||
setting.Db.Database,
|
||||
setting.Db.Charset)
|
||||
}
|
||||
|
||||
return dsn
|
||||
}
|
||||
|
||||
|
||||
// 获取数据库配置
|
||||
func getDbConfig() map[string]string {
|
||||
var db map[string]string = make(map[string]string)
|
||||
db["user"] = app.Setting.Key("db.user").String()
|
||||
db["password"] = app.Setting.Key("db.password").String()
|
||||
db["host"] = app.Setting.Key("db.host").String()
|
||||
db["port"] = app.Setting.Key("db.port").String()
|
||||
db["database"] = app.Setting.Key("db.database").String()
|
||||
db["charset"] = app.Setting.Key("db.charset").String()
|
||||
db["prefix"] = app.Setting.Key("db.prefix").String()
|
||||
db["engine"] = app.Setting.Key("db.engine").String()
|
||||
db["max_idle_conns"] = app.Setting.Key("db.max.idle.conns").String()
|
||||
db["max_open_conns"] = app.Setting.Key("db.max.open.conns").String()
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
func keepDbAlived(engine *xorm.Engine) {
|
||||
t := time.Tick(180 * time.Second)
|
||||
for {
|
||||
|
||||
+8
-1
@@ -44,6 +44,7 @@ type Task struct {
|
||||
NotifyStatus int8 `xorm:"smallint notnull default 1"` // 任务执行结束是否通知 0: 不通知 1: 失败通知 2: 执行结束通知
|
||||
NotifyType int8 `xorm:"smallint notnull default 0"` // 通知类型 1: 邮件 2: slack
|
||||
NotifyReceiverId string `xorm:"varchar(256) notnull default '' "` // 通知接受者ID, setting表主键ID,多个ID逗号分隔
|
||||
Tag string `xorm:"varchar(32) notnull default ''"`
|
||||
Remark string `xorm:"varchar(100) notnull default ''"` // 备注
|
||||
Status Status `xorm:"tinyint notnull index default 0"` // 状态 1:正常 0:停止
|
||||
Created time.Time `xorm:"datetime notnull created"` // 创建时间
|
||||
@@ -73,6 +74,7 @@ func (task *Task) CreateTestTask() {
|
||||
task.Level = TaskLevelParent
|
||||
task.Protocol = TaskHTTP
|
||||
task.Spec = "*/30 * * * * *"
|
||||
task.Tag = "test-task"
|
||||
// 查询IP地址区域信息
|
||||
task.Command = "http://ip.taobao.com/service/getIpInfo.php?ip=117.27.140.253"
|
||||
task.Status = Enabled
|
||||
@@ -81,7 +83,7 @@ func (task *Task) CreateTestTask() {
|
||||
|
||||
func (task *Task) UpdateBean(id int) (int64, error) {
|
||||
return Db.ID(id).
|
||||
Cols("name,spec,protocol,command,timeout,multi,retry_times,remark,notify_status,notify_type,notify_receiver_id, dependency_task_id, dependency_status").
|
||||
Cols("name,spec,protocol,command,timeout,multi,retry_times,remark,notify_status,notify_type,notify_receiver_id, dependency_task_id, dependency_status, tag").
|
||||
Update(task)
|
||||
}
|
||||
|
||||
@@ -261,5 +263,10 @@ func (task *Task) parseWhere(session *xorm.Session, params CommonMap) {
|
||||
if ok && status.(int) > -1 {
|
||||
session.And("status = ?", status)
|
||||
}
|
||||
|
||||
tag, ok := params["Tag"]
|
||||
if ok && tag.(string) != "" {
|
||||
session.And("tag = ? ", tag)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-2
@@ -12,7 +12,6 @@ type TaskHostDetail struct {
|
||||
Name string
|
||||
Port int
|
||||
Alias string
|
||||
CertFile string
|
||||
}
|
||||
|
||||
func (TaskHostDetail) TableName() string {
|
||||
@@ -49,7 +48,7 @@ func (th *TaskHost) Add(taskId int, hostIds []int) error {
|
||||
|
||||
func (th *TaskHost) GetHostIdsByTaskId(taskId int) ([]TaskHostDetail, error) {
|
||||
list := make([]TaskHostDetail, 0)
|
||||
fields := "th.id,th.host_id,h.alias,h.name,h.port,h.cert_file"
|
||||
fields := "th.id,th.host_id,h.alias,h.name,h.port"
|
||||
err := Db.Alias("th").
|
||||
Join("LEFT", hostTableName(), "th.host_id=h.id").
|
||||
Where("th.task_id = ?", taskId).
|
||||
|
||||
+2
-2
@@ -5,10 +5,10 @@ import (
|
||||
|
||||
"github.com/ouqiang/gocron/modules/logger"
|
||||
"github.com/ouqiang/gocron/modules/utils"
|
||||
"gopkg.in/ini.v1"
|
||||
"io/ioutil"
|
||||
"strconv"
|
||||
"strings"
|
||||
"github.com/ouqiang/gocron/modules/setting"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -18,7 +18,7 @@ var (
|
||||
DataDir string // 存放session等
|
||||
AppConfig string // 应用配置文件
|
||||
Installed bool // 应用是否安装过
|
||||
Setting *ini.Section // 应用配置
|
||||
Setting *setting.Setting // 应用配置
|
||||
VersionId int // 版本号
|
||||
VersionFile string // 版本号文件
|
||||
)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io/ioutil"
|
||||
"errors"
|
||||
"fmt"
|
||||
"google.golang.org/grpc/credentials"
|
||||
)
|
||||
|
||||
type Certificate struct {
|
||||
CAFile string
|
||||
CertFile string
|
||||
KeyFile string
|
||||
ServerName string
|
||||
}
|
||||
|
||||
func (c Certificate) GetTLSConfigForServer() (*tls.Config, error) {
|
||||
certificate, err := tls.LoadX509KeyPair(
|
||||
c.CertFile,
|
||||
c.KeyFile,
|
||||
)
|
||||
|
||||
certPool := x509.NewCertPool()
|
||||
bs, err := ioutil.ReadFile(c.CAFile)
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf("failed to read client ca cert: %s", err))
|
||||
}
|
||||
|
||||
ok := certPool.AppendCertsFromPEM(bs)
|
||||
if !ok {
|
||||
return nil, errors.New("failed to append client certs")
|
||||
}
|
||||
|
||||
|
||||
tlsConfig := &tls.Config{
|
||||
ClientAuth: tls.RequireAndVerifyClientCert,
|
||||
Certificates: []tls.Certificate{certificate},
|
||||
ClientCAs: certPool,
|
||||
}
|
||||
|
||||
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
func (c Certificate) GetTransportCredsForClient() (credentials.TransportCredentials, error) {
|
||||
certificate, err := tls.LoadX509KeyPair(
|
||||
c.CertFile,
|
||||
c.KeyFile,
|
||||
)
|
||||
|
||||
certPool := x509.NewCertPool()
|
||||
bs, err := ioutil.ReadFile(c.CAFile)
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf("failed to read ca cert: %s", err))
|
||||
}
|
||||
|
||||
ok := certPool.AppendCertsFromPEM(bs)
|
||||
if !ok {
|
||||
return nil, errors.New("failed to append certs")
|
||||
}
|
||||
|
||||
transportCreds := credentials.NewTLS(&tls.Config{
|
||||
ServerName: c.ServerName,
|
||||
Certificates: []tls.Certificate{certificate},
|
||||
RootCAs: certPool,
|
||||
})
|
||||
|
||||
return transportCreds, nil
|
||||
}
|
||||
@@ -16,11 +16,11 @@ var (
|
||||
errUnavailable = errors.New("无法连接远程服务器")
|
||||
)
|
||||
|
||||
func ExecWithRetry(ip string, port int, certFile string,taskReq *pb.TaskRequest) (string, error) {
|
||||
tryTimes := 15
|
||||
func ExecWithRetry(ip string, port int, taskReq *pb.TaskRequest) (string, error) {
|
||||
tryTimes := 60
|
||||
i := 0
|
||||
for i < tryTimes {
|
||||
output, err := Exec(ip, port, certFile, taskReq)
|
||||
output, err := Exec(ip, port, taskReq)
|
||||
if err != errUnavailable {
|
||||
return output, err
|
||||
}
|
||||
@@ -31,14 +31,14 @@ func ExecWithRetry(ip string, port int, certFile string,taskReq *pb.TaskRequest)
|
||||
return "", errUnavailable
|
||||
}
|
||||
|
||||
func Exec(ip string, port int, certFile string, taskReq *pb.TaskRequest) (string, error) {
|
||||
func Exec(ip string, port int, taskReq *pb.TaskRequest) (string, error) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
logger.Error("panic#rpc/client.go:Exec#", err)
|
||||
}
|
||||
} ()
|
||||
addr := fmt.Sprintf("%s:%d", ip, port)
|
||||
conn, err := grpcpool.Pool.Get(addr, certFile)
|
||||
conn, err := grpcpool.Pool.Get(addr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@ import (
|
||||
"time"
|
||||
"google.golang.org/grpc"
|
||||
"errors"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"github.com/ouqiang/gocron/modules/rpc/auth"
|
||||
"github.com/ouqiang/gocron/modules/app"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -32,12 +33,12 @@ type GRPCPool struct {
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
func (p *GRPCPool) Get(addr, certFile string) (*grpc.ClientConn, error) {
|
||||
func (p *GRPCPool) Get(addr string) (*grpc.ClientConn, error) {
|
||||
p.RLock()
|
||||
pool, ok := p.conns[addr]
|
||||
p.RUnlock()
|
||||
if !ok {
|
||||
err := p.newCommonPool(addr, certFile)
|
||||
err := p.newCommonPool(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -88,7 +89,7 @@ func (p *GRPCPool) ReleaseAll() {
|
||||
}
|
||||
|
||||
// 初始化底层连接池
|
||||
func (p *GRPCPool) newCommonPool(addr, certFile string) (error) {
|
||||
func (p *GRPCPool) newCommonPool(addr string) (error) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
commonPool, ok := p.conns[addr]
|
||||
@@ -99,17 +100,25 @@ func (p *GRPCPool) newCommonPool(addr, certFile string) (error) {
|
||||
InitialCap: 1,
|
||||
MaxCap: 30,
|
||||
Factory: func() (interface{}, error) {
|
||||
if certFile == "" {
|
||||
if !app.Setting.EnableTLS {
|
||||
return grpc.Dial(addr, grpc.WithInsecure())
|
||||
}
|
||||
|
||||
server := strings.Split(addr, ":")
|
||||
creds, err := credentials.NewClientTLSFromFile(certFile, server[0])
|
||||
|
||||
certificate := auth.Certificate{
|
||||
CAFile: app.Setting.CAFile,
|
||||
CertFile: app.Setting.CertFile,
|
||||
KeyFile: app.Setting.KeyFile,
|
||||
ServerName: server[0],
|
||||
}
|
||||
|
||||
transportCreds, err := certificate.GetTransportCredsForClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return grpc.Dial(addr, grpc.WithTransportCredentials(creds))
|
||||
return grpc.Dial(addr, grpc.WithTransportCredentials(transportCreds))
|
||||
},
|
||||
Close: func(v interface{}) error {
|
||||
conn, ok := v.(*grpc.ClientConn)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
pb "github.com/ouqiang/gocron/modules/rpc/proto"
|
||||
"github.com/ouqiang/gocron/modules/utils"
|
||||
"github.com/ouqiang/gocron/modules/rpc/auth"
|
||||
"google.golang.org/grpc/credentials"
|
||||
)
|
||||
|
||||
@@ -30,36 +31,35 @@ func (s Server) Run(ctx context.Context, req *pb.TaskRequest) (*pb.TaskResponse,
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func Start(addr, certFile, keyFile string) {
|
||||
func Start(addr string, enableTLS bool, certificate auth.Certificate) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
grpclog.Println("panic", err)
|
||||
}
|
||||
} ()
|
||||
|
||||
l, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
grpclog.Fatal(err)
|
||||
}
|
||||
|
||||
var s *grpc.Server
|
||||
if certFile != "" {
|
||||
// TLS认证
|
||||
creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
|
||||
if enableTLS {
|
||||
tlsConfig, err := certificate.GetTLSConfigForServer()
|
||||
if err != nil {
|
||||
grpclog.Fatalf("Failed to generate credentials %v", err)
|
||||
grpclog.Fatal(err)
|
||||
}
|
||||
|
||||
s = grpc.NewServer(grpc.Creds(creds))
|
||||
opt := grpc.Creds(credentials.NewTLS(tlsConfig))
|
||||
s = grpc.NewServer(opt)
|
||||
pb.RegisterTaskServer(s, Server{})
|
||||
grpclog.Printf("listen %s with TLS", addr)
|
||||
} else {
|
||||
s = grpc.NewServer()
|
||||
pb.RegisterTaskServer(s, Server{})
|
||||
grpclog.Println("listen ", addr)
|
||||
grpclog.Printf("listen %s", addr)
|
||||
}
|
||||
|
||||
err = s.Serve(l)
|
||||
if err != nil {
|
||||
grpclog.Fatal(err)
|
||||
}
|
||||
grpclog.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,19 +3,84 @@ package setting
|
||||
import (
|
||||
"errors"
|
||||
"gopkg.in/ini.v1"
|
||||
"github.com/ouqiang/gocron/modules/utils"
|
||||
"github.com/ouqiang/gocron/modules/logger"
|
||||
)
|
||||
|
||||
const DefaultSection = "default"
|
||||
|
||||
type Setting struct {
|
||||
Db struct{
|
||||
Engine string
|
||||
Host string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
Database string
|
||||
Prefix string
|
||||
Charset string
|
||||
MaxIdleConns int
|
||||
MaxOpenConns int
|
||||
}
|
||||
AllowIps string
|
||||
AppName string
|
||||
ApiKey string
|
||||
ApiSecret string
|
||||
ApiSignEnable bool
|
||||
|
||||
EnableTLS bool
|
||||
CAFile string
|
||||
CertFile string
|
||||
KeyFile string
|
||||
}
|
||||
|
||||
// 读取配置
|
||||
func Read(filename string) (*ini.Section,error) {
|
||||
func Read(filename string) (*Setting,error) {
|
||||
config, err := ini.Load(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
section := config.Section(DefaultSection)
|
||||
|
||||
return section, nil
|
||||
var s Setting
|
||||
|
||||
s.Db.Engine = section.Key("db.engine").MustString("mysql")
|
||||
s.Db.Host = section.Key("db.host").MustString("127.0.0.1")
|
||||
s.Db.Port = section.Key("db.port").MustInt(3306)
|
||||
s.Db.User = section.Key("db.user").MustString("")
|
||||
s.Db.Password = section.Key("db.password").MustString("")
|
||||
s.Db.Database = section.Key("db.database").MustString("gocron")
|
||||
s.Db.Prefix = section.Key("db.prefix").MustString("")
|
||||
s.Db.Charset = section.Key("db.charset").MustString("utf8")
|
||||
s.Db.MaxIdleConns = section.Key("db.max.idle.conns").MustInt(30)
|
||||
s.Db.MaxOpenConns = section.Key("db.max.open.conns").MustInt(100)
|
||||
|
||||
s.AllowIps = section.Key("allow_ips").MustString("")
|
||||
s.AppName = section.Key("app.name").MustString("定时任务管理系统")
|
||||
s.ApiKey = section.Key("api.key").MustString("")
|
||||
s.ApiSecret = section.Key("api.secret").MustString("")
|
||||
s.ApiSignEnable = section.Key("api.sign.enable").MustBool(true)
|
||||
|
||||
s.EnableTLS = section.Key("enable_tls").MustBool(false)
|
||||
s.CAFile = section.Key("ca_file").MustString("")
|
||||
s.CertFile = section.Key("cert_file").MustString("")
|
||||
s.KeyFile = section.Key("key_file").MustString("")
|
||||
|
||||
if s.EnableTLS {
|
||||
if !utils.FileExist(s.CAFile) {
|
||||
logger.Fatalf("failed to read ca cert file: %s", s.CAFile)
|
||||
}
|
||||
|
||||
if !utils.FileExist(s.CertFile) {
|
||||
logger.Fatalf("failed to read client cert file: %s", s.CertFile)
|
||||
}
|
||||
|
||||
if !utils.FileExist(s.KeyFile) {
|
||||
logger.Fatalf("failed to read client key file: %s", s.KeyFile)
|
||||
}
|
||||
}
|
||||
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// 写入配置
|
||||
|
||||
+5
-10
@@ -64,7 +64,6 @@ type HostForm struct {
|
||||
Name string `binding:"Required;MaxSize(64)"`
|
||||
Alias string `binding:"Required;MaxSize(32)"`
|
||||
Port int `binding:"Required;Range(1-65535)"`
|
||||
CertFile string
|
||||
Remark string
|
||||
}
|
||||
|
||||
@@ -94,12 +93,6 @@ func Store(ctx *macaron.Context, form HostForm) string {
|
||||
hostModel.Alias = strings.TrimSpace(form.Alias)
|
||||
hostModel.Port = form.Port
|
||||
hostModel.Remark = strings.TrimSpace(form.Remark)
|
||||
hostModel.CertFile = strings.TrimSpace(form.CertFile)
|
||||
|
||||
if hostModel.CertFile != "" && !utils.FileExist(hostModel.CertFile) {
|
||||
return json.CommonFailure("证书文件不存在或无权限访问")
|
||||
}
|
||||
|
||||
isCreate := false
|
||||
oldHostModel := new(models.Host)
|
||||
err = oldHostModel.Find(int(id))
|
||||
@@ -107,7 +100,6 @@ func Store(ctx *macaron.Context, form HostForm) string {
|
||||
return json.CommonFailure("主机不存在")
|
||||
}
|
||||
|
||||
|
||||
if id > 0 {
|
||||
_, err = hostModel.UpdateBean(id)
|
||||
} else {
|
||||
@@ -120,7 +112,10 @@ func Store(ctx *macaron.Context, form HostForm) string {
|
||||
|
||||
if !isCreate {
|
||||
oldAddr := fmt.Sprintf("%s:%d", oldHostModel.Name, oldHostModel.Port)
|
||||
grpcpool.Pool.Release(oldAddr)
|
||||
newAddr := fmt.Sprintf("%s:%d", hostModel.Name, hostModel.Port)
|
||||
if oldAddr != newAddr {
|
||||
grpcpool.Pool.Release(oldAddr)
|
||||
}
|
||||
|
||||
taskModel := new(models.Task)
|
||||
tasks, err := taskModel.ActiveListByHostId(id)
|
||||
@@ -180,7 +175,7 @@ func Ping(ctx *macaron.Context) string {
|
||||
taskReq := &rpc.TaskRequest{}
|
||||
taskReq.Command = "echo hello"
|
||||
taskReq.Timeout = 10
|
||||
output, err := client.Exec(hostModel.Name, hostModel.Port, hostModel.CertFile, taskReq)
|
||||
output, err := client.Exec(hostModel.Name, hostModel.Port, taskReq)
|
||||
if err != nil {
|
||||
return json.CommonFailure("连接失败-" + err.Error() + " " + output, err)
|
||||
}
|
||||
|
||||
@@ -120,6 +120,10 @@ func writeConfig(form InstallForm) error {
|
||||
"app.name", "定时任务管理系统", // 应用名称
|
||||
"api.key", "",
|
||||
"api.secret", "",
|
||||
"enable_tls", "false",
|
||||
"ca_file", "",
|
||||
"cert_file", "",
|
||||
"key_file", "",
|
||||
}
|
||||
|
||||
return setting.Write(dbConfig, app.AppConfig)
|
||||
@@ -139,14 +143,14 @@ func createAdminUser(form InstallForm) error {
|
||||
|
||||
// 测试数据库连接
|
||||
func testDbConnection(form InstallForm) error {
|
||||
var dbConfig map[string]string = make(map[string]string)
|
||||
dbConfig["engine"] = form.DbType
|
||||
dbConfig["host"] = form.DbHost
|
||||
dbConfig["port"] = strconv.Itoa(form.DbPort)
|
||||
dbConfig["user"] = form.DbUsername
|
||||
dbConfig["password"] = form.DbPassword
|
||||
dbConfig["charset"] = "utf8"
|
||||
db, err := models.CreateTmpDb(dbConfig)
|
||||
var s setting.Setting
|
||||
s.Db.Engine = form.DbType
|
||||
s.Db.Host = form.DbHost
|
||||
s.Db.Port = form.DbPort
|
||||
s.Db.User = form.DbUsername
|
||||
s.Db.Password = form.DbPassword
|
||||
s.Db.Charset = "utf8"
|
||||
db, err := models.CreateTmpDb(&s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+5
-9
@@ -184,7 +184,7 @@ func checkAppInstall(m *macaron.Macaron) {
|
||||
|
||||
// IP验证, 通过反向代理访问gocron,需设置Header X-Real-IP才能获取到客户端真实IP
|
||||
func ipAuth(ctx *macaron.Context) {
|
||||
allowIpsStr := app.Setting.Key("allow_ips").String()
|
||||
allowIpsStr := app.Setting.AllowIps
|
||||
if allowIpsStr == "" {
|
||||
return
|
||||
}
|
||||
@@ -230,20 +230,16 @@ func setShareData(ctx *macaron.Context, sess session.Store) {
|
||||
}
|
||||
ctx.Data["LoginUsername"] = user.Username(sess)
|
||||
ctx.Data["LoginUid"] = user.Uid(sess)
|
||||
ctx.Data["AppName"] = app.Setting.Key("app.name").String()
|
||||
ctx.Data["AppName"] = app.Setting.AppName
|
||||
}
|
||||
|
||||
/** API接口签名验证 **/
|
||||
func apiAuth(ctx *macaron.Context) {
|
||||
apiSignEnable := app.Setting.Key("api.sign.enable").String()
|
||||
apiSignEnable = strings.TrimSpace(apiSignEnable)
|
||||
if apiSignEnable == "false" {
|
||||
if !app.Setting.ApiSignEnable {
|
||||
return
|
||||
}
|
||||
apiKey := app.Setting.Key("api.key").String()
|
||||
apiSecret := app.Setting.Key("api.secret").String()
|
||||
apiKey = strings.TrimSpace(apiKey)
|
||||
apiSecret = strings.TrimSpace(apiSecret)
|
||||
apiKey := strings.TrimSpace(app.Setting.ApiKey)
|
||||
apiSecret := strings.TrimSpace(app.Setting.ApiSecret)
|
||||
json := utils.JsonResponse{}
|
||||
if apiKey == "" || apiSecret == "" {
|
||||
msg := json.CommonFailure("使用API前, 请先配置密钥")
|
||||
|
||||
@@ -29,6 +29,7 @@ type TaskForm struct {
|
||||
Multi int8 `binding:"In(1,2)"`
|
||||
RetryTimes int8
|
||||
HostId string
|
||||
Tag string
|
||||
Remark string
|
||||
NotifyStatus int8 `binding:"In(1,2,3)"`
|
||||
NotifyType int8 `binding:"In(1,2,3)"`
|
||||
@@ -63,8 +64,8 @@ func Index(ctx *macaron.Context) {
|
||||
if ok {
|
||||
safeNameHTML = template.HTMLEscapeString(name)
|
||||
}
|
||||
PageParams := fmt.Sprintf("id=%d&host_id=%d&name=%s&protocol=%d&status=%d&page_size=%d",
|
||||
queryParams["Id"], queryParams["HostId"], safeNameHTML, queryParams["Protocol"], queryParams["Status"], queryParams["PageSize"]);
|
||||
PageParams := fmt.Sprintf("id=%d&host_id=%d&name=%s&protocol=%d&tag=%s&status=%d&page_size=%d",
|
||||
queryParams["Id"], queryParams["HostId"], safeNameHTML, queryParams["Protocol"], queryParams["Tag"], queryParams["Status"], queryParams["PageSize"]);
|
||||
queryParams["PageParams"] = template.URL(PageParams)
|
||||
p := paginater.New(int(total), queryParams["PageSize"].(int), queryParams["Page"].(int), 5)
|
||||
ctx.Data["Pagination"] = p
|
||||
@@ -131,6 +132,7 @@ func Store(ctx *macaron.Context, form TaskForm) string {
|
||||
taskModel.Protocol = form.Protocol
|
||||
taskModel.Command = form.Command
|
||||
taskModel.Timeout = form.Timeout
|
||||
taskModel.Tag = form.Tag
|
||||
taskModel.Remark = form.Remark
|
||||
taskModel.Multi = form.Multi
|
||||
taskModel.RetryTimes = form.RetryTimes
|
||||
@@ -301,6 +303,7 @@ func parseQueryParams(ctx *macaron.Context) (models.CommonMap) {
|
||||
params["HostId"] = ctx.QueryInt("host_id")
|
||||
params["Name"] = ctx.QueryTrim("name")
|
||||
params["Protocol"] = ctx.QueryInt("protocol")
|
||||
params["Tag"] = ctx.QueryTrim("tag")
|
||||
status := ctx.QueryInt("status")
|
||||
if status >=0 {
|
||||
status -= 1
|
||||
|
||||
+1
-35
@@ -14,9 +14,6 @@ import (
|
||||
rpcClient "github.com/ouqiang/gocron/modules/rpc/client"
|
||||
pb "github.com/ouqiang/gocron/modules/rpc/proto"
|
||||
"strings"
|
||||
"text/template"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
)
|
||||
|
||||
// 定时任务调度管理器
|
||||
@@ -181,7 +178,7 @@ func (h *RPCHandler) Run(taskModel models.Task) (result string, err error) {
|
||||
var resultChan chan TaskResult = make(chan TaskResult, len(taskModel.Hosts))
|
||||
for _, taskHost := range taskModel.Hosts {
|
||||
go func(th models.TaskHostDetail) {
|
||||
output, err := rpcClient.ExecWithRetry(th.Name, th.Port, th.CertFile, taskRequest)
|
||||
output, err := rpcClient.ExecWithRetry(th.Name, th.Port, taskRequest)
|
||||
var errorMessage string = ""
|
||||
if err != nil {
|
||||
errorMessage = err.Error()
|
||||
@@ -304,9 +301,6 @@ func beforeExecJob(taskModel models.Task) (taskLogId int64) {
|
||||
|
||||
// 任务执行后置操作
|
||||
func afterExecJob(taskModel models.Task, taskResult TaskResult, taskLogId int64) {
|
||||
if taskResult.Err != nil {
|
||||
taskResult.Result = taskResult.Err.Error() + "\n" + taskResult.Result
|
||||
}
|
||||
_, err := updateTaskLog(taskLogId, taskResult)
|
||||
if err != nil {
|
||||
logger.Error("任务结束#更新任务日志失败-", err)
|
||||
@@ -349,39 +343,11 @@ func execDependencyTask(taskModel models.Task, taskResult TaskResult) {
|
||||
}
|
||||
serviceTask := new(Task)
|
||||
for _, task := range tasks {
|
||||
task.Command = appendResultToCommand(task.Command, taskResult)
|
||||
task.Spec = fmt.Sprintf("依赖任务(主任务ID-%d)", taskModel.Id)
|
||||
serviceTask.Run(task)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加主任务执行结果到子任务命令中, 占位符{{.Code}} {{.Message}}
|
||||
*/
|
||||
func appendResultToCommand(command string, taskResult TaskResult) string {
|
||||
var code int8 = 0
|
||||
if taskResult.Err != nil {
|
||||
code = 1
|
||||
}
|
||||
data := map[string]interface{} {
|
||||
"Code": code,
|
||||
"Message": base64.StdEncoding.EncodeToString([]byte(taskResult.Result)),
|
||||
}
|
||||
var buf *bytes.Buffer = new(bytes.Buffer)
|
||||
tmpl, err := template.New("command").Parse(command)
|
||||
if err != nil {
|
||||
logger.Errorf("替换子任务命令占位符失败#%s", err.Error())
|
||||
return command
|
||||
}
|
||||
err = tmpl.Execute(buf, data)
|
||||
if err != nil {
|
||||
logger.Errorf("替换子任务命令占位符失败#%s", err.Error())
|
||||
return command
|
||||
}
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// 发送任务结果通知
|
||||
func SendNotification(taskModel models.Task, taskResult TaskResult) {
|
||||
var statusName string
|
||||
|
||||
@@ -36,15 +36,6 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="two fields">
|
||||
<div class="field">
|
||||
<label>证书路径</label>
|
||||
<div class="ui small input">
|
||||
<input type="text" name="cert_file" value="{{{.Host.CertFile}}}"
|
||||
placeholder="data/certs/server.pem">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="two fields">
|
||||
<div class="field">
|
||||
<label>备注</label>
|
||||
|
||||
@@ -36,7 +36,6 @@
|
||||
<th>主机名</th>
|
||||
<th>别名</th>
|
||||
<th>端口</th>
|
||||
<th>证书</th>
|
||||
<th>备注</th>
|
||||
<th>操作</th>
|
||||
</tr>
|
||||
@@ -48,7 +47,6 @@
|
||||
<td>{{{.Name}}}</td>
|
||||
<td>{{{.Alias}}}</td>
|
||||
<td>{{{.Port}}}</td>
|
||||
<td>{{{.CertFile}}}</td>
|
||||
<td>{{{.Remark}}}</td>
|
||||
<td class="operation">
|
||||
<a class="ui purple button" href="/host/edit/{{{.Id}}}">编辑</a>
|
||||
|
||||
@@ -22,6 +22,11 @@
|
||||
<div class="field">
|
||||
<input type="text" placeholder="任务名称" name="name" value="{{{.Params.Name}}}">
|
||||
</div>
|
||||
<div class="field">
|
||||
<input type="text" placeholder="标签名称" name="tag" value="{{{.Params.Tag}}}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="six fields search">
|
||||
<div class="field">
|
||||
<select name="host_id" id="hostId">
|
||||
<option value="">选择节点</option>
|
||||
@@ -67,6 +72,7 @@
|
||||
<th>任务ID</th>
|
||||
<th>任务名称</th>
|
||||
<th>任务类型</th>
|
||||
<th>标签</th>
|
||||
<th>cron表达式</th>
|
||||
<th>执行方式</th>
|
||||
<th>超时时间</th>
|
||||
@@ -89,6 +95,7 @@
|
||||
<td>{{{.Id}}}</td>
|
||||
<td>{{{.Name}}}</td>
|
||||
<td>{{{if eq .Level 1}}}主任务{{{else}}}子任务{{{end}}}</td>
|
||||
<td>{{{.Tag}}}</td>
|
||||
<td>{{{.Spec}}}</td>
|
||||
<td>{{{if eq .Protocol 1}}} HTTP {{{else if eq .Protocol 2}}} SHELL {{{end}}}</td>
|
||||
<td>{{{if eq .Timeout -1}}}后台运行{{{else if gt .Timeout 0}}}{{{.Timeout}}}秒{{{else}}}不限制{{{end}}}</td>
|
||||
@@ -101,22 +108,23 @@
|
||||
</td>
|
||||
<td>
|
||||
{{{if eq .Level 1}}}
|
||||
{{{if eq .Status 1}}}<span style="color: green;">激活</span>{{{else}}}<span style="color: red;">停止<span>{{{end}}}
|
||||
{{{if eq .Status 1}}}<span><i class="checkmark big icon"></i></span>{{{else}}}<span><i class="minus big icon"></i><span>{{{end}}}
|
||||
{{{end}}}
|
||||
</td>
|
||||
<td>
|
||||
<div class="ui buttons operation">
|
||||
<a class="ui purple button" href="/task/edit/{{{.Id}}}">编辑</a>
|
||||
|
||||
<a href="/task/edit/{{{.Id}}}" ><i class="edit big icon" title="编辑"></i></a>
|
||||
{{{if eq .Level 1}}}
|
||||
{{{if eq .Status 1}}}
|
||||
<button class="ui primary button" @click="changeStatus({{{.Id}}},{{{.Status}}})">停止</button>
|
||||
<a href="javascript:void(0);" @click="changeStatus({{{.Id}}},{{{.Status}}})"><i class="pause circle big icon" title="停止"></i></a>
|
||||
{{{else}}}
|
||||
<button class="ui blue button" @click="changeStatus({{{.Id}}},{{{.Status}}})">激活 </button>
|
||||
<a href="javascript:void(0);" @click="changeStatus({{{.Id}}},{{{.Status}}})"><i class="play big icon" title="激活"></i></a>
|
||||
{{{end}}}
|
||||
{{{end}}}
|
||||
<button class="ui positive button" @click="remove({{{.Id}}})">删除</button> <br>
|
||||
<button class="ui twitter button" @click="run({{{.Id}}})">手动运行</button>
|
||||
<a class="ui instagram button" href="/task/log?task_id={{{.Id}}}">查看日志</a>
|
||||
<a href="javascript:void(0);" @click="remove({{{.Id}}})"><i class="remove big icon" title="删除"></i></a>
|
||||
<a href="javascript:void(0);" @click="run({{{.Id}}})"><i class="rocket big icon" title="手动执行"></i></a>
|
||||
<a href="/task/log?task_id={{{.Id}}}"><i class="bar chart icon big" title="查看日志"></i></a>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
@@ -22,6 +22,15 @@
|
||||
<input type="text" name="name" placeholder="任务名称" value="{{{.Task.Name}}}">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="field">
|
||||
<label>
|
||||
<div class="content">标签名称</div>
|
||||
</label>
|
||||
<div class="ui small input">
|
||||
<input type="text" name="tag" placeholder="标签用于任务分类" value="{{{.Task.Tag}}}">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="two fields">
|
||||
<div class="field">
|
||||
|
||||
Reference in New Issue
Block a user