feature(1.2.7): 新增 cron_server - 后台任务

- import jakecoffman/cron ;
- 可在 WEB 界面中进行新增任务、编辑任务、手动执行任务等;
- 新增 cron.log 记录后台任务执行的日志;
- 调整项目初始化,使其支持安装后台任务模块;
This commit is contained in:
新亮
2021-08-28 15:09:20 +08:00
parent b382919cb5
commit 382c7d3c8f
37 changed files with 2097 additions and 90 deletions
@@ -41,7 +41,7 @@ type listResponse struct {
Pagination struct {
Total int `json:"total"`
CurrentPage int `json:"current_page"`
PrePageCount int `json:"pre_page_count"`
PerPageCount int `json:"per_page_count"`
} `json:"pagination"`
}
@@ -109,7 +109,7 @@ func (h *handler) List() core.HandlerFunc {
return
}
res.Pagination.Total = cast.ToInt(resCountData)
res.Pagination.PrePageCount = pageSize
res.Pagination.PerPageCount = pageSize
res.Pagination.CurrentPage = page
res.List = make([]listData, len(resListData))
@@ -40,7 +40,7 @@ type listResponse struct {
Pagination struct {
Total int `json:"total"`
CurrentPage int `json:"current_page"`
PrePageCount int `json:"pre_page_count"`
PerPageCount int `json:"per_page_count"`
} `json:"pagination"`
}
@@ -109,7 +109,7 @@ func (h *handler) List() core.HandlerFunc {
return
}
res.Pagination.Total = cast.ToInt(resCountData)
res.Pagination.PrePageCount = pageSize
res.Pagination.PerPageCount = pageSize
res.Pagination.CurrentPage = page
res.List = make([]listData, len(resListData))
@@ -1,12 +1,12 @@
package cron_handler
import (
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"net/http"
"github.com/xinliangnote/go-gin-api/internal/api/service/cron_service"
"github.com/xinliangnote/go-gin-api/internal/pkg/code"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"github.com/xinliangnote/go-gin-api/pkg/errno"
"github.com/spf13/cast"
+68
View File
@@ -0,0 +1,68 @@
package cron_handler
import (
"net/http"
"github.com/xinliangnote/go-gin-api/internal/pkg/code"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"github.com/xinliangnote/go-gin-api/pkg/errno"
"github.com/spf13/cast"
)
type executeRequest struct {
Id string `uri:"id"` // HashID
}
type executeResponse struct {
Id int `json:"id"` // ID
}
// Execute 手动执行单条任务
// @Summary 手动执行单条任务
// @Description 手动执行单条任务
// @Tags API.cron
// @Accept json
// @Produce json
// @Param id path string true "hashId"
// @Success 200 {object} detailResponse
// @Failure 400 {object} code.Failure
// @Router /api/cron/:id [patch]
func (h *handler) Execute() core.HandlerFunc {
return func(ctx core.Context) {
req := new(executeRequest)
res := new(executeResponse)
if err := ctx.ShouldBindURI(req); err != nil {
ctx.AbortWithError(errno.NewError(
http.StatusBadRequest,
code.ParamBindError,
validation.Error(err)).WithErr(err),
)
return
}
ids, err := h.hashids.HashidsDecode(req.Id)
if err != nil {
ctx.AbortWithError(errno.NewError(
http.StatusBadRequest,
code.HashIdsDecodeError,
code.Text(code.HashIdsDecodeError)).WithErr(err),
)
return
}
err = h.cronService.Execute(ctx, cast.ToInt32(ids[0]))
if err != nil {
ctx.AbortWithError(errno.NewError(
http.StatusBadRequest,
code.CronExecuteError,
code.Text(code.CronExecuteError)).WithErr(err),
)
return
}
res.Id = ids[0]
ctx.Payload(res)
}
}
@@ -1,13 +1,13 @@
package cron_handler
import (
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"net/http"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/xinliangnote/go-gin-api/internal/api/service/cron_service"
"github.com/xinliangnote/go-gin-api/internal/pkg/code"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"github.com/xinliangnote/go-gin-api/pkg/errno"
"github.com/xinliangnote/go-gin-api/pkg/time_parse"
@@ -50,7 +50,7 @@ type listResponse struct {
Pagination struct {
Total int `json:"total"`
CurrentPage int `json:"current_page"`
PrePageCount int `json:"pre_page_count"`
PerPageCount int `json:"per_page_count"`
} `json:"pagination"`
}
@@ -119,7 +119,7 @@ func (h *handler) List() core.HandlerFunc {
}
res.Pagination.Total = cast.ToInt(resCountData)
res.Pagination.PrePageCount = pageSize
res.Pagination.PerPageCount = pageSize
res.Pagination.CurrentPage = page
res.List = make([]listData, len(resListData))
@@ -1,12 +1,13 @@
package cron_handler
import (
"net/http"
"github.com/xinliangnote/go-gin-api/internal/api/service/cron_service"
"github.com/xinliangnote/go-gin-api/internal/pkg/code"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"github.com/xinliangnote/go-gin-api/pkg/errno"
"net/http"
)
type modifyRequest struct {
@@ -1,11 +1,11 @@
package cron_handler
import (
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"net/http"
"github.com/xinliangnote/go-gin-api/internal/pkg/code"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/validation"
"github.com/xinliangnote/go-gin-api/pkg/errno"
)
@@ -3,6 +3,7 @@ package cron_handler
import (
"github.com/xinliangnote/go-gin-api/configs"
"github.com/xinliangnote/go-gin-api/internal/api/service/cron_service"
"github.com/xinliangnote/go-gin-api/internal/cron/cron_server"
"github.com/xinliangnote/go-gin-api/internal/pkg/cache"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/db"
@@ -40,6 +41,11 @@ type Handler interface {
// @Tags API.cron
// @Router /api/cron/:id [get]
Detail() core.HandlerFunc
// Execute 手动执行任务
// @Tags API.cron
// @Router /api/cron/:id [patch]
Execute() core.HandlerFunc
}
type handler struct {
@@ -49,12 +55,12 @@ type handler struct {
cronService cron_service.Service
}
func New(logger *zap.Logger, db db.Repo, cache cache.Repo) Handler {
func New(logger *zap.Logger, db db.Repo, cache cache.Repo, cron cron_server.Server) Handler {
return &handler{
logger: logger,
cache: cache,
hashids: hash.New(configs.Get().HashIds.Secret, configs.Get().HashIds.Length),
cronService: cron_service.New(db, cache),
cronService: cron_service.New(db, cache, cron),
}
}
+9 -5
View File
@@ -2,6 +2,7 @@ package cron_service
import (
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/xinliangnote/go-gin-api/internal/cron/cron_server"
"github.com/xinliangnote/go-gin-api/internal/pkg/cache"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/db"
@@ -17,18 +18,21 @@ type Service interface {
PageList(ctx core.Context, searchData *SearchData) (listData []*cron_task_repo.CronTask, err error)
PageListCount(ctx core.Context, searchData *SearchData) (total int64, err error)
UpdateUsed(ctx core.Context, id int32, used int32) (err error)
Execute(ctx core.Context, id int32) (err error)
Detail(ctx core.Context, searchOneData *SearchOneData) (info *cron_task_repo.CronTask, err error)
}
type service struct {
db db.Repo
cache cache.Repo
db db.Repo
cache cache.Repo
cronServer cron_server.Server
}
func New(db db.Repo, cache cache.Repo) Service {
func New(db db.Repo, cache cache.Repo, cron cron_server.Server) Service {
return &service{
db: db,
cache: cache,
db: db,
cache: cache,
cronServer: cron,
}
}
@@ -45,5 +45,7 @@ func (s *service) Create(ctx core.Context, createData *CreateCronTaskData) (id i
return 0, err
}
s.cronServer.AddTask(model)
return
}
@@ -0,0 +1,21 @@
package cron_service
import (
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
)
func (s *service) Execute(ctx core.Context, id int32) (err error) {
qb := cron_task_repo.NewQueryBuilder()
qb.WhereId(db_repo.EqualPredicate, id)
info, err := qb.QueryOne(s.db.GetDbR().WithContext(ctx.RequestContext()))
if err != nil {
return err
}
info.Spec = "手动执行"
go s.cronServer.AddJob(info)()
return nil
}
@@ -4,6 +4,8 @@ import (
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/spf13/cast"
)
type ModifyCronTaskData struct {
@@ -49,5 +51,21 @@ func (s *service) Modify(ctx core.Context, id int32, modifyData *ModifyCronTaskD
return err
}
// region 操作定时任务 避免主从同步延迟,在这需要查询主库
if modifyData.IsUsed == cron_task_repo.IsUsedNo {
s.cronServer.RemoveTask(cast.ToInt(id))
} else {
qb = cron_task_repo.NewQueryBuilder()
qb.WhereId(db_repo.EqualPredicate, id)
info, err := qb.QueryOne(s.db.GetDbW().WithContext(ctx.RequestContext()))
if err != nil {
return err
}
s.cronServer.RemoveTask(cast.ToInt(id))
s.cronServer.AddTask(info)
}
// endregion
return
}
@@ -4,6 +4,8 @@ import (
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/spf13/cast"
)
func (s *service) UpdateUsed(ctx core.Context, id int32, used int32) (err error) {
@@ -19,5 +21,22 @@ func (s *service) UpdateUsed(ctx core.Context, id int32, used int32) (err error)
return err
}
// region 操作定时任务 避免主从同步延迟,在这需要查询主库
if used == cron_task_repo.IsUsedNo {
s.cronServer.RemoveTask(cast.ToInt(id))
} else {
qb = cron_task_repo.NewQueryBuilder()
qb.WhereId(db_repo.EqualPredicate, id)
info, err := qb.QueryOne(s.db.GetDbW().WithContext(ctx.RequestContext()))
if err != nil {
return err
}
s.cronServer.RemoveTask(cast.ToInt(id))
s.cronServer.AddTask(info)
}
// endregion
return
}
+85
View File
@@ -0,0 +1,85 @@
package cron_server
import (
"sync"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/xinliangnote/go-gin-api/internal/pkg/cache"
"github.com/xinliangnote/go-gin-api/internal/pkg/db"
"github.com/xinliangnote/go-gin-api/pkg/errors"
"github.com/jakecoffman/cron"
"go.uber.org/zap"
)
var _ Server = (*server)(nil)
type taskCount struct {
wg sync.WaitGroup
exit chan struct{}
}
func (tc *taskCount) i() {}
func (tc *taskCount) Add() {
tc.wg.Add(1)
}
func (tc *taskCount) Done() {
tc.wg.Done()
}
func (tc *taskCount) Exit() {
tc.wg.Done()
<-tc.exit
}
func (tc *taskCount) Wait() {
tc.Add()
tc.wg.Wait()
close(tc.exit)
}
type server struct {
logger *zap.Logger
db db.Repo
cache cache.Repo
cron *cron.Cron
taskCount *taskCount
}
type Server interface {
i()
Start()
Stop()
AddTask(task *cron_task_repo.CronTask)
AddJob(task *cron_task_repo.CronTask) cron.FuncJob
RemoveTask(taskId int)
}
func New(logger *zap.Logger, db db.Repo, cache cache.Repo) (Server, error) {
if logger == nil {
return nil, errors.New("logger required")
}
if db == nil {
return nil, errors.New("db required")
}
if cache == nil {
return nil, errors.New("cache required")
}
return &server{
logger: logger,
db: db,
cache: cache,
cron: cron.New(),
taskCount: &taskCount{
wg: sync.WaitGroup{},
exit: make(chan struct{}),
},
}, nil
}
func (s *server) i() {}
@@ -0,0 +1,19 @@
package cron_server
import (
"fmt"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/jakecoffman/cron"
)
func (s *server) AddJob(task *cron_task_repo.CronTask) cron.FuncJob {
return func() {
s.taskCount.Add()
defer s.taskCount.Done()
msg := fmt.Sprintf("开始执行任务:(%d)%s [%s]", task.Id, task.Name, task.Spec)
s.logger.Info(msg)
}
}
@@ -0,0 +1,16 @@
package cron_server
import (
"strings"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"github.com/spf13/cast"
)
func (s *server) AddTask(task *cron_task_repo.CronTask) {
spec := "0 " + strings.TrimSpace(task.Spec)
name := cast.ToString(task.Id)
s.cron.AddFunc(spec, s.AddJob(task), name)
}
@@ -0,0 +1,8 @@
package cron_server
import "github.com/spf13/cast"
func (s *server) RemoveTask(taskId int) {
name := cast.ToString(taskId)
s.cron.RemoveJob(name)
}
@@ -0,0 +1,49 @@
package cron_server
import (
"fmt"
"math"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo"
"github.com/xinliangnote/go-gin-api/internal/api/repository/db_repo/cron_task_repo"
"go.uber.org/zap"
)
func (s *server) Start() {
s.cron.Start()
go s.taskCount.Wait()
qb := cron_task_repo.NewQueryBuilder()
qb.WhereIsUsed(db_repo.EqualPredicate, cron_task_repo.IsUsedYES)
totalNum, err := qb.Count(s.db.GetDbR())
if err != nil {
s.logger.Fatal("cron initialize tasks count err", zap.Error(err))
}
pageSize := 50
maxPage := int(math.Ceil(float64(totalNum) / float64(pageSize)))
taskNum := 0
s.logger.Info("开始初始化后台任务")
for page := 1; page <= maxPage; page++ {
qb = cron_task_repo.NewQueryBuilder()
qb.WhereIsUsed(db_repo.EqualPredicate, cron_task_repo.IsUsedYES)
listData, err := qb.
Limit(pageSize).
Offset((page - 1) * pageSize).
OrderById(false).
QueryAll(s.db.GetDbR())
if err != nil {
s.logger.Fatal("cron initialize tasks list err", zap.Error(err))
}
for _, item := range listData {
s.AddTask(item)
taskNum++
}
}
s.logger.Info(fmt.Sprintf("后台任务初始化完成,总数量:%d", taskNum))
}
@@ -0,0 +1,6 @@
package cron_server
func (s *server) Stop() {
s.cron.Stop()
s.taskCount.Exit()
}
+5 -4
View File
@@ -61,10 +61,11 @@ const (
MenuListActionError = 20307
MenuDeleteActionError = 20308
CronCreateError = 20401
CronUpdateError = 20402
CronListError = 20403
CronDetailError = 20404
CronCreateError = 20401
CronUpdateError = 20402
CronListError = 20403
CronDetailError = 20404
CronExecuteError = 20405
)
func Text(code int) string {
+5 -4
View File
@@ -53,8 +53,9 @@ var enUSText = map[int]string{
MenuListActionError: "Failed to get menu action list",
MenuDeleteActionError: "Failed to delete menu action",
CronCreateError: "Failed to create cron",
CronUpdateError: "Failed to update menu",
CronListError: "Failed to get cron list",
CronDetailError: "Failed to get cron detail",
CronCreateError: "Failed to create cron",
CronUpdateError: "Failed to update menu",
CronListError: "Failed to get cron list",
CronDetailError: "Failed to get cron detail",
CronExecuteError: "Failed to execute cron",
}
+5 -4
View File
@@ -53,8 +53,9 @@ var zhCNText = map[int]string{
MenuListActionError: "获取菜单栏功能权限列表失败",
MenuDeleteActionError: "删除菜单栏功能权限失败",
CronCreateError: "创建后台任务失败",
CronUpdateError: "更新后台任务失败",
CronListError: "获取定时任务列表失败",
CronDetailError: "获取定时任务详情失败",
CronCreateError: "创建后台任务失败",
CronUpdateError: "更新后台任务失败",
CronListError: "获取定时任务列表失败",
CronDetailError: "获取定时任务详情失败",
CronExecuteError: "手动执行定时任务失败",
}
+23 -11
View File
@@ -2,6 +2,7 @@ package router
import (
"github.com/xinliangnote/go-gin-api/configs"
"github.com/xinliangnote/go-gin-api/internal/cron/cron_server"
"github.com/xinliangnote/go-gin-api/internal/pkg/cache"
"github.com/xinliangnote/go-gin-api/internal/pkg/core"
"github.com/xinliangnote/go-gin-api/internal/pkg/db"
@@ -16,22 +17,24 @@ import (
)
type resource struct {
mux core.Mux
logger *zap.Logger
db db.Repo
cache cache.Repo
grpConn grpc.ClientConn
middles middleware.Middleware
mux core.Mux
logger *zap.Logger
db db.Repo
cache cache.Repo
grpConn grpc.ClientConn
middles middleware.Middleware
cronServer cron_server.Server
}
type Server struct {
Mux core.Mux
Db db.Repo
Cache cache.Repo
GrpClient grpc.ClientConn
Mux core.Mux
Db db.Repo
Cache cache.Repo
GrpClient grpc.ClientConn
CronServer cron_server.Server
}
func NewHTTPServer(logger *zap.Logger) (*Server, error) {
func NewHTTPServer(logger *zap.Logger, cronLogger *zap.Logger) (*Server, error) {
if logger == nil {
return nil, errors.New("logger required")
}
@@ -66,6 +69,14 @@ func NewHTTPServer(logger *zap.Logger) (*Server, error) {
logger.Fatal("new grpc err", zap.Error(err))
}
r.grpConn = gRPCRepo
// 初始化 CRON Server
cronServer, err := cron_server.New(cronLogger, dbRepo, cacheRepo)
if err != nil {
logger.Fatal("new cron err", zap.Error(err))
}
cronServer.Start()
r.cronServer = cronServer
}
mux, err := core.New(logger,
@@ -97,6 +108,7 @@ func NewHTTPServer(logger *zap.Logger) (*Server, error) {
s.Db = r.db
s.Cache = r.cache
s.GrpClient = r.grpConn
s.CronServer = r.cronServer
return s, nil
}
+2 -1
View File
@@ -81,12 +81,13 @@ func setApiRouter(r *resource) {
api.PATCH("/config/email", configHandler.Email())
// cron
cronHandler := cron_handler.New(r.logger, r.db, r.cache)
cronHandler := cron_handler.New(r.logger, r.db, r.cache, r.cronServer)
api.POST("/cron", cronHandler.Create())
api.GET("/cron", cronHandler.List())
api.GET("/cron/:id", cronHandler.Detail())
api.POST("/cron/:id", cronHandler.Modify())
api.PATCH("/cron/used", cronHandler.UpdateUsed())
api.PATCH("/cron/exec/:id", cronHandler.Execute())
}
}
@@ -79,7 +79,8 @@ func CreateMenuActionTableDataSql() (sql string) {
sql += "(43, 24, 'POST', '/api/cron/*', 'init'),"
sql += "(44, 24, 'GET', '/api/cron', 'init'),"
sql += "(45, 24, 'GET', '/api/cron/*', 'init'),"
sql += "(46, 24, 'PATCH', '/api/cron/used', 'init');"
sql += "(46, 24, 'PATCH', '/api/cron/used', 'init'),"
sql += "(47, 24, 'PATCH', '/api/cron/exec/*', 'init');"
return
}
@@ -44,7 +44,7 @@ func (h *handler) LogsView() core.HandlerFunc {
}
return func(c core.Context) {
readLineFromEnd, err := file.NewReadLineFromEnd(configs.ProjectLogFile)
readLineFromEnd, err := file.NewReadLineFromEnd(configs.ProjectAccessLogFile)
if err != nil {
h.logger.Error("NewReadLineFromEnd err", zap.Error(err))
}