417 lines
12 KiB
Go
417 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"clear/utils"
|
|
"clear/utils/color"
|
|
"flag"
|
|
"fmt"
|
|
"github.com/shirou/gopsutil/v3/disk"
|
|
"gopkg.in/yaml.v2"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
type Conf struct {
|
|
Dir string
|
|
Prefix string
|
|
Suffix string
|
|
BeforeTime int64 `yaml:"beforeTime"`
|
|
Mode string
|
|
Retain int
|
|
MinSize int `yaml:"minSize"`
|
|
Debug bool
|
|
}
|
|
type FileType struct {
|
|
Path string
|
|
Name string
|
|
Size int64
|
|
ModTime time.Time
|
|
}
|
|
|
|
// EmailContent 邮件内容
|
|
var EmailContent bytes.Buffer
|
|
|
|
// 清理大小总计
|
|
var ClearSize int64
|
|
|
|
// 预估清理大小
|
|
var EstimateSize int64
|
|
|
|
const TimeFormat = "2006-01-02 15:04:05"
|
|
|
|
// 重新计算文件大小的最小值
|
|
var MinRecountFileSize int64
|
|
|
|
// 安全的扩展名
|
|
var safeExt = []string{".log", ".json", ".gz", ".tgz", ".tar", ".zip", ".rar", ".mp4", ".out", ".bz2", ".txt"}
|
|
|
|
func init() {
|
|
MinRecountFileSize = 10 * 1024 * 1024
|
|
}
|
|
|
|
func main() {
|
|
var minSize float64
|
|
var configDir string
|
|
var configFile string
|
|
var email string
|
|
var help bool
|
|
flag.BoolVar(&help, "help", false, "显示帮助")
|
|
flag.Float64Var(&minSize, "min-size", 10, "阈值比例，例如：磁盘使用率大于90%后执行")
|
|
flag.StringVar(&configDir, "config-dir", "./rules/", "配置目录")
|
|
flag.StringVar(&configFile, "config-file", "", "指定配置文件")
|
|
flag.StringVar(&email, "email", "", "指定邮箱收件人")
|
|
flag.Parse()
|
|
|
|
if help {
|
|
flag.Usage()
|
|
os.Exit(0)
|
|
}
|
|
|
|
parts, _ := disk.Partitions(true)
|
|
var minDiskSize uint64
|
|
|
|
//小于1Gb的磁盘，不做处理
|
|
minDiskSize = 1024 * 1024 * 1024
|
|
isCheck := false
|
|
diskData := ""
|
|
for _, diskinfo := range parts {
|
|
data, _ := disk.Usage(diskinfo.Mountpoint)
|
|
if data.Total < minDiskSize {
|
|
continue
|
|
}
|
|
|
|
if data.UsedPercent > minSize-1 {
|
|
isCheck = true
|
|
UsedPercent := fmt.Sprintf("%.1f", data.UsedPercent)
|
|
DiskTotalSize := int64(data.Total)
|
|
LeftSize := int64(data.Free)
|
|
diskData = "挂载点: " + data.Path + ", 使用比例: " + UsedPercent + "%, 磁盘容量:" + utils.FormatFileSize(DiskTotalSize) + ", 剩余容量: " + utils.FormatFileSize(LeftSize) + ", 磁盘类型:" + data.Fstype
|
|
fmt.Println(data)
|
|
break
|
|
}
|
|
|
|
}
|
|
|
|
if isCheck == false {
|
|
fmt.Println("磁盘容量没有达到设定的阈值:", minSize, "% , 正常退出")
|
|
os.Exit(0)
|
|
}
|
|
|
|
nowTime := time.Now().Format(TimeFormat)
|
|
fmt.Println("[开始时间]", nowTime)
|
|
fmt.Println("+-----------------------------------------------------------------------+")
|
|
fmt.Println("[磁盘信息]", diskData)
|
|
fmt.Println("+-----------------------------------------------------------------------+")
|
|
EmailContent.WriteString("[开始时间] " + nowTime + "\n")
|
|
EmailContent.WriteString("+-----------------------------------------------------------------------+\n")
|
|
hostname, _ := os.Hostname()
|
|
ip := utils.GetIp()
|
|
EmailContent.WriteString("[主机名] " + hostname + "\n")
|
|
EmailContent.WriteString("[IP] " + ip + "\n")
|
|
EmailContent.WriteString("[磁盘信息] " + diskData + "\n")
|
|
EmailContent.WriteString("+-----------------------------------------------------------------------+\n")
|
|
|
|
files, err := ioutil.ReadDir(configDir)
|
|
if err != nil {
|
|
fmt.Println("配置目录不存在", err)
|
|
os.Exit(101)
|
|
}
|
|
if configFile == "" {
|
|
for _, cFile := range files {
|
|
ext := path.Ext(cFile.Name())
|
|
if ext != ".yml" {
|
|
continue
|
|
}
|
|
err = findConf(configDir + cFile.Name())
|
|
if err != nil {
|
|
continue
|
|
}
|
|
}
|
|
} else {
|
|
_ = findConf(configDir + configFile)
|
|
}
|
|
|
|
endTime := time.Now().Format(TimeFormat)
|
|
fmt.Println("+-----------------------------------------------------------------------+")
|
|
successClearSize := utils.FormatFileSize(ClearSize)
|
|
estimateSize := utils.FormatFileSize(EstimateSize)
|
|
fmt.Println("[成功清理]", successClearSize)
|
|
fmt.Println("[预估清理]", estimateSize)
|
|
|
|
EmailContent.WriteString("+-----------------------------------------------------------------------+\n")
|
|
EmailContent.WriteString("[成功清理] " + successClearSize + "\n")
|
|
EmailContent.WriteString("[预估清理] " + estimateSize + "\n")
|
|
EmailContent.WriteString("[完成时间]" + endTime + "\n")
|
|
if email != "" {
|
|
|
|
isCheck = false
|
|
for _, diskinfo := range parts {
|
|
data, _ := disk.Usage(diskinfo.Mountpoint)
|
|
if data.Total < minDiskSize {
|
|
continue
|
|
}
|
|
if data.UsedPercent > minSize {
|
|
isCheck = true
|
|
break
|
|
}
|
|
}
|
|
if isCheck == true {
|
|
EmailContent.WriteString("\n 清理任务已执行，但清理规则似乎未完全覆盖，需要人工处理处理 \n")
|
|
} else {
|
|
EmailContent.WriteString("\n 磁盘容量已下降，可忽略本邮件 \n")
|
|
}
|
|
|
|
content := "<pre>" + EmailContent.String() + "</pre>"
|
|
if isCheck == true {
|
|
utils.SendEmailApi(email, "【磁盘容量不足警告】🔥", content, "日志清理")
|
|
} else {
|
|
utils.SendEmailApi(email, "磁盘清理完成通知", content, "日志清理")
|
|
}
|
|
|
|
}
|
|
fmt.Println("[完成时间]", endTime)
|
|
}
|
|
func findConf(filename string) error {
|
|
|
|
var confList []Conf
|
|
yamlFile, err := ioutil.ReadFile(filename)
|
|
//log.Println("yamlFile:", string(yamlFile))
|
|
if err != nil {
|
|
log.Printf("yamlFile.Get err #%v ", err)
|
|
return err
|
|
}
|
|
|
|
err = yaml.Unmarshal(yamlFile, &confList)
|
|
if err != nil {
|
|
log.Fatalf("配置文件格式错误:%s , %v", filename, err)
|
|
return err
|
|
}
|
|
//打印配置
|
|
//fmt.Println(confList)
|
|
if len(confList) > 0 {
|
|
fmt.Println("[配置文件]", filename)
|
|
fmt.Println("+-----------------------------------------------------------------------+")
|
|
|
|
EmailContent.WriteString("[配置文件] ➜ " + filename + "\n")
|
|
EmailContent.WriteString("+-----------------------------------------------------------------------+\n")
|
|
|
|
for _, conf := range confList {
|
|
tmpdir := strings.TrimRight(conf.Dir, "/")
|
|
stringCount := strings.Count(tmpdir, "/")
|
|
//获取前缀是否配置并满足规则，必须包含 .
|
|
stringCountPrefix := 1
|
|
if conf.Prefix != "" {
|
|
stringCountPrefix = strings.Count(conf.Prefix, ".")
|
|
}
|
|
|
|
if stringCount > 1 && stringCountPrefix > 0 {
|
|
ListDir(conf, conf.Dir)
|
|
} else if stringCountPrefix == 0 {
|
|
fmt.Println(color.Magenta("[异常]"), "文件前缀不符合规范,必须包含 \".\" ➜ 当前配置为:", conf.Prefix)
|
|
EmailContent.WriteString("[异常] 文件前缀不符合规范,必须包含 \".\" ➜ 当前配置为:" + conf.Prefix + "\n")
|
|
} else {
|
|
fmt.Println(color.Magenta("[异常]"), "目录规范不符合规范 ➜ 禁止根目录和一级目录 ➜", conf.Dir)
|
|
EmailContent.WriteString("[异常] 目录规范不符合规范 ➜ 禁止根目录和一级目录 ➜ " + conf.Dir + "\n")
|
|
}
|
|
|
|
}
|
|
} else {
|
|
fmt.Println("+-----------------------------------------------------------------------+")
|
|
fmt.Println("[配置为空]", filename)
|
|
}
|
|
return nil
|
|
}
|
|
func ListDir(config Conf, folder string) error {
|
|
files, errDir := ioutil.ReadDir(folder)
|
|
|
|
if errDir != nil {
|
|
fmt.Println("[提示]", errDir)
|
|
return nil
|
|
}
|
|
files = sortByTime(files)
|
|
var filelist []FileType
|
|
for _, file := range files {
|
|
if file.IsDir() {
|
|
ListDir(config, folder+"/"+file.Name())
|
|
} else {
|
|
// 输出绝对路径
|
|
strAbsPath, errPath := filepath.Abs(folder + "/" + file.Name())
|
|
if errPath != nil {
|
|
fmt.Println(errPath)
|
|
}
|
|
//error.log.20200705
|
|
//%2Fworkspace%2Flogs%2Fservice%2Ftest%2Faccesslog.log.20200721
|
|
tmpfile := strings.ReplaceAll(file.Name(), "%2F", "")
|
|
match, _ := regexp.MatchString("^[A-Za-z0-9-_]+\\.log[.-][0-9-]+$", tmpfile)
|
|
if match == false {
|
|
//86_70_20200715160311.cache.172.16.49.19
|
|
match, _ = regexp.MatchString("^[A-Za-z0-9_-]+\\.cache\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$", file.Name())
|
|
}
|
|
//后缀优先
|
|
if config.Suffix != "" {
|
|
//获取后缀
|
|
ext := path.Ext(strAbsPath)
|
|
if ext == ".properties" {
|
|
continue
|
|
}
|
|
if ext == "" {
|
|
continue
|
|
}
|
|
|
|
if match == false {
|
|
//验证后缀有没有在安全范围
|
|
if utils.InArray(ext, safeExt) == false {
|
|
continue
|
|
}
|
|
if ext != config.Suffix {
|
|
continue
|
|
}
|
|
}
|
|
} else if config.Prefix != "" {
|
|
ext := path.Ext(strAbsPath)
|
|
//后缀长度必须大于8
|
|
if len(ext) < 8 {
|
|
continue
|
|
}
|
|
//判断前缀是否匹配,0表示，出现在最前面
|
|
if strings.Index(file.Name(), config.Prefix) != 0 {
|
|
continue
|
|
}
|
|
} else if match == false {
|
|
continue
|
|
}
|
|
//大于10M的文件，重新计算真实的大小
|
|
//fmt.Println(file.Size(),file.Name())
|
|
if file.Size() > MinRecountFileSize {
|
|
nf := file.Sys().(*syscall.Stat_t)
|
|
nfsize := nf.Blocks * 512
|
|
//如果大小 4096 * 2 小于 2 个扇区大小，忽略文件
|
|
if nfsize < 8192 {
|
|
continue
|
|
}
|
|
}
|
|
var fileInfo FileType
|
|
fileInfo.Path = strAbsPath
|
|
fileInfo.Size = file.Size()
|
|
fileInfo.Name = file.Name()
|
|
fileInfo.ModTime = file.ModTime()
|
|
//fmt.Println(strAbsPath,file.Size())
|
|
filelist = append(filelist, fileInfo)
|
|
}
|
|
}
|
|
//fmt.Println(filelist)
|
|
//如果配置了保留数量 > 0
|
|
minFileSize := config.MinSize
|
|
if minFileSize > 0 {
|
|
minFileSize = minFileSize * 1024 * 1024
|
|
}
|
|
beforeTime := config.BeforeTime
|
|
if beforeTime > 0 {
|
|
beforeTime = time.Now().Unix() - beforeTime*60
|
|
}
|
|
//fmt.Println(config.MinSize,config)
|
|
if config.Retain > 0 {
|
|
if len(filelist) > config.Retain {
|
|
for i, file := range filelist {
|
|
//fmt.Println(file.ModTime.Format(TimeFormat),file.Path)
|
|
if i >= config.Retain {
|
|
RunRule(config, file, int64(minFileSize), beforeTime)
|
|
}
|
|
}
|
|
} else {
|
|
return nil
|
|
}
|
|
} else {
|
|
for _, file := range filelist {
|
|
RunRule(config, file, int64(minFileSize), beforeTime)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
func RunRule(config Conf, file FileType, minFileSize int64, beforeTime int64) error {
|
|
filesize := utils.FormatFileSize(file.Size)
|
|
colorSize := color.Cyan(filesize)
|
|
|
|
if minFileSize > 0 && file.Size < minFileSize {
|
|
fmt.Println(color.Green("[忽略]"), file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize)
|
|
return nil
|
|
}
|
|
if beforeTime > 0 && file.ModTime.Unix() > beforeTime {
|
|
fmt.Println(color.Green("[忽略]"), file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize)
|
|
return nil
|
|
}
|
|
var result string
|
|
if config.Mode == "rm" {
|
|
if config.Debug == false {
|
|
//执行删除
|
|
err := os.Remove(file.Path)
|
|
if err != nil {
|
|
result = "[✖]"
|
|
} else {
|
|
result = "[✔]"
|
|
ClearSize += file.Size
|
|
}
|
|
fmt.Println("[删除]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, result)
|
|
EmailContent.WriteString("[删除]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " " + result + "\n")
|
|
|
|
} else {
|
|
fmt.Println("[删除]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, "[Debug]")
|
|
EmailContent.WriteString("[删除]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " [Debug]\n")
|
|
}
|
|
EstimateSize += file.Size
|
|
} else if config.Mode == "waring" {
|
|
fmt.Println(color.Red("[告警]"), color.Red(file.ModTime.Format(TimeFormat)), "➜", color.Red(file.Path), colorSize)
|
|
EmailContent.WriteString("[告警]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + "\n")
|
|
return nil
|
|
} else {
|
|
//文件大小为0的，不做清空处理
|
|
if file.Size == 0 {
|
|
return nil
|
|
}
|
|
if config.Debug == false {
|
|
err := os.Truncate(file.Path, 0)
|
|
//f, err := os.OpenFile(file.Path, os.O_WRONLY|os.O_TRUNC, 0600)
|
|
if err != nil {
|
|
result = "[✖]"
|
|
} else {
|
|
result = "[✔]"
|
|
ClearSize += file.Size
|
|
}
|
|
//f.WriteString("")
|
|
//f.Close()
|
|
fmt.Println("[清空]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, result)
|
|
EmailContent.WriteString("[清空]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " " + result + "\n")
|
|
} else {
|
|
fmt.Println("[清空]", file.ModTime.Format(TimeFormat), "➜", file.Path, colorSize, "[Debug]")
|
|
EmailContent.WriteString("[清空]" + file.ModTime.Format(TimeFormat) + " ➜ " + file.Path + " " + filesize + " [Debug]\n")
|
|
}
|
|
EstimateSize += file.Size
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func sortByTime(pl []os.FileInfo) []os.FileInfo {
|
|
sort.Slice(pl, func(i, j int) bool {
|
|
flag := false
|
|
if pl[i].ModTime().After(pl[j].ModTime()) {
|
|
flag = true
|
|
} else if pl[i].ModTime().Equal(pl[j].ModTime()) {
|
|
if pl[i].Name() < pl[j].Name() {
|
|
flag = true
|
|
}
|
|
}
|
|
return flag
|
|
})
|
|
return pl
|
|
}
|