Commit 8637c8fe by usual2970

功能初步完善

parents
/logana
/logana.ini
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
package collect
import (
"bufio"
"io"
"os"
"regexp"
"strconv"
//"strings"
//"fmt"
"errors"
"sync"
"time"
)
const TIME_FORMAT = "2006-01-02 15:04:05"
/*
125.119.84.63 - - [16/Oct/2017:14:43:18 +0800] "GET /css/laydate/laydate.js HTTP/2.0" 304 87 "https://hp.yidianling.com/crm-client/depart" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"
*/
type Item struct {
Ip string
Time string
Method string
Url string
Status int
BodySize int
Proto string
Referer string
UserAgent string
}
type ItemCallback func(*Item)
type Collect struct {
filePath string
preg string
needUnzip bool
itemCallback ItemCallback
wg *sync.WaitGroup
}
func NewCollect(filePath string, needUnzip bool) *Collect {
c := &Collect{}
c.init(filePath, needUnzip)
return c
}
func (c *Collect) init(filePath string, needUnzip bool) {
c.filePath = filePath
c.needUnzip = needUnzip
c.wg = &sync.WaitGroup{}
}
func (c *Collect) GetItems() error {
filePath := c.filePath
if c.needUnzip {
unZip(c.filePath)
filePath = dstFile(c.filePath)
}
f, err := os.Open(filePath)
if err != nil {
return err
}
defer f.Close()
rd := bufio.NewReader(f)
for {
line, err := rd.ReadString('\n') //以'\n'为结束符读入一行
if err != nil && io.EOF != err {
return err
}
if err != nil && io.EOF == err {
break
}
item, err := c.GetItem(line)
if c.itemCallback != nil && err == nil {
c.handleItem(item)
}
}
return nil
}
func (c *Collect) GetItem(line string) (*Item, error) {
itemR := regexp.MustCompile("(.*?)\\s.*?\\s.*?\\s\\[(.*?)\\]\\s\"(.*?)\\s(.*?)\\s(.*?)\"\\s(\\d+?)\\s(\\d+?)\\s\"(.*?)\"\\s\"(.*?)\"")
matches := itemR.FindStringSubmatch(line)
if len(matches) == 0 {
return nil, errors.New("match failed")
}
status, _ := strconv.Atoi(matches[6])
bodySize, _ := strconv.Atoi(matches[7])
t1, _ := time.Parse(
TIME_FORMAT,
NginxTimeFormat(matches[2]))
return &Item{
Ip: matches[1],
Time: t1.String(),
Method: matches[3],
Url: matches[4],
Status: status,
BodySize: bodySize,
Proto: matches[5],
Referer: matches[8],
UserAgent: matches[9],
}, nil
}
func (c *Collect) OnItem(callback ItemCallback) {
c.itemCallback = callback
}
func (c *Collect) handleItem(item *Item) {
c.itemCallback(item)
}
package collect
import (
"fmt"
"testing"
)
// func TestGetOne(t *testing.T) {
// c := NewCollect("./access.log", "asdf")
// err:=c.GetItems()
// fmt.Println(err)
// }
func TestOnItem(t *testing.T) {
c := NewCollect("./access.log.2.gz", true)
c.OnItem(func(item *Item) {
fmt.Println(item)
})
err := c.GetItems()
fmt.Println(err)
}
package collect
import (
"compress/gzip"
"fmt"
"io"
"os"
"strings"
)
const openFileOptions int = os.O_CREATE | os.O_RDWR
const openFilePermissions os.FileMode = 0660
var numMonth = map[string]string{
"Jan": "01",
"Feb": "02",
"Mar": "03",
"Apr": "04",
"May": "05",
"June": "06",
"July": "07",
"Aug": "08",
"Sept": "09",
"Oct": "10",
"Nov": "11",
"Dec": "12",
}
func NginxTimeFormat(t string) string {
mainTime := strings.Split(t, " ")[0]
timeSplit := strings.Split(mainTime, ":")
daySplit := strings.Split(timeSplit[0], "/")
return daySplit[2] + "-" + numMonth[daySplit[1]] + "-" + daySplit[0] + " " + timeSplit[1] + ":" + timeSplit[2] + ":" + timeSplit[3]
}
func unZip(zipFile string) {
handle, err := openFile(zipFile)
if err != nil {
fmt.Println("[ERROR] Opening file:", err)
}
defer closeFile(handle)
zipReader, err := gzip.NewReader(handle)
if err != nil {
fmt.Println("[ERROR] New gzip reader:", err)
}
defer zipReader.Close()
dest, _ := openFile(dstFile(zipFile))
_, err = io.Copy(dest, zipReader)
if err != nil {
fmt.Println("[ERROR] New gzip reader:", err)
}
}
func dstFile(zipFile string) string {
fileSplit := strings.Split(zipFile, ".")
fileSplit = fileSplit[:len(fileSplit)-1]
dest := strings.Join(fileSplit, ".")
return dest
}
func openFile(fileToOpen string) (*os.File, error) {
return os.OpenFile(fileToOpen, openFileOptions, openFilePermissions)
}
func closeFile(handle *os.File) {
if handle == nil {
return
}
err := handle.Close()
if err != nil {
fmt.Println("[ERROR] Closing file:", err)
}
}
package collect
import (
"testing"
)
func TestUnZip(t *testing.T) {
//unZip("./access.log.2.gz")
}
package main
import (
"database/sql"
"fmt"
_ "github.com/go-sql-driver/mysql"
"github.com/usual2970/logana/collect"
"log"
"regexp"
"gopkg.in/ini.v1"
)
var db *sql.DB
var cfg *ini.File
const CFG_DATA=`
[collect]
file_path=/var/log/nginx/access.log.2.gz
need_unzip=true
[db]
dblink=ydl:tt12345@tcp(ydldbhost:3306)/ydl?charset=utf8
`
func main() {
collectSec,_:=cfg.GetSection("collect")
needUnzip,_:=collectSec.Key("need_unzip").Bool()
c := collect.NewCollect(collectSec.Key("file_path").String(), needUnzip)
c.OnItem(func(item *collect.Item) {
if re := regexp.MustCompile("Baiduspider"); re.MatchString(item.UserAgent) {
//插入数据
stmt, err := db.Prepare("INSERT site_access_record SET ip=?,access_time=?,method=?,url=?,status_code=?,body_size=?,proto=?,referer=?,user_agent=?")
checkErr(err)
res, err := stmt.Exec(
item.Ip,
item.Time,
item.Method,
item.Url,
item.Status,
item.BodySize,
item.Proto,
item.Referer,
item.UserAgent)
fmt.Println(res)
}
})
err := c.GetItems()
fmt.Println(err)
}
func checkErr(err error) {
if err != nil {
log.Fatal(err)
}
}
func init() {
var err error
cfg,_=ini.Load([]byte(CFG_DATA), "./logana.ini")
dbSec,_:=cfg.GetSection("db")
db, err = sql.Open("mysql", dbSec.Key("dblink").String())
if err != nil {
log.Fatal(err)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment