diff --git a/mgr/dataflow.go b/mgr/dataflow.go index 197659a00..6bf8db100 100644 --- a/mgr/dataflow.go +++ b/mgr/dataflow.go @@ -387,13 +387,13 @@ func checkSampleData(sampleData []string, logParser parser.Parser) ([]string, er } func getTransformerCreator(transformerConfig map[string]interface{}) (transforms.Creator, error) { - transformKeyType, ok := transformerConfig[transforms.KeyType] + transformKeyType, ok := transformerConfig[KeyType] if !ok { - return nil, fmt.Errorf("missing param %s", transforms.KeyType) + return nil, fmt.Errorf("missing param %s", KeyType) } transformKeyTypeStr, ok := transformKeyType.(string) if !ok { - return nil, fmt.Errorf("param %s must be of type string", transforms.KeyType) + return nil, fmt.Errorf("param %s must be of type string", KeyType) } create, ok := transforms.Transformers[transformKeyTypeStr] diff --git a/mgr/runner.go b/mgr/runner.go index cc546aa71..f13b4e123 100644 --- a/mgr/runner.go +++ b/mgr/runner.go @@ -22,6 +22,7 @@ import ( "github.com/qiniu/logkit/parser" _ "github.com/qiniu/logkit/parser/builtin" "github.com/qiniu/logkit/parser/config" + parserconfig "github.com/qiniu/logkit/parser/config" "github.com/qiniu/logkit/parser/qiniu" "github.com/qiniu/logkit/reader" _ "github.com/qiniu/logkit/reader/builtin" @@ -267,16 +268,22 @@ func NewLogExportRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal, r return nil, err } } - parser, err := pr.NewLogParser(rc.ParserConf) + ps, err := pr.NewLogParser(rc.ParserConf) if err != nil { return nil, err } + var serverConfigs = make([]map[string]interface{}, 0, 10) + if serverParser, ok := ps.(parser.ServerParser); ok { + if serverParser.ServerConfig() != nil { + serverConfigs = append(serverConfigs, serverParser.ServerConfig()) + } + } + transformers, err := createTransformers(rc) if err != nil { return nil, err } - var serverConfigs = make([]map[string]interface{}, 0, len(transformers)) for _, transform := range transformers { if serverTransformer, ok := transform.(transforms.ServerTansformer); ok { if serverTransformer.ServerConfig() != nil { @@ -317,7 +324,7 @@ func NewLogExportRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal, r if err != nil { return nil, fmt.Errorf("runner %v add sender router error, %v", rc.RunnerName, err) } - runner, err = NewLogExportRunnerWithService(runnerInfo, rd, cl, parser, transformers, senders, router, meta) + runner, err = NewLogExportRunnerWithService(runnerInfo, rd, cl, ps, transformers, senders, router, meta) if err != nil { return runner, err } @@ -335,7 +342,7 @@ func createTransformers(rc RunnerConfig) ([]transforms.Transformer, error) { transformers := make([]transforms.Transformer, 0) for idx := range rc.Transforms { tConf := rc.Transforms[idx] - tp := tConf[transforms.KeyType] + tp := tConf[KeyType] if tp == nil { return nil, fmt.Errorf("transformer config type is empty %v", tConf) } @@ -1484,12 +1491,12 @@ func setPandoraServerConfig(senderConfig conf.MapConf, serverConfigs []map[strin var err error for _, serverConfig := range serverConfigs { - keyType, ok := serverConfig[transforms.KeyType].(string) + keyType, ok := serverConfig[KeyType].(string) if !ok { continue } switch keyType { - case ip.Name: + case ip.Name, parserconfig.TypeLinuxAudit: if senderConfig, err = setIPConfig(senderConfig, serverConfig); err != nil { return senderConfig, err } @@ -1506,13 +1513,13 @@ func setIPConfig(senderConfig conf.MapConf, serverConfig map[string]interface{}) } autoCreate := senderConfig[senderConf.KeyPandoraAutoCreate] - transformAt, transformAtOk := serverConfig[transforms.TransformAt].(string) - if !transformAtOk { + processAt, processAtOk := serverConfig[ProcessAt].(string) + if !processAtOk { return senderConfig, nil } senderConfig[senderConf.KeyPandoraAutoCreate] = removeServerIPSchema(senderConfig[senderConf.KeyPandoraAutoCreate], key) - if transformAt == ip.Local { + if processAt == Local { return senderConfig, nil } diff --git a/mgr/runner_test.go b/mgr/runner_test.go index 9f863340e..d7bcf1e5a 100644 --- a/mgr/runner_test.go +++ b/mgr/runner_test.go @@ -2036,8 +2036,8 @@ func Test_setSenderConfig(t *testing.T) { serverConfigs := []map[string]interface{}{ { - transforms.KeyType: ip.Name, - transforms.TransformAt: ip.Server, + KeyType: ip.Name, + ProcessAt: Server, }, } actualConfig, err := setPandoraServerConfig(senderConfig, serverConfigs) @@ -2046,9 +2046,9 @@ func Test_setSenderConfig(t *testing.T) { serverConfigs = []map[string]interface{}{ { - transforms.KeyType: ip.Name, - transforms.TransformAt: ip.Server, - "key": "ip", + KeyType: ip.Name, + ProcessAt: Server, + "key": "ip", }, } actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs) @@ -2060,9 +2060,9 @@ func Test_setSenderConfig(t *testing.T) { } serverConfigs = []map[string]interface{}{ { - transforms.KeyType: ip.Name, - transforms.TransformAt: ip.Local, - "key": "a.b", + KeyType: ip.Name, + ProcessAt: Local, + "key": "a.b", }, } actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs) @@ -2071,7 +2071,7 @@ func Test_setSenderConfig(t *testing.T) { serverConfigs = []map[string]interface{}{ { - transforms.KeyType: "other", + KeyType: "other", }, } actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs) @@ -2080,9 +2080,9 @@ func Test_setSenderConfig(t *testing.T) { serverConfigs = []map[string]interface{}{ { - transforms.KeyType: ip.Name, - transforms.TransformAt: ip.Server, - "key": "ip.ip", + KeyType: ip.Name, + ProcessAt: Server, + "key": "ip.ip", }, } actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs) diff --git a/parser/builtin/builtin.go b/parser/builtin/builtin.go index 3c898f182..0c7949ba7 100644 --- a/parser/builtin/builtin.go +++ b/parser/builtin/builtin.go @@ -7,6 +7,7 @@ import ( _ "github.com/qiniu/logkit/parser/grok" _ "github.com/qiniu/logkit/parser/json" _ "github.com/qiniu/logkit/parser/kafkarest" + _ "github.com/qiniu/logkit/parser/linuxaudit" _ "github.com/qiniu/logkit/parser/logfmt" _ "github.com/qiniu/logkit/parser/mysql" _ "github.com/qiniu/logkit/parser/nginx" diff --git a/parser/config/config.go b/parser/config/config.go index e322d840d..53311a251 100644 --- a/parser/config/config.go +++ b/parser/config/config.go @@ -60,17 +60,18 @@ const ( // ModeUsages 和 ModeTooltips 用途说明 var ( ModeUsages = KeyValueSlice{ - {TypeRaw, "按原始日志逐行发送", ""}, - {TypeJSON, "按 json 格式解析", ""}, - {TypeNginx, "按 nginx 日志解析", ""}, - {TypeGrok, "按 grok 格式解析", ""}, - {TypeCSV, "按 csv 格式解析", ""}, - {TypeSyslog, "按 syslog 格式解析", ""}, - {TypeLogv1, "按七牛日志库格式解析", ""}, - {TypeKafkaRest, "按 kafkarest 日志解析", ""}, + {TypeRaw, "原始日志逐行发送", ""}, + {TypeJSON, "json 格式解析", ""}, + {TypeNginx, "nginx 日志解析", ""}, + {TypeGrok, "grok 格式解析", ""}, + {TypeCSV, "csv 格式解析", ""}, + {TypeSyslog, "syslog 格式解析", ""}, + {TypeLogv1, "七牛日志库格式解析", ""}, + {TypeKafkaRest, "kafkarest 日志解析", ""}, {TypeEmpty, "通过解析清空数据", ""}, - {TypeMySQL, "按 mysql 慢请求日志解析", ""}, + {TypeMySQL, "mysql 慢请求日志解析", ""}, {TypeKeyValue, "key value 日志解析", ""}, + {TypeLinuxAudit, "redhat 审计日志解析", ""}, } ModeToolTips = KeyValueSlice{ @@ -85,6 +86,7 @@ var ( {TypeEmpty, "通过解析清空数据", ""}, {TypeMySQL, "解析mysql的慢请求日志。", ""}, {TypeKeyValue, "按照key value解析日志", ""}, + {TypeLinuxAudit, "按 redhat 审计日志解析", ""}, } ) @@ -369,6 +371,30 @@ var ModeKeyOptions = map[string][]Option{ OptionKeepRawData, }, TypeLogfmt: { + { + KeyName: KeySplitter, + ChooseOnly: false, + Default: "=", + DefaultNoUse: false, + Description: "分隔符(splitter)", + }, + OptionParserName, + OptionDisableRecordErrData, + OptionKeepRawData, + }, + TypeKeyValue: { + { + KeyName: KeySplitter, + ChooseOnly: false, + Default: "=", + DefaultNoUse: false, + Description: "分隔符(splitter)", + }, + OptionParserName, + OptionDisableRecordErrData, + OptionKeepRawData, + }, + TypeLinuxAudit: { OptionParserName, OptionDisableRecordErrData, OptionKeepRawData, @@ -396,4 +422,7 @@ SELECT count(*) from mysql.rds_replication_status WHERE master_host IS NOT NULL #`, TypeLogfmt: `ts=2018-01-02T03:04:05.123Z lvl=5 msg="error" log_id=123456abc method=PUT duration=1.23 log_id=123456abc`, + TypeKeyValue: `ts=2018-01-02T03:04:05.123Z lvl=5 msg="error" log_id=123456abc +method=PUT duration=1.23 log_id=123456abc`, + TypeLinuxAudit: `type=SYSCALL msg=audit(1364481363.243:24287): arch=c000003e syscall=2 success=no exit=-13 a0=7fffd19c5592 a1=0 a2=7fffd19c4b50`, } diff --git a/parser/config/models.go b/parser/config/models.go index 30f6ec010..27de94856 100644 --- a/parser/config/models.go +++ b/parser/config/models.go @@ -30,6 +30,7 @@ const ( TypeMySQL = "mysqllog" TypeLogfmt = "logfmt" TypeKeyValue = "KV" + TypeLinuxAudit = "linuxaudit" ) // 数据常量类型 diff --git a/parser/linuxaudit/audit.go b/parser/linuxaudit/audit.go new file mode 100644 index 000000000..078582fb9 --- /dev/null +++ b/parser/linuxaudit/audit.go @@ -0,0 +1,282 @@ +package linuxaudit + +import ( + "strconv" + "strings" + "sync" + "unicode" + + "github.com/qiniu/log" + "github.com/qiniu/logkit/conf" + "github.com/qiniu/logkit/parser" + . "github.com/qiniu/logkit/parser/config" + . "github.com/qiniu/logkit/utils/models" +) + +const DefaultCheckKey = 5 + +type Parser struct { + name string + disableRecordErrData bool + numRoutine int + keepRawData bool +} + +func init() { + parser.RegisterConstructor(TypeLinuxAudit, NewParser) +} + +func NewParser(c conf.MapConf) (parser.Parser, error) { + name, _ := c.GetStringOr(KeyParserName, "") + disableRecordErrData, _ := c.GetBoolOr(KeyDisableRecordErrData, false) + keepRawData, _ := c.GetBoolOr(KeyKeepRawData, false) + numRoutine := MaxProcs + if numRoutine == 0 { + numRoutine = 1 + } + return &Parser{ + name: name, + disableRecordErrData: disableRecordErrData, + numRoutine: numRoutine, + keepRawData: keepRawData, + }, nil +} + +func (p *Parser) Parse(lines []string) ([]Data, error) { + var ( + lineLen = len(lines) + datas = make([]Data, lineLen) + se = &StatsError{} + numRoutine = p.numRoutine + + sendChan = make(chan parser.ParseInfo) + resultChan = make(chan parser.ParseResult) + wg = new(sync.WaitGroup) + ) + if lineLen < numRoutine { + numRoutine = lineLen + } + + for i := 0; i < numRoutine; i++ { + wg.Add(1) + go parser.ParseLine(sendChan, resultChan, wg, true, p.parse) + } + + go func() { + wg.Wait() + close(resultChan) + }() + + go func() { + for idx, line := range lines { + sendChan <- parser.ParseInfo{ + Line: line, + Index: idx, + } + } + close(sendChan) + }() + + var parseResultSlice = make(parser.ParseResultSlice, lineLen) + for resultInfo := range resultChan { + parseResultSlice[resultInfo.Index] = resultInfo + } + + se.DatasourceSkipIndex = make([]int, lineLen) + datasourceIndex := 0 + dataIndex := 0 + for _, parseResult := range parseResultSlice { + if len(parseResult.Line) == 0 { + se.DatasourceSkipIndex[datasourceIndex] = parseResult.Index + datasourceIndex++ + continue + } + + if parseResult.Err != nil { + se.AddErrors() + se.LastError = parseResult.Err.Error() + errData := make(Data) + if !p.disableRecordErrData { + errData[KeyPandoraStash] = parseResult.Line + } else if !p.keepRawData { + se.DatasourceSkipIndex[datasourceIndex] = parseResult.Index + datasourceIndex++ + } + if p.keepRawData { + errData[KeyRawData] = parseResult.Line + } + if !p.disableRecordErrData || p.keepRawData { + datas[dataIndex] = errData + dataIndex++ + } + continue + } + if len(parseResult.Data) < 1 { //数据为空时不发送 + se.LastError = "parsed no data by line " + parseResult.Line + se.AddErrors() + continue + } + se.AddSuccess() + if p.keepRawData { + parseResult.Data[KeyRawData] = parseResult.Line + } + datas[dataIndex] = parseResult.Data + dataIndex++ + } + + se.DatasourceSkipIndex = se.DatasourceSkipIndex[:datasourceIndex] + datas = datas[:dataIndex] + if se.Errors == 0 { + return datas, nil + } + return datas, se +} + +func (p *Parser) Name() string { + return p.name +} + +func (p *Parser) Type() string { + return TypeLinuxAudit +} + +func (p *Parser) parse(line string) (Data, error) { + var ( + data = make(Data) + tmp string + key string + prefix bool + ) + + for idx, c := range line { + if c == '\'' { + prefix = !prefix + p.processSubLine(tmp, line[idx+1:], key, data) + tmp, key = "", "" + } + + if prefix || c == '"' { + continue + } + + if c == '=' { + key = tmp + tmp = "" + continue + } + + if unicode.IsSpace(c) { + processSpace(key, tmp, data) + tmp, key = "", "" + continue + } + + tmp += string(c) + } + + if key != "" { + if strings.HasSuffix(tmp, ":") { + tmp = strings.TrimSuffix(tmp, ":") + } + setData(key, tmp, data) + } + return data, nil +} + +func processSpace(key, tmp string, data Data) { + if key == "" { + return + } + + if strings.HasSuffix(tmp, ":") { + tmp = strings.TrimSuffix(tmp, ":") + } + if key == "msg" && strings.HasPrefix(tmp, "audit(") { + if getTimestampID(tmp, data) { + return + } + } + tmp = strings.TrimSuffix(tmp, ":") + setData(key, tmp, data) + + return +} + +func getTimestampID(tmp string, data Data) bool { + tmp = strings.TrimPrefix(tmp, "audit(") + tmp = strings.TrimSuffix(tmp, ")") + arr := strings.SplitN(tmp, ":", 2) + if len(arr) < 2 { + return false + } + + timestamp := strings.TrimSpace(arr[0]) + if idx := strings.Index(arr[0], "."); idx != -1 { + timestamp = arr[0][:idx] + arr[0][idx+1:] + } + tm, err := GetTime(timestamp) + if err != nil { + log.Errorf("parse msg timestamp: %s failed: %v", timestamp, err) + data["msg_timestamp"] = strings.TrimSpace(arr[0]) + } else { + data["msg_timestamp"] = FormatWithUserOption("", 0, tm) + } + + data["msg_id"] = strings.TrimSpace(arr[1]) + return true +} + +func (p *Parser) processSubLine(tmp, tmpLine, key string, data Data) { + if key == "" { + return + } + + if tmp != "" { + tmp = strings.TrimSuffix(tmp, ":") + setData(key, tmp, data) + return + } + + suffix := strings.Index(tmpLine, "'") + if suffix != -1 { + value, err := p.parse(tmpLine[:suffix+1]) + if err != nil { + log.Errorf("parse line: %s failed: %v", tmpLine[:suffix+1], err) + } else { + setData(key, value, data) + } + } + return +} + +func setData(key string, value interface{}, data Data) { + if key == "" { + return + } + _, ok := data[key] + if !ok { + data[key] = value + return + } + + i := 1 + finalKey := key + "_" + strconv.Itoa(i) + for ; i <= DefaultCheckKey; i++ { + _, ok = data[finalKey] + if !ok { + data[finalKey] = value + return + } + finalKey = key + "_" + strconv.Itoa(i) + } + data[key] = value +} + +func (t *Parser) ServerConfig() map[string]interface{} { + config := make(map[string]interface{}) + config[KeyType] = TypeLinuxAudit + config[ProcessAt] = Server + config["key"] = "arr" + + return config +} diff --git a/parser/linuxaudit/audit_test.go b/parser/linuxaudit/audit_test.go new file mode 100644 index 000000000..ba7bee6ba --- /dev/null +++ b/parser/linuxaudit/audit_test.go @@ -0,0 +1,232 @@ +package linuxaudit + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + + . "github.com/qiniu/logkit/parser/config" + . "github.com/qiniu/logkit/utils/models" +) + +func TestParse(t *testing.T) { + tests := []struct { + s []string + expectData []Data + }{ + { + expectData: []Data{}, + }, + { + s: []string{`type=SYSCALL msg=audit(1364481363.243:24287): arch=c000003e syscall=2 success=no exit=-13 a0=7fffd19c5592 a1=0 a2=7fffd19c4b50`, + `type=CWD msg='op=PAM:secret test1="a" res=success' + cwd="/home/shadowman" `, + `type=PATH msg=audit(1364481363.243:24287): item=0 name="/etc/ssh/sshd_config" inode=409248 dev=fd:00 dev=system_u:object_r:etc_t:s0`}, + expectData: []Data{ + { + "arch": "c000003e", + "type": "SYSCALL", + "msg_timestamp": "1364481363.243", + "msg_id": "24287", + "syscall": "2", + "success": "no", + "exit": "-13", + "a0": "7fffd19c5592", + "a1": "0", + "a2": "7fffd19c4b50", + }, + { + "type": "CWD", + "cwd": "/home/shadowman", + "msg": Data{"op": "PAM:secret", "test1": "a", "res": "success"}, + }, + { + "type": "PATH", + "msg_timestamp": "1364481363.243", + "msg_id": "24287", + "item": "0", + "name": "/etc/ssh/sshd_config", + "inode": "409248", + "dev": "fd:00", + "dev_1": "system_u:object_r:etc_t:s0", + }, + }, + }, + } + l := Parser{ + name: TypeLinuxAudit, + } + for _, tt := range tests { + got, err := l.Parse(tt.s) + if c, ok := err.(*StatsError); ok { + err = errors.New(c.LastError) + assert.Equal(t, int64(0), c.Errors) + } + + for i, m := range got { + assert.Equal(t, tt.expectData[i], m) + } + } +} + +func Test_parseLine(t *testing.T) { + tests := []struct { + line string + expectData Data + }{ + { + expectData: Data{}, + }, + { + line: `type=SYSCALL msg=audit(1364481363.243:24287): arch=c000003e syscall=2 success=no exit=-13 a0=7fffd19c5592 a1=0 a2=7fffd19c4b50`, + expectData: Data{ + "arch": "c000003e", + "type": "SYSCALL", + "msg_timestamp": "2013-03-28T14:36:03.243Z", + "msg_id": "24287", + "syscall": "2", + "success": "no", + "exit": "-13", + "a0": "7fffd19c5592", + "a1": "0", + "a2": "7fffd19c4b50", + }, + }, + { + line: `type=CWD msg='op=PAM:secret test1="a" res=success' + cwd="/home/shadowman" `, + expectData: Data{ + "type": "CWD", + "cwd": "/home/shadowman", + "msg": Data{"op": "PAM:secret", "test1": "a", "res": "success"}, + }, + }, + { + line: `type=PATH msg=audit(1364481363.243:24287): item=0 name="/etc/ssh/sshd_config" inode=409248 dev=fd:00 dev=system_u:object_r:etc_t:s0`, + expectData: Data{ + "type": "PATH", + "msg_timestamp": "2013-03-28T14:36:03.243Z", + "msg_id": "24287", + "item": "0", + "name": "/etc/ssh/sshd_config", + "inode": "409248", + "dev": "fd:00", + "dev_1": "system_u:object_r:etc_t:s0", + }, + }, + } + l := Parser{ + name: TypeLinuxAudit, + } + for _, tt := range tests { + got, err := l.parse(tt.line) + assert.Nil(t, err) + assert.Equal(t, len(tt.expectData), len(got)) + for i, m := range got { + assert.Equal(t, tt.expectData[i], m) + } + } +} + +func Test_processSpace(t *testing.T) { + tests := []struct { + key string + line string + data Data + expect Data + }{ + { + data: Data{}, + }, + { + key: "a", + line: "b", + data: Data{}, + expect: Data{"a": "b"}, + }, + { + key: "msg", + line: "audit(111111:222)", + data: Data{}, + expect: Data{"msg_timestamp": "2005-03-18T01:40:00Z", "msg_id": "222"}, + }, + } + + for _, test := range tests { + processSpace(test.key, test.line, test.data) + assert.EqualValues(t, len(test.expect), len(test.data)) + for key, value := range test.expect { + val, ok := test.data[key] + assert.True(t, ok) + assert.EqualValues(t, value, val) + } + } +} + +func Test_getTimestampID(t *testing.T) { + tests := []struct { + line string + data Data + success bool + }{ + { + data: Data{}, + }, + { + line: "a", + data: Data{}, + }, + { + line: "audit(111111:222)", + data: Data{}, + success: true, + }, + } + + for _, test := range tests { + actual := getTimestampID(test.line, test.data) + assert.EqualValues(t, test.success, actual) + if actual { + _, ok := test.data["msg_timestamp"] + assert.True(t, ok) + _, ok = test.data["msg_id"] + assert.True(t, ok) + } + } +} + +func Test_setData(t *testing.T) { + tests := []struct { + key string + line string + data Data + expect Data + }{ + { + data: Data{}, + }, + { + key: "a", + line: "b", + data: Data{}, + expect: Data{"a": "b"}, + }, + { + key: "msg", + line: "audit(111111:222)", + data: Data{}, + expect: Data{"msg": "audit(111111:222)"}, + }, + } + + for _, test := range tests { + setData(test.key, test.line, test.data) + assert.EqualValues(t, len(test.expect), len(test.data)) + for key, value := range test.expect { + val, ok := test.data[key] + assert.True(t, ok) + assert.EqualValues(t, value, val) + } + } +} diff --git a/parser/parser.go b/parser/parser.go index d586a8bfb..c56a38f46 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -17,6 +17,10 @@ type Parser interface { Parse(lines []string) (datas []Data, err error) } +type ServerParser interface { + ServerConfig() map[string]interface{} +} + type ParserType interface { Type() string } diff --git a/transforms/ip/ip.go b/transforms/ip/ip.go index cdb74ba0b..d1432776f 100644 --- a/transforms/ip/ip.go +++ b/transforms/ip/ip.go @@ -20,9 +20,6 @@ const ( Latitude = "Latitude" Longitude = "Longitude" DistrictCode = "DistrictCode" - - Local = "local" - Server = "server" ) var ( @@ -317,8 +314,8 @@ func (t *Transformer) Close() error { func (t *Transformer) ServerConfig() map[string]interface{} { config := make(map[string]interface{}) - config[transforms.KeyType] = Name - config[transforms.TransformAt] = t.TransformAt + config[KeyType] = Name + config[ProcessAt] = t.TransformAt config["key"] = t.Key return config diff --git a/transforms/registry.go b/transforms/registry.go index c04874f49..d4c6a4c3a 100644 --- a/transforms/registry.go +++ b/transforms/registry.go @@ -5,12 +5,8 @@ import ( ) const ( - KeyType = "type" - TransformAt = "transform_at" -) -const ( TransformTypeString = "string" TransformTypeLong = "long" TransformTypeFloat = "float" diff --git a/utils/models/models.go b/utils/models/models.go index 81e7e097c..dd85e7242 100644 --- a/utils/models/models.go +++ b/utils/models/models.go @@ -21,6 +21,11 @@ const ( const ( CheckPattern = "^[a-zA-Z_][a-zA-Z0-9_]{0,127}$" + KeyType = "type" + ProcessAt = "process_at" + Local = "local" + Server = "server" + GlobalKeyName = "name" ExtraInfo = "extra_info" /* 该选项兼容如下配置 KeyPandoraExtraInfo */ diff --git a/utils/models/utils.go b/utils/models/utils.go index bacda3562..0df4f0127 100644 --- a/utils/models/utils.go +++ b/utils/models/utils.go @@ -750,18 +750,10 @@ func ConvertDate(layoutBefore, layoutAfter string, offset int, loc *time.Locatio } news := s timestamp := strconv.FormatInt(news, 10) - timeSecondPrecision := 16 - //补齐16位 - for i := len(timestamp); i < timeSecondPrecision; i++ { - timestamp += "0" - } - // 取前16位，截取精度 微妙 - timestamp = timestamp[0:timeSecondPrecision] - t, err := strconv.ParseInt(timestamp, 10, 64) + tm, err := GetTime(timestamp) if err != nil { return v, err } - tm := time.Unix(0, t*int64(time.Microsecond)) return FormatWithUserOption(layoutAfter, offset, tm), nil } @@ -1050,3 +1042,18 @@ func CheckPath(path string) (string, error) { CheckFileMode(realPath, fileMode) return realPath, nil } + +func GetTime(timestamp string) (time.Time, error) { + timeSecondPrecision := 19 + //补齐19位 + for i := len(timestamp); i < timeSecondPrecision; i++ { + timestamp += "0" + } + // 取前19位，截取精度 纳妙 + timestamp = timestamp[0:timeSecondPrecision] + t, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return time.Time{}, err + } + return time.Unix(0, t), nil +} diff --git a/vendor/github.com/axgle/mahonia/LICENSE b/vendor/github.com/axgle/mahonia/LICENSE new file mode 100644 index 000000000..eddd9ddb4 --- /dev/null +++ b/vendor/github.com/axgle/mahonia/LICENSE @@ -0,0 +1,2 @@ +BSD 3-Clause License +https://opensource.org/licenses/BSD-3-Clause diff --git a/vendor/github.com/axgle/mahonia/README.md b/vendor/github.com/axgle/mahonia/README.md index db8f250ab..f92adba71 100644 --- a/vendor/github.com/axgle/mahonia/README.md +++ b/vendor/github.com/axgle/mahonia/README.md @@ -25,3 +25,6 @@ example fmt.Println(enc.ConvertString("hello,世界")) } +donate +------- +https://github.com/axgle/mahonia/wiki/Donate diff --git a/vendor/vendor.json b/vendor/vendor.json index c02afb750..3e26ff34a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -195,10 +195,10 @@ "revisionTime": "2018-04-03T20:44:34Z" }, { - "checksumSHA1": "ucu0eJ90b2Jlj+KPt0YJzGfwm7s=", + "checksumSHA1": "kNtoxGIa3rApgMx/PREQeKqjKw8=", "path": "github.com/axgle/mahonia", - "revision": "c528b747d92d41ca581a7acf3e604c396fc7c034", - "revisionTime": "2015-10-19T00:40:08Z" + "revision": "3358181d7394e26beccfae0ffde05193ef3be33a", + "revisionTime": "2018-02-08T00:28:26Z" }, { "checksumSHA1": "OW70e5OFoTTxOwUdcuiejGmxF+A=",