From d0b98db00c9f6936a191b8387b8abf7984b8e4e6 Mon Sep 17 00:00:00 2001 From: Felix Hao Date: Tue, 2 Apr 2019 16:58:20 +0800 Subject: [PATCH] fix log field (#3) --- README.md | 16 ++- pkg/log/field.go | 58 +++++++++ pkg/log/handler.go | 11 +- pkg/log/internal/LICENSE.txt | 2 +- pkg/log/internal/{ => core}/buffer.go | 0 pkg/log/internal/{ => core}/buffer_test.go | 0 pkg/log/internal/{ => core}/bufferpool.go | 0 pkg/log/internal/{ => core}/encoder.go | 0 pkg/log/internal/core/field.go | 122 ++++++++++++++++++ pkg/log/internal/{ => core}/json_encoder.go | 0 pkg/log/internal/{ => core}/pool.go | 0 pkg/log/internal/{ => core}/pool_test.go | 0 pkg/log/internal/field.go | 6 - .../internal/filewriter/filewriter_test.go | 2 +- pkg/log/log.go | 90 +------------ pkg/log/util.go | 14 +- pkg/log/verbose.go | 2 +- pkg/naming/README.md | 9 ++ pkg/naming/discovery/discovery.go | 29 +---- pkg/naming/naming.go | 1 - pkg/net/http/blademaster/client.go | 15 --- 21 files changed, 223 insertions(+), 154 deletions(-) create mode 100644 pkg/log/field.go rename pkg/log/internal/{ => core}/buffer.go (100%) rename pkg/log/internal/{ => core}/buffer_test.go (100%) rename pkg/log/internal/{ => core}/bufferpool.go (100%) rename pkg/log/internal/{ => core}/encoder.go (100%) create mode 100644 pkg/log/internal/core/field.go rename pkg/log/internal/{ => core}/json_encoder.go (100%) rename pkg/log/internal/{ => core}/pool.go (100%) rename pkg/log/internal/{ => core}/pool_test.go (100%) delete mode 100644 pkg/log/internal/field.go diff --git a/README.md b/README.md index 7696d6a1a..ba9e2649b 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,15 @@ Kratos是[bilibili](https://www.bilibili.com)开源的一套Go微服务框架， # TODOs -- [ ] log&log-agent @围城 -- [ ] config @志辉 -- [ ] bm @佳辉 -- [ ] warden @龙虾 -- [x] naming discovery @堂辉 -- [ ] cache&database @小旭 -- [ ] kratos tool @普余 +- [x] naming discovery +- [x] log +- [ ] log-agent +- [ ] config +- [ ] bm +- [ ] warden +- [ ] cache +- [ ] database +- [ ] kratos tool # issues diff --git a/pkg/log/field.go b/pkg/log/field.go new file mode 100644 index 000000000..fe2609b95 --- /dev/null +++ b/pkg/log/field.go @@ -0,0 +1,58 @@ +package log + +import ( + "math" + "time" + + "github.com/bilibili/Kratos/pkg/log/internal/core" +) + +// D represents a map of entry level data used for structured logging. +// type D map[string]interface{} +type D = core.Field + +// KVString construct Field with string value. +func KVString(key string, value string) D { + return D{Key: key, Type: core.StringType, StringVal: value} +} + +// KVInt construct Field with int value. +func KVInt(key string, value int) D { + return D{Key: key, Type: core.IntTpye, Int64Val: int64(value)} +} + +// KVInt64 construct D with int64 value. +func KVInt64(key string, value int64) D { + return D{Key: key, Type: core.Int64Type, Int64Val: value} +} + +// KVUint construct Field with uint value. +func KVUint(key string, value uint) D { + return D{Key: key, Type: core.UintType, Int64Val: int64(value)} +} + +// KVUint64 construct Field with uint64 value. +func KVUint64(key string, value uint64) D { + return D{Key: key, Type: core.Uint64Type, Int64Val: int64(value)} +} + +// KVFloat32 construct Field with float32 value. +func KVFloat32(key string, value float32) D { + return D{Key: key, Type: core.Float32Type, Int64Val: int64(math.Float32bits(value))} +} + +// KVFloat64 construct Field with float64 value. +func KVFloat64(key string, value float64) D { + return D{Key: key, Type: core.Float64Type, Int64Val: int64(math.Float64bits(value))} +} + +// KVDuration construct Field with Duration value. +func KVDuration(key string, value time.Duration) D { + return D{Key: key, Type: core.DurationType, Int64Val: int64(value)} +} + +// KV return a log kv for logging field. +// NOTE: use KV{type name} can avoid object alloc and get better performance. []~(￣▽￣)~*干杯 +func KV(key string, value interface{}) D { + return D{Key: key, Value: value} +} diff --git a/pkg/log/handler.go b/pkg/log/handler.go index 1dffd74d7..44c2397e3 100644 --- a/pkg/log/handler.go +++ b/pkg/log/handler.go @@ -76,19 +76,20 @@ type Handlers struct { // Log handlers logging. func (hs Handlers) Log(c context.Context, lv Level, d ...D) { - var fn string + var hasSource bool for i := range d { if _, ok := hs.filters[d[i].Key]; ok { d[i].Value = "***" } if d[i].Key == _source { - fn = d[i].Value.(string) + hasSource = true } } - if fn == "" { - fn = funcName(4) + if !hasSource { + fn := funcName(3) + d = append(d, KVString(_source, fn)) } - d = append(d, KV(_source, fn), KV(_time, time.Now()), KV(_levelValue, int(lv)), KV(_level, lv.String())) + d = append(d, KV(_time, time.Now()), KVInt64(_levelValue, int64(lv)), KVString(_level, lv.String())) for _, h := range hs.handlers { h.Log(c, lv, d...) } diff --git a/pkg/log/internal/LICENSE.txt b/pkg/log/internal/LICENSE.txt index 1b5d6ae6c..8ed1527bd 100644 --- a/pkg/log/internal/LICENSE.txt +++ b/pkg/log/internal/LICENSE.txt @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +THE SOFTWARE. \ No newline at end of file diff --git a/pkg/log/internal/buffer.go b/pkg/log/internal/core/buffer.go similarity index 100% rename from pkg/log/internal/buffer.go rename to pkg/log/internal/core/buffer.go diff --git a/pkg/log/internal/buffer_test.go b/pkg/log/internal/core/buffer_test.go similarity index 100% rename from pkg/log/internal/buffer_test.go rename to pkg/log/internal/core/buffer_test.go diff --git a/pkg/log/internal/bufferpool.go b/pkg/log/internal/core/bufferpool.go similarity index 100% rename from pkg/log/internal/bufferpool.go rename to pkg/log/internal/core/bufferpool.go diff --git a/pkg/log/internal/encoder.go b/pkg/log/internal/core/encoder.go similarity index 100% rename from pkg/log/internal/encoder.go rename to pkg/log/internal/core/encoder.go diff --git a/pkg/log/internal/core/field.go b/pkg/log/internal/core/field.go new file mode 100644 index 000000000..7c7e30ca8 --- /dev/null +++ b/pkg/log/internal/core/field.go @@ -0,0 +1,122 @@ +package core + +import ( + "fmt" + "math" + "time" + + xtime "github.com/bilibili/Kratos/pkg/time" +) + +// FieldType represent D value type +type FieldType int32 + +// DType enum +const ( + UnknownType FieldType = iota + StringType + IntTpye + Int64Type + UintType + Uint64Type + Float32Type + Float64Type + DurationType +) + +// Field is for encoder +type Field struct { + Key string + Value interface{} + Type FieldType + StringVal string + Int64Val int64 +} + +// AddTo exports a field through the ObjectEncoder interface. It's primarily +// useful to library authors, and shouldn't be necessary in most applications. +func (f Field) AddTo(enc ObjectEncoder) { + if f.Type == UnknownType { + f.assertAddTo(enc) + return + } + switch f.Type { + case StringType: + enc.AddString(f.Key, f.StringVal) + case IntTpye: + enc.AddInt(f.Key, int(f.Int64Val)) + case Int64Type: + enc.AddInt64(f.Key, f.Int64Val) + case UintType: + enc.AddUint(f.Key, uint(f.Int64Val)) + case Uint64Type: + enc.AddUint64(f.Key, uint64(f.Int64Val)) + case Float32Type: + enc.AddFloat32(f.Key, math.Float32frombits(uint32(f.Int64Val))) + case Float64Type: + enc.AddFloat64(f.Key, math.Float64frombits(uint64(f.Int64Val))) + case DurationType: + enc.AddDuration(f.Key, time.Duration(f.Int64Val)) + default: + panic(fmt.Sprintf("unknown field type: %v", f)) + } +} + +func (f Field) assertAddTo(enc ObjectEncoder) { + // assert interface + switch val := f.Value.(type) { + case bool: + enc.AddBool(f.Key, val) + case complex128: + enc.AddComplex128(f.Key, val) + case complex64: + enc.AddComplex64(f.Key, val) + case float64: + enc.AddFloat64(f.Key, val) + case float32: + enc.AddFloat32(f.Key, val) + case int: + enc.AddInt(f.Key, val) + case int64: + enc.AddInt64(f.Key, val) + case int32: + enc.AddInt32(f.Key, val) + case int16: + enc.AddInt16(f.Key, val) + case int8: + enc.AddInt8(f.Key, val) + case string: + enc.AddString(f.Key, val) + case uint: + enc.AddUint(f.Key, val) + case uint64: + enc.AddUint64(f.Key, val) + case uint32: + enc.AddUint32(f.Key, val) + case uint16: + enc.AddUint16(f.Key, val) + case uint8: + enc.AddUint8(f.Key, val) + case []byte: + enc.AddByteString(f.Key, val) + case uintptr: + enc.AddUintptr(f.Key, val) + case time.Time: + enc.AddTime(f.Key, val) + case xtime.Time: + enc.AddTime(f.Key, val.Time()) + case time.Duration: + enc.AddDuration(f.Key, val) + case xtime.Duration: + enc.AddDuration(f.Key, time.Duration(val)) + case error: + enc.AddString(f.Key, val.Error()) + case fmt.Stringer: + enc.AddString(f.Key, val.String()) + default: + err := enc.AddReflected(f.Key, val) + if err != nil { + enc.AddString(fmt.Sprintf("%sError", f.Key), err.Error()) + } + } +} diff --git a/pkg/log/internal/json_encoder.go b/pkg/log/internal/core/json_encoder.go similarity index 100% rename from pkg/log/internal/json_encoder.go rename to pkg/log/internal/core/json_encoder.go diff --git a/pkg/log/internal/pool.go b/pkg/log/internal/core/pool.go similarity index 100% rename from pkg/log/internal/pool.go rename to pkg/log/internal/core/pool.go diff --git a/pkg/log/internal/pool_test.go b/pkg/log/internal/core/pool_test.go similarity index 100% rename from pkg/log/internal/pool_test.go rename to pkg/log/internal/core/pool_test.go diff --git a/pkg/log/internal/field.go b/pkg/log/internal/field.go deleted file mode 100644 index 3116973e3..000000000 --- a/pkg/log/internal/field.go +++ /dev/null @@ -1,6 +0,0 @@ -package core - -// Field is for encoder -type Field interface { - AddTo(enc ObjectEncoder) -} diff --git a/pkg/log/internal/filewriter/filewriter_test.go b/pkg/log/internal/filewriter/filewriter_test.go index 2b3612ff7..6178da78b 100644 --- a/pkg/log/internal/filewriter/filewriter_test.go +++ b/pkg/log/internal/filewriter/filewriter_test.go @@ -24,7 +24,7 @@ func touch(dir, name string) { func TestMain(m *testing.M) { ret := m.Run() - // os.RemoveAll(logdir) + os.RemoveAll(logdir) os.Exit(ret) } diff --git a/pkg/log/log.go b/pkg/log/log.go index 436abc3c3..1c3e4c283 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -7,7 +7,6 @@ import ( "os" "strconv" "strings" - "time" "github.com/bilibili/Kratos/pkg/conf/env" ) @@ -115,83 +114,6 @@ func Init(conf *Config) { c = conf } -// D represents a map of entry level data used for structured logging. -// type D map[string]interface{} -type D struct { - Key string - Value interface{} -} - -// AddTo exports a field through the ObjectEncoder interface. It's primarily -// useful to library authors, and shouldn't be necessary in most applications. -func (d D) AddTo(enc core.ObjectEncoder) { - var err error - switch val := d.Value.(type) { - case bool: - enc.AddBool(d.Key, val) - case complex128: - enc.AddComplex128(d.Key, val) - case complex64: - enc.AddComplex64(d.Key, val) - case float64: - enc.AddFloat64(d.Key, val) - case float32: - enc.AddFloat32(d.Key, val) - case int: - enc.AddInt(d.Key, val) - case int64: - enc.AddInt64(d.Key, val) - case int32: - enc.AddInt32(d.Key, val) - case int16: - enc.AddInt16(d.Key, val) - case int8: - enc.AddInt8(d.Key, val) - case string: - enc.AddString(d.Key, val) - case uint: - enc.AddUint(d.Key, val) - case uint64: - enc.AddUint64(d.Key, val) - case uint32: - enc.AddUint32(d.Key, val) - case uint16: - enc.AddUint16(d.Key, val) - case uint8: - enc.AddUint8(d.Key, val) - case []byte: - enc.AddByteString(d.Key, val) - case uintptr: - enc.AddUintptr(d.Key, val) - case time.Time: - enc.AddTime(d.Key, val) - case xtime.Time: - enc.AddTime(d.Key, val.Time()) - case time.Duration: - enc.AddDuration(d.Key, val) - case xtime.Duration: - enc.AddDuration(d.Key, time.Duration(val)) - case error: - enc.AddString(d.Key, val.Error()) - case fmt.Stringer: - enc.AddString(d.Key, val.String()) - default: - err = enc.AddReflected(d.Key, val) - } - - if err != nil { - enc.AddString(fmt.Sprintf("%sError", d.Key), err.Error()) - } -} - -// KV return a log kv for logging field. -func KV(key string, value interface{}) D { - return D{ - Key: key, - Value: value, - } -} - type logFilter []string func (f *logFilter) String() string { @@ -209,32 +131,32 @@ func (f *logFilter) Set(value string) error { // Info logs a message at the info log level. func Info(format string, args ...interface{}) { - h.Log(context.Background(), _infoLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(context.Background(), _infoLevel, KVString(_log, fmt.Sprintf(format, args...))) } // Warn logs a message at the warning log level. func Warn(format string, args ...interface{}) { - h.Log(context.Background(), _warnLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(context.Background(), _warnLevel, KVString(_log, fmt.Sprintf(format, args...))) } // Error logs a message at the error log level. func Error(format string, args ...interface{}) { - h.Log(context.Background(), _errorLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(context.Background(), _errorLevel, KVString(_log, fmt.Sprintf(format, args...))) } // Infoc logs a message at the info log level. func Infoc(ctx context.Context, format string, args ...interface{}) { - h.Log(ctx, _infoLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(ctx, _infoLevel, KVString(_log, fmt.Sprintf(format, args...))) } // Errorc logs a message at the error log level. func Errorc(ctx context.Context, format string, args ...interface{}) { - h.Log(ctx, _errorLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(ctx, _errorLevel, KVString(_log, fmt.Sprintf(format, args...))) } // Warnc logs a message at the warning log level. func Warnc(ctx context.Context, format string, args ...interface{}) { - h.Log(ctx, _warnLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(ctx, _warnLevel, KVString(_log, fmt.Sprintf(format, args...))) } // Infov logs a message at the info log level. diff --git a/pkg/log/util.go b/pkg/log/util.go index de0454377..648664b0a 100644 --- a/pkg/log/util.go +++ b/pkg/log/util.go @@ -5,15 +5,12 @@ import ( "fmt" "runtime" "strconv" - "sync" "github.com/bilibili/Kratos/pkg/conf/env" "github.com/bilibili/Kratos/pkg/net/metadata" "github.com/bilibili/Kratos/pkg/net/trace" ) -var fm sync.Map - func addExtraField(ctx context.Context, fields map[string]interface{}) { if t, ok := trace.FromContext(ctx); ok { if s, ok := t.(fmt.Stringer); ok { @@ -42,13 +39,8 @@ func addExtraField(ctx context.Context, fields map[string]interface{}) { // funcName get func name. func funcName(skip int) (name string) { - if pc, _, lineNo, ok := runtime.Caller(skip); ok { - if v, ok := fm.Load(pc); ok { - name = v.(string) - } else { - name = runtime.FuncForPC(pc).Name() + ":" + strconv.FormatInt(int64(lineNo), 10) - fm.Store(pc, name) - } + if _, file, lineNo, ok := runtime.Caller(skip); ok { + return file + ":" + strconv.Itoa(lineNo) } - return + return "unknown:0" } diff --git a/pkg/log/verbose.go b/pkg/log/verbose.go index c45af4824..8c23effee 100644 --- a/pkg/log/verbose.go +++ b/pkg/log/verbose.go @@ -85,7 +85,7 @@ func V(v int32) Verbose { // Info logs a message at the info log level. func (v Verbose) Info(format string, args ...interface{}) { if v { - h.Log(context.Background(), _infoLevel, KV(_log, fmt.Sprintf(format, args...))) + h.Log(context.Background(), _infoLevel, KVString(_log, fmt.Sprintf(format, args...))) } } diff --git a/pkg/naming/README.md b/pkg/naming/README.md index 5c4e8ac6a..959efc91d 100644 --- a/pkg/naming/README.md +++ b/pkg/naming/README.md @@ -3,3 +3,12 @@ ## 项目简介 服务发现、服务注册相关的SDK集合 + +## 现状 + +目前默认实现了B站开源的[Discovery](https://github.com/bilibili/discovery)服务注册与发现SDK。 +但在使用之前，请确认discovery服务部署完成，并将该discovery.go内`fixConfig`方法的默认配置进行完善。 + +## 使用 + +可实现`naming`内的`Builder`&`Resolver`&`Registry`接口用于服务注册与发现，比如B站内部还实现了zk的。 diff --git a/pkg/naming/discovery/discovery.go b/pkg/naming/discovery/discovery.go index e04399329..a773ac3cb 100644 --- a/pkg/naming/discovery/discovery.go +++ b/pkg/naming/discovery/discovery.go @@ -21,7 +21,7 @@ import ( bm "github.com/bilibili/Kratos/pkg/net/http/blademaster" "github.com/bilibili/Kratos/pkg/net/netutil" "github.com/bilibili/Kratos/pkg/net/netutil/breaker" - "github.com/bilibili/Kratos/pkg/str" + xstr "github.com/bilibili/Kratos/pkg/str" xtime "github.com/bilibili/Kratos/pkg/time" ) @@ -53,13 +53,10 @@ var ( // Config discovery configures. type Config struct { - Nodes []string - Key string - Secret string - Region string - Zone string - Env string - Host string + Nodes []string + Zone string + Env string + Host string } // Discovery is discovery client. @@ -88,10 +85,7 @@ type appInfo struct { func fixConfig(c *Config) { if len(c.Nodes) == 0 { - c.Nodes = []string{"api.bilibili.co"} - } - if env.Region != "" { - c.Region = env.Region + c.Nodes = []string{"NOTE: please config a default HOST"} } if env.Zone != "" { c.Zone = env.Zone @@ -136,11 +130,7 @@ func Build(id string) naming.Resolver { // New new a discovery client. func New(c *Config) (d *Discovery) { if c == nil { - c = &Config{ - Nodes: []string{"discovery.bilibili.co", "api.bilibili.co"}, - Key: "discovery", - Secret: "discovery", - } + c = &Config{} } fixConfig(c) ctx, cancel := context.WithCancel(context.Background()) @@ -154,10 +144,6 @@ func New(c *Config) (d *Discovery) { } // httpClient cfg := &bm.ClientConfig{ - App: &bm.App{ - Key: c.Key, - Secret: c.Secret, - }, Dial: xtime.Duration(3 * time.Second), Timeout: xtime.Duration(40 * time.Second), Breaker: &breaker.Config{ @@ -676,7 +662,6 @@ func (d *Discovery) broadcast(apps map[string]naming.InstancesInfo) { func (d *Discovery) newParams(conf *Config) url.Values { params := url.Values{} - params.Set("region", conf.Region) params.Set("zone", conf.Zone) params.Set("env", conf.Env) params.Set("hostname", conf.Host) diff --git a/pkg/naming/naming.go b/pkg/naming/naming.go index 8ec9cda4f..9b1bfa303 100644 --- a/pkg/naming/naming.go +++ b/pkg/naming/naming.go @@ -53,7 +53,6 @@ type Scheduler struct { // Resolver resolve naming service type Resolver interface { Fetch(context.Context) (*InstancesInfo, bool) - //Unwatch(id string) Watch() <-chan struct{} Close() error } diff --git a/pkg/net/http/blademaster/client.go b/pkg/net/http/blademaster/client.go index 45678bb31..eacdaa5f9 100644 --- a/pkg/net/http/blademaster/client.go +++ b/pkg/net/http/blademaster/client.go @@ -43,15 +43,8 @@ func init() { } } -// App bilibili intranet authorization. -type App struct { - Key string - Secret string -} - // ClientConfig is http client conf. type ClientConfig struct { - *App Dial xtime.Duration Timeout xtime.Duration KeepAlive xtime.Duration @@ -95,10 +88,6 @@ func NewClient(c *ClientConfig) *Client { client.urlConf = make(map[string]*ClientConfig) client.hostConf = make(map[string]*ClientConfig) client.breaker = breaker.NewGroup(c.Breaker) - // check appkey - if c.Key == "" || c.Secret == "" { - panic("http client must config appkey and appsecret") - } if c.Timeout <= 0 { panic("must config http timeout!!!") } @@ -120,10 +109,6 @@ func (client *Client) SetTransport(t xhttp.RoundTripper) { // SetConfig set client config. func (client *Client) SetConfig(c *ClientConfig) { client.mutex.Lock() - if c.App != nil { - client.conf.App.Key = c.App.Key - client.conf.App.Secret = c.App.Secret - } if c.Timeout > 0 { client.conf.Timeout = c.Timeout }