From 038b0ebb27fbd9829c0b64ba8a1d70bc9da26abd Mon Sep 17 00:00:00 2001 From: Felix Hao Date: Thu, 4 Apr 2019 15:55:54 +0800 Subject: [PATCH] add conf paladin (#5) --- go.mod | 2 + pkg/conf/paladin/README.md | 72 ++++++++ pkg/conf/paladin/client.go | 49 ++++++ pkg/conf/paladin/default.go | 86 ++++++++++ pkg/conf/paladin/example_test.go | 112 ++++++++++++ pkg/conf/paladin/file.go | 194 +++++++++++++++++++++ pkg/conf/paladin/file_test.go | 108 ++++++++++++ pkg/conf/paladin/helper.go | 76 ++++++++ pkg/conf/paladin/helper_test.go | 286 +++++++++++++++++++++++++++++++ pkg/conf/paladin/map.go | 55 ++++++ pkg/conf/paladin/map_test.go | 94 ++++++++++ pkg/conf/paladin/mock.go | 40 +++++ pkg/conf/paladin/mock_test.go | 37 ++++ pkg/conf/paladin/toml.go | 73 ++++++++ pkg/conf/paladin/value.go | 157 +++++++++++++++++ pkg/conf/paladin/value_test.go | 206 ++++++++++++++++++++++ 16 files changed, 1647 insertions(+) create mode 100644 pkg/conf/paladin/README.md create mode 100644 pkg/conf/paladin/client.go create mode 100644 pkg/conf/paladin/default.go create mode 100644 pkg/conf/paladin/example_test.go create mode 100644 pkg/conf/paladin/file.go create mode 100644 pkg/conf/paladin/file_test.go create mode 100644 pkg/conf/paladin/helper.go create mode 100644 pkg/conf/paladin/helper_test.go create mode 100644 pkg/conf/paladin/map.go create mode 100644 pkg/conf/paladin/map_test.go create mode 100644 pkg/conf/paladin/mock.go create mode 100644 pkg/conf/paladin/mock_test.go create mode 100644 pkg/conf/paladin/toml.go create mode 100644 pkg/conf/paladin/value.go create mode 100644 pkg/conf/paladin/value_test.go diff --git a/go.mod b/go.mod index d7274efab..7d6c86f2b 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,9 @@ module github.com/bilibili/Kratos require ( + github.com/BurntSushi/toml v0.3.1 github.com/fatih/color v1.7.0 + github.com/fsnotify/fsnotify v1.4.7 github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/gogo/protobuf v1.2.0 diff --git a/pkg/conf/paladin/README.md b/pkg/conf/paladin/README.md new file mode 100644 index 000000000..6db087389 --- /dev/null +++ b/pkg/conf/paladin/README.md @@ -0,0 +1,72 @@ +#### paladin + +##### 项目简介 + +paladin 是一个config SDK客户端，包括了file、mock几个抽象功能，方便使用本地文件或者sven配置中心，并且集成了对象自动reload功能。 + + +local files: +``` +demo -conf=/data/conf/app/msm-servie.toml +// or dir +demo -conf=/data/conf/app/ + +``` +example: +``` +type exampleConf struct { + Bool bool + Int int64 + Float float64 + String string +} + +func (e *exampleConf) Set(text string) error { + var ec exampleConf + if err := toml.Unmarshal([]byte(text), &ec); err != nil { + return err + } + *e = ec + return nil +} + +func ExampleClient() { + if err := paladin.Init(); err != nil { + panic(err) + } + var ( + ec exampleConf + eo exampleConf + m paladin.TOML + strs []string + ) + // config unmarshal + if err := paladin.Get("example.toml").UnmarshalTOML(&ec); err != nil { + panic(err) + } + // config setter + if err := paladin.Watch("example.toml", &ec); err != nil { + panic(err) + } + // paladin map + if err := paladin.Watch("example.toml", &m); err != nil { + panic(err) + } + s, err := m.Value("key").String() + b, err := m.Value("key").Bool() + i, err := m.Value("key").Int64() + f, err := m.Value("key").Float64() + // value slice + err = m.Value("strings").Slice(&strs) + // watch key + for event := range paladin.WatchEvent(context.TODO(), "key") { + fmt.Println(event) + } +} +``` + +##### 编译环境 + +- **请只用 Golang v1.12.x 以上版本编译执行** + +##### 依赖包 diff --git a/pkg/conf/paladin/client.go b/pkg/conf/paladin/client.go new file mode 100644 index 000000000..62c0532fe --- /dev/null +++ b/pkg/conf/paladin/client.go @@ -0,0 +1,49 @@ +package paladin + +import ( + "context" +) + +const ( + // EventAdd config add event. + EventAdd EventType = iota + // EventUpdate config update event. + EventUpdate + // EventRemove config remove event. + EventRemove +) + +// EventType is config event. +type EventType int + +// Event is watch event. +type Event struct { + Event EventType + Key string + Value string +} + +// Watcher is config watcher. +type Watcher interface { + WatchEvent(context.Context, ...string) <-chan Event + Close() error +} + +// Setter is value setter. +type Setter interface { + Set(string) error +} + +// Getter is value getter. +type Getter interface { + // Get a config value by a config key(may be a sven filename). + Get(string) *Value + // GetAll return all config key->value map. + GetAll() *Map +} + +// Client is config client. +type Client interface { + Watcher + Getter +} diff --git a/pkg/conf/paladin/default.go b/pkg/conf/paladin/default.go new file mode 100644 index 000000000..bd7dfb88a --- /dev/null +++ b/pkg/conf/paladin/default.go @@ -0,0 +1,86 @@ +package paladin + +import ( + "context" + "flag" + + "github.com/bilibili/Kratos/pkg/log" +) + +var ( + // DefaultClient default client. + DefaultClient Client + confPath string + vars = make(map[string][]Setter) // NOTE: no thread safe +) + +func init() { + flag.StringVar(&confPath, "conf", "", "default config path") +} + +// Init init config client. +func Init() (err error) { + if confPath != "" { + DefaultClient, err = NewFile(confPath) + } else { + // TODO: config service + return + } + if err != nil { + return + } + go func() { + for event := range DefaultClient.WatchEvent(context.Background()) { + if event.Event != EventUpdate && event.Event != EventAdd { + continue + } + if sets, ok := vars[event.Key]; ok { + for _, s := range sets { + if err := s.Set(event.Value); err != nil { + log.Error("paladin: vars:%v event:%v error(%v)", s, event, err) + } + } + } + } + }() + return +} + +// Watch watch on a key. The configuration implements the setter interface, which is invoked when the configuration changes. +func Watch(key string, s Setter) error { + v := DefaultClient.Get(key) + str, err := v.Raw() + if err != nil { + return err + } + if err := s.Set(str); err != nil { + return err + } + vars[key] = append(vars[key], s) + return nil +} + +// WatchEvent watch on multi keys. Events are returned when the configuration changes. +func WatchEvent(ctx context.Context, keys ...string) <-chan Event { + return DefaultClient.WatchEvent(ctx, keys...) +} + +// Get return value by key. +func Get(key string) *Value { + return DefaultClient.Get(key) +} + +// GetAll return all config map. +func GetAll() *Map { + return DefaultClient.GetAll() +} + +// Keys return values key. +func Keys() []string { + return DefaultClient.GetAll().Keys() +} + +// Close close watcher. +func Close() error { + return DefaultClient.Close() +} diff --git a/pkg/conf/paladin/example_test.go b/pkg/conf/paladin/example_test.go new file mode 100644 index 000000000..3969bf505 --- /dev/null +++ b/pkg/conf/paladin/example_test.go @@ -0,0 +1,112 @@ +package paladin_test + +import ( + "context" + "fmt" + + "github.com/bilibili/Kratos/pkg/conf/paladin" + + "github.com/BurntSushi/toml" +) + +type exampleConf struct { + Bool bool + Int int64 + Float float64 + String string + Strings []string +} + +func (e *exampleConf) Set(text string) error { + var ec exampleConf + if err := toml.Unmarshal([]byte(text), &ec); err != nil { + return err + } + *e = ec + return nil +} + +// ExampleClient is a example client usage. +// exmaple.toml: +/* + bool = true + int = 100 + float = 100.1 + string = "text" + strings = ["a", "b", "c"] +*/ +func ExampleClient() { + if err := paladin.Init(); err != nil { + panic(err) + } + var ec exampleConf + // var setter + if err := paladin.Watch("example.toml", &ec); err != nil { + panic(err) + } + if err := paladin.Get("example.toml").UnmarshalTOML(&ec); err != nil { + panic(err) + } + // use exampleConf + // watch event key + go func() { + for event := range paladin.WatchEvent(context.TODO(), "key") { + fmt.Println(event) + } + }() +} + +// ExampleMap is a example map usage. +// exmaple.toml: +/* + bool = true + int = 100 + float = 100.1 + string = "text" + strings = ["a", "b", "c"] + + [object] + string = "text" + bool = true + int = 100 + float = 100.1 + strings = ["a", "b", "c"] +*/ +func ExampleMap() { + var ( + m paladin.TOML + strs []string + ) + // paladin toml + if err := paladin.Watch("example.toml", &m); err != nil { + panic(err) + } + // value string + s, err := m.Get("string").String() + if err != nil { + s = "default" + } + fmt.Println(s) + // value bool + b, err := m.Get("bool").Bool() + if err != nil { + b = false + } + fmt.Println(b) + // value int + i, err := m.Get("int").Int64() + if err != nil { + i = 100 + } + fmt.Println(i) + // value float + f, err := m.Get("float").Float64() + if err != nil { + f = 100.1 + } + fmt.Println(f) + // value slice + if err = m.Get("strings").Slice(&strs); err == nil { + fmt.Println(strs) + } +} diff --git a/pkg/conf/paladin/file.go b/pkg/conf/paladin/file.go new file mode 100644 index 000000000..6050cc7b7 --- /dev/null +++ b/pkg/conf/paladin/file.go @@ -0,0 +1,194 @@ +package paladin + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +const ( + defaultChSize = 10 +) + +var _ Client = &file{} + +// file is file config client. +type file struct { + values *Map + rawVal map[string]*Value + + watchChs map[string][]chan Event + mx sync.Mutex + wg sync.WaitGroup + + base string + done chan struct{} +} + +func readAllPaths(base string) ([]string, error) { + fi, err := os.Stat(base) + if err != nil { + return nil, fmt.Errorf("check local config file fail! error: %s", err) + } + // dirs or file to paths + var paths []string + if fi.IsDir() { + files, err := ioutil.ReadDir(base) + if err != nil { + return nil, fmt.Errorf("read dir %s error: %s", base, err) + } + for _, file := range files { + if !file.IsDir() { + paths = append(paths, path.Join(base, file.Name())) + } + } + } else { + paths = append(paths, base) + } + return paths, nil +} + +func loadValuesFromPaths(paths []string) (map[string]*Value, error) { + // laod config file to values + var err error + values := make(map[string]*Value, len(paths)) + for _, fpath := range paths { + if values[path.Base(fpath)], err = loadValue(fpath); err != nil { + return nil, err + } + } + return values, nil +} + +func loadValue(fpath string) (*Value, error) { + data, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + content := string(data) + return &Value{val: content, raw: content}, nil +} + +// NewFile new a config file client. +// conf = /data/conf/app/ +// conf = /data/conf/app/xxx.toml +func NewFile(base string) (Client, error) { + // paltform slash + base = filepath.FromSlash(base) + + paths, err := readAllPaths(base) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, fmt.Errorf("empty config path") + } + + rawVal, err := loadValuesFromPaths(paths) + if err != nil { + return nil, err + } + + valMap := &Map{} + valMap.Store(rawVal) + fc := &file{ + values: valMap, + rawVal: rawVal, + watchChs: make(map[string][]chan Event), + + base: base, + done: make(chan struct{}, 1), + } + + fc.wg.Add(1) + go fc.daemon() + + return fc, nil +} + +// Get return value by key. +func (f *file) Get(key string) *Value { + return f.values.Get(key) +} + +// GetAll return value map. +func (f *file) GetAll() *Map { + return f.values +} + +// WatchEvent watch multi key. +func (f *file) WatchEvent(ctx context.Context, keys ...string) <-chan Event { + f.mx.Lock() + defer f.mx.Unlock() + ch := make(chan Event, defaultChSize) + for _, key := range keys { + f.watchChs[key] = append(f.watchChs[key], ch) + } + return ch +} + +// Close close watcher. +func (f *file) Close() error { + f.done <- struct{}{} + f.wg.Wait() + return nil +} + +// file config daemon to watch file modification +func (f *file) daemon() { + defer f.wg.Done() + fswatcher, err := fsnotify.NewWatcher() + if err != nil { + log.Printf("create file watcher fail! reload function will lose efficacy error: %s", err) + return + } + if err = fswatcher.Add(f.base); err != nil { + log.Printf("create fsnotify for base path %s fail %s, reload function will lose efficacy", f.base, err) + return + } + log.Printf("start watch filepath: %s", f.base) + for event := range fswatcher.Events { + switch event.Op { + // use vim edit config will trigger rename + case fsnotify.Write, fsnotify.Create: + f.reloadFile(event.Name) + case fsnotify.Chmod: + default: + log.Printf("unsupport event %s ingored", event) + } + } +} + +func (f *file) reloadFile(name string) { + // NOTE: in some case immediately read file content after receive event + // will get old content, sleep 100ms make sure get correct content. + time.Sleep(100 * time.Millisecond) + key := filepath.Base(name) + val, err := loadValue(name) + if err != nil { + log.Printf("load file %s error: %s, skipped", name, err) + return + } + f.rawVal[key] = val + f.values.Store(f.rawVal) + + f.mx.Lock() + chs := f.watchChs[key] + f.mx.Unlock() + + for _, ch := range chs { + select { + case ch <- Event{Event: EventUpdate, Value: val.raw}: + default: + log.Printf("event channel full discard file %s update event", name) + } + } +} diff --git a/pkg/conf/paladin/file_test.go b/pkg/conf/paladin/file_test.go new file mode 100644 index 000000000..af457e97a --- /dev/null +++ b/pkg/conf/paladin/file_test.go @@ -0,0 +1,108 @@ +package paladin + +import ( + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNewFile(t *testing.T) { + // test data + path := "/tmp/test_conf/" + assert.Nil(t, os.MkdirAll(path, 0700)) + assert.Nil(t, ioutil.WriteFile(path+"test.toml", []byte(` + text = "hello" + number = 100 + slice = [1, 2, 3] + sliceStr = ["1", "2", "3"] + `), 0644)) + // test client + cli, err := NewFile(path + "test.toml") + assert.Nil(t, err) + assert.NotNil(t, cli) + // test map + m := Map{} + text, err := cli.Get("test.toml").String() + assert.Nil(t, err) + assert.Nil(t, m.Set(text), "text") + s, err := m.Get("text").String() + assert.Nil(t, err) + assert.Equal(t, s, "hello", "text") + n, err := m.Get("number").Int64() + assert.Nil(t, err) + assert.Equal(t, n, int64(100), "number") +} + +func TestNewFilePath(t *testing.T) { + // test data + path := "/tmp/test_conf/" + assert.Nil(t, os.MkdirAll(path, 0700)) + assert.Nil(t, ioutil.WriteFile(path+"test.toml", []byte(` + text = "hello" + number = 100 + `), 0644)) + assert.Nil(t, ioutil.WriteFile(path+"abc.toml", []byte(` + text = "hello" + number = 100 + `), 0644)) + // test client + cli, err := NewFile(path) + assert.Nil(t, err) + assert.NotNil(t, cli) + // test map + m := Map{} + text, err := cli.Get("test.toml").String() + assert.Nil(t, err) + assert.Nil(t, m.Set(text), "text") + s, err := m.Get("text").String() + assert.Nil(t, err, s) + assert.Equal(t, s, "hello", "text") + n, err := m.Get("number").Int64() + assert.Nil(t, err, s) + assert.Equal(t, n, int64(100), "number") +} + +func TestFileEvent(t *testing.T) { + // test data + path := "/tmp/test_conf_event/" + assert.Nil(t, os.MkdirAll(path, 0700)) + assert.Nil(t, ioutil.WriteFile(path+"test.toml", []byte(` + text = "hello" + number = 100 + `), 0644)) + assert.Nil(t, ioutil.WriteFile(path+"abc.toml", []byte(` + text = "hello" + number = 100 + `), 0644)) + // test client + cli, err := NewFile(path) + assert.Nil(t, err) + assert.NotNil(t, cli) + ch := cli.WatchEvent(context.Background(), "test.toml", "abc.toml") + time.Sleep(time.Millisecond) + ioutil.WriteFile(path+"test.toml", []byte(`hello`), 0644) + timeout := time.NewTimer(time.Second) + select { + case <-timeout.C: + t.Fatalf("run test timeout") + case ev := <-ch: + assert.Equal(t, EventUpdate, ev.Event) + assert.Equal(t, "hello", ev.Value) + } + ioutil.WriteFile(path+"abc.toml", []byte(`test`), 0644) + select { + case <-timeout.C: + t.Fatalf("run test timeout") + case ev := <-ch: + assert.Equal(t, EventUpdate, ev.Event) + assert.Equal(t, "test", ev.Value) + } + content1, _ := cli.Get("test.toml").String() + assert.Equal(t, "hello", content1) + content2, _ := cli.Get("abc.toml").String() + assert.Equal(t, "test", content2) +} diff --git a/pkg/conf/paladin/helper.go b/pkg/conf/paladin/helper.go new file mode 100644 index 000000000..115d438fc --- /dev/null +++ b/pkg/conf/paladin/helper.go @@ -0,0 +1,76 @@ +package paladin + +import "time" + +// Bool return bool value. +func Bool(v *Value, def bool) bool { + b, err := v.Bool() + if err != nil { + return def + } + return b +} + +// Int return int value. +func Int(v *Value, def int) int { + i, err := v.Int() + if err != nil { + return def + } + return i +} + +// Int32 return int32 value. +func Int32(v *Value, def int32) int32 { + i, err := v.Int32() + if err != nil { + return def + } + return i +} + +// Int64 return int64 value. +func Int64(v *Value, def int64) int64 { + i, err := v.Int64() + if err != nil { + return def + } + return i +} + +// Float32 return float32 value. +func Float32(v *Value, def float32) float32 { + f, err := v.Float32() + if err != nil { + return def + } + return f +} + +// Float64 return float32 value. +func Float64(v *Value, def float64) float64 { + f, err := v.Float64() + if err != nil { + return def + } + return f +} + +// String return string value. +func String(v *Value, def string) string { + s, err := v.String() + if err != nil { + return def + } + return s +} + +// Duration parses a duration string. A duration string is a possibly signed sequence of decimal numbers +// each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +func Duration(v *Value, def time.Duration) time.Duration { + dur, err := v.Duration() + if err != nil { + return def + } + return dur +} diff --git a/pkg/conf/paladin/helper_test.go b/pkg/conf/paladin/helper_test.go new file mode 100644 index 000000000..aaa17d4d4 --- /dev/null +++ b/pkg/conf/paladin/helper_test.go @@ -0,0 +1,286 @@ +package paladin + +import ( + "testing" + "time" +) + +func TestBool(t *testing.T) { + type args struct { + v *Value + def bool + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "ok", + args: args{v: &Value{val: true}}, + want: true, + }, + { + name: "fail", + args: args{v: &Value{}}, + want: false, + }, + { + name: "default", + args: args{v: &Value{}, def: true}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Bool(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Bool() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInt(t *testing.T) { + type args struct { + v *Value + def int + } + tests := []struct { + name string + args args + want int + }{ + { + name: "ok", + args: args{v: &Value{val: int64(2233)}}, + want: 2233, + }, + { + name: "fail", + args: args{v: &Value{}}, + want: 0, + }, + { + name: "default", + args: args{v: &Value{}, def: 2233}, + want: 2233, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Int(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Int() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInt32(t *testing.T) { + type args struct { + v *Value + def int32 + } + tests := []struct { + name string + args args + want int32 + }{ + { + name: "ok", + args: args{v: &Value{val: int64(2233)}}, + want: 2233, + }, + { + name: "fail", + args: args{v: &Value{}}, + want: 0, + }, + { + name: "default", + args: args{v: &Value{}, def: 2233}, + want: 2233, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Int32(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Int32() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInt64(t *testing.T) { + type args struct { + v *Value + def int64 + } + tests := []struct { + name string + args args + want int64 + }{ + { + name: "ok", + args: args{v: &Value{val: int64(2233)}}, + want: 2233, + }, + { + name: "fail", + args: args{v: &Value{}}, + want: 0, + }, + { + name: "default", + args: args{v: &Value{}, def: 2233}, + want: 2233, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Int64(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Int64() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFloat32(t *testing.T) { + type args struct { + v *Value + def float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "ok", + args: args{v: &Value{val: float64(2233)}}, + want: float32(2233), + }, + { + name: "fail", + args: args{v: &Value{}}, + want: 0, + }, + { + name: "default", + args: args{v: &Value{}, def: float32(2233)}, + want: float32(2233), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Float32(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Float32() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFloat64(t *testing.T) { + type args struct { + v *Value + def float64 + } + tests := []struct { + name string + args args + want float64 + }{ + { + name: "ok", + args: args{v: &Value{val: float64(2233)}}, + want: float64(2233), + }, + { + name: "fail", + args: args{v: &Value{}}, + want: 0, + }, + { + name: "default", + args: args{v: &Value{}, def: float64(2233)}, + want: float64(2233), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Float64(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Float64() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestString(t *testing.T) { + type args struct { + v *Value + def string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "ok", + args: args{v: &Value{val: "test"}}, + want: "test", + }, + { + name: "fail", + args: args{v: &Value{}}, + want: "", + }, + { + name: "default", + args: args{v: &Value{}, def: "test"}, + want: "test", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := String(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDuration(t *testing.T) { + type args struct { + v *Value + def time.Duration + } + tests := []struct { + name string + args args + want time.Duration + }{ + { + name: "ok", + args: args{v: &Value{val: "1s"}}, + want: time.Second, + }, + { + name: "fail", + args: args{v: &Value{}}, + want: 0, + }, + { + name: "default", + args: args{v: &Value{}, def: time.Second}, + want: time.Second, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Duration(tt.args.v, tt.args.def); got != tt.want { + t.Errorf("Duration() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/conf/paladin/map.go b/pkg/conf/paladin/map.go new file mode 100644 index 000000000..f9de70a1b --- /dev/null +++ b/pkg/conf/paladin/map.go @@ -0,0 +1,55 @@ +package paladin + +import ( + "strings" + "sync/atomic" +) + +// keyNamed key naming to lower case. +func keyNamed(key string) string { + return strings.ToLower(key) +} + +// Map is config map, key(filename) -> value(file). +type Map struct { + values atomic.Value +} + +// Store sets the value of the Value to values map. +func (m *Map) Store(values map[string]*Value) { + dst := make(map[string]*Value, len(values)) + for k, v := range values { + dst[keyNamed(k)] = v + } + m.values.Store(dst) +} + +// Load returns the value set by the most recent Store. +func (m *Map) Load() map[string]*Value { + return m.values.Load().(map[string]*Value) +} + +// Exist check if values map exist a key. +func (m *Map) Exist(key string) bool { + _, ok := m.Load()[keyNamed(key)] + return ok +} + +// Get return get value by key. +func (m *Map) Get(key string) *Value { + v, ok := m.Load()[keyNamed(key)] + if ok { + return v + } + return &Value{} +} + +// Keys return map keys. +func (m *Map) Keys() []string { + values := m.Load() + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + return keys +} diff --git a/pkg/conf/paladin/map_test.go b/pkg/conf/paladin/map_test.go new file mode 100644 index 000000000..6d98ea8a7 --- /dev/null +++ b/pkg/conf/paladin/map_test.go @@ -0,0 +1,94 @@ +package paladin_test + +import ( + "testing" + + "github.com/bilibili/Kratos/pkg/conf/paladin" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/assert" +) + +type fruit struct { + Fruit []struct { + Name string + } +} + +func (f *fruit) Set(text string) error { + return toml.Unmarshal([]byte(text), f) +} + +func TestMap(t *testing.T) { + s := ` + # kv + text = "hello" + number = 100 + point = 100.1 + boolean = true + KeyCase = "test" + + # slice + numbers = [1, 2, 3] + strings = ["a", "b", "c"] + empty = [] + [[fruit]] + name = "apple" + [[fruit]] + name = "banana" + + # table + [database] + server = "192.168.1.1" + connection_max = 5000 + enabled = true + + [pool] + [pool.breaker] + xxx = "xxx" + ` + m := paladin.Map{} + assert.Nil(t, m.Set(s), s) + str, err := m.Get("text").String() + assert.Nil(t, err) + assert.Equal(t, str, "hello", "text") + n, err := m.Get("number").Int64() + assert.Nil(t, err) + assert.Equal(t, n, int64(100), "number") + p, err := m.Get("point").Float64() + assert.Nil(t, err) + assert.Equal(t, p, 100.1, "point") + b, err := m.Get("boolean").Bool() + assert.Nil(t, err) + assert.Equal(t, b, true, "boolean") + // key lower case + lb, err := m.Get("Boolean").Bool() + assert.Nil(t, err) + assert.Equal(t, lb, true, "boolean") + lt, err := m.Get("KeyCase").String() + assert.Nil(t, err) + assert.Equal(t, lt, "test", "key case") + var sliceInt []int64 + err = m.Get("numbers").Slice(&sliceInt) + assert.Nil(t, err) + assert.Equal(t, sliceInt, []int64{1, 2, 3}) + var sliceStr []string + err = m.Get("strings").Slice(&sliceStr) + assert.Nil(t, err) + assert.Equal(t, []string{"a", "b", "c"}, sliceStr) + err = m.Get("strings").Slice(&sliceStr) + assert.Nil(t, err) + assert.Equal(t, []string{"a", "b", "c"}, sliceStr) + // errors + err = m.Get("strings").Slice(sliceInt) + assert.NotNil(t, err) + err = m.Get("strings").Slice(&sliceInt) + assert.NotNil(t, err) + var obj struct { + Name string + } + err = m.Get("strings").Slice(obj) + assert.NotNil(t, err) + err = m.Get("strings").Slice(&obj) + assert.NotNil(t, err) +} diff --git a/pkg/conf/paladin/mock.go b/pkg/conf/paladin/mock.go new file mode 100644 index 000000000..1792d95bd --- /dev/null +++ b/pkg/conf/paladin/mock.go @@ -0,0 +1,40 @@ +package paladin + +import ( + "context" +) + +var _ Client = &Mock{} + +// Mock is Mock config client. +type Mock struct { + C chan Event + *Map +} + +// NewMock new a config mock client. +func NewMock(vs map[string]string) *Mock { + values := make(map[string]*Value, len(vs)) + for k, v := range vs { + values[k] = &Value{val: v, raw: v} + } + m := new(Map) + m.Store(values) + return &Mock{Map: m, C: make(chan Event)} +} + +// GetAll return value map. +func (m *Mock) GetAll() *Map { + return m.Map +} + +// WatchEvent watch multi key. +func (m *Mock) WatchEvent(ctx context.Context, key ...string) <-chan Event { + return m.C +} + +// Close close watcher. +func (m *Mock) Close() error { + close(m.C) + return nil +} diff --git a/pkg/conf/paladin/mock_test.go b/pkg/conf/paladin/mock_test.go new file mode 100644 index 000000000..0d088ed2c --- /dev/null +++ b/pkg/conf/paladin/mock_test.go @@ -0,0 +1,37 @@ +package paladin_test + +import ( + "testing" + + "github.com/bilibili/Kratos/pkg/conf/paladin" + + "github.com/stretchr/testify/assert" +) + +func TestMock(t *testing.T) { + cs := map[string]string{ + "key_toml": ` + key_bool = true + key_int = 100 + key_float = 100.1 + key_string = "text" + `, + } + cli := paladin.NewMock(cs) + // test vlaue + var m paladin.TOML + err := cli.Get("key_toml").Unmarshal(&m) + assert.Nil(t, err) + b, err := m.Get("key_bool").Bool() + assert.Nil(t, err) + assert.Equal(t, b, true) + i, err := m.Get("key_int").Int64() + assert.Nil(t, err) + assert.Equal(t, i, int64(100)) + f, err := m.Get("key_float").Float64() + assert.Nil(t, err) + assert.Equal(t, f, float64(100.1)) + s, err := m.Get("key_string").String() + assert.Nil(t, err) + assert.Equal(t, s, "text") +} diff --git a/pkg/conf/paladin/toml.go b/pkg/conf/paladin/toml.go new file mode 100644 index 000000000..87f92771c --- /dev/null +++ b/pkg/conf/paladin/toml.go @@ -0,0 +1,73 @@ +package paladin + +import ( + "bytes" + "reflect" + "strconv" + + "github.com/BurntSushi/toml" + "github.com/pkg/errors" +) + +// TOML is toml map. +type TOML = Map + +// Set set the map by value. +func (m *TOML) Set(text string) error { + if err := m.UnmarshalText([]byte(text)); err != nil { + return err + } + return nil +} + +// UnmarshalText implemented toml. +func (m *TOML) UnmarshalText(text []byte) error { + raws := map[string]interface{}{} + if err := toml.Unmarshal(text, &raws); err != nil { + return err + } + values := map[string]*Value{} + for k, v := range raws { + k = keyNamed(k) + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Map: + buf := bytes.NewBuffer(nil) + err := toml.NewEncoder(buf).Encode(v) + // b, err := toml.Marshal(v) + if err != nil { + return err + } + // NOTE: value is map[string]interface{} + values[k] = &Value{val: v, raw: buf.String()} + case reflect.Slice: + raw := map[string]interface{}{ + k: v, + } + buf := bytes.NewBuffer(nil) + err := toml.NewEncoder(buf).Encode(raw) + // b, err := toml.Marshal(raw) + if err != nil { + return err + } + // NOTE: value is []interface{} + values[k] = &Value{val: v, raw: buf.String()} + case reflect.Bool: + b := v.(bool) + values[k] = &Value{val: b, raw: strconv.FormatBool(b)} + case reflect.Int64: + i := v.(int64) + values[k] = &Value{val: i, raw: strconv.FormatInt(i, 10)} + case reflect.Float64: + f := v.(float64) + values[k] = &Value{val: f, raw: strconv.FormatFloat(f, 'f', -1, 64)} + case reflect.String: + s := v.(string) + values[k] = &Value{val: s, raw: s} + default: + return errors.Errorf("UnmarshalTOML: unknown kind(%v)", rv.Kind()) + } + } + m.Store(values) + return nil +} diff --git a/pkg/conf/paladin/value.go b/pkg/conf/paladin/value.go new file mode 100644 index 000000000..ee6dd3db2 --- /dev/null +++ b/pkg/conf/paladin/value.go @@ -0,0 +1,157 @@ +package paladin + +import ( + "encoding" + "reflect" + "time" + + "github.com/BurntSushi/toml" + "github.com/pkg/errors" +) + +// ErrNotExist value key not exist. +var ( + ErrNotExist = errors.New("paladin: value key not exist") + ErrTypeAssertion = errors.New("paladin: value type assertion no match") + ErrDifferentTypes = errors.New("paladin: value different types") +) + +// Value is config value, maybe a json/toml/ini/string file. +type Value struct { + val interface{} + slice interface{} + raw string +} + +// Bool return bool value. +func (v *Value) Bool() (bool, error) { + if v.val == nil { + return false, ErrNotExist + } + b, ok := v.val.(bool) + if !ok { + return false, ErrTypeAssertion + } + return b, nil +} + +// Int return int value. +func (v *Value) Int() (int, error) { + i, err := v.Int64() + return int(i), err +} + +// Int32 return int32 value. +func (v *Value) Int32() (int32, error) { + i, err := v.Int64() + return int32(i), err +} + +// Int64 return int64 value. +func (v *Value) Int64() (int64, error) { + if v.val == nil { + return 0, ErrNotExist + } + i, ok := v.val.(int64) + if !ok { + return 0, ErrTypeAssertion + } + return i, nil +} + +// Float32 return float32 value. +func (v *Value) Float32() (float32, error) { + f, err := v.Float64() + if err != nil { + return 0.0, err + } + return float32(f), nil +} + +// Float64 return float64 value. +func (v *Value) Float64() (float64, error) { + if v.val == nil { + return 0.0, ErrNotExist + } + f, ok := v.val.(float64) + if !ok { + return 0.0, ErrTypeAssertion + } + return f, nil +} + +// String return string value. +func (v *Value) String() (string, error) { + if v.val == nil { + return "", ErrNotExist + } + s, ok := v.val.(string) + if !ok { + return "", ErrTypeAssertion + } + return s, nil +} + +// Duration parses a duration string. A duration string is a possibly signed sequence of decimal numbers +// each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +func (v *Value) Duration() (time.Duration, error) { + s, err := v.String() + if err != nil { + return time.Duration(0), err + } + return time.ParseDuration(s) +} + +// Raw return raw value. +func (v *Value) Raw() (string, error) { + if v.val == nil { + return "", ErrNotExist + } + return v.raw, nil +} + +// Slice scan a slcie interface, if slice has element it will be discard. +func (v *Value) Slice(dst interface{}) error { + // NOTE: val is []interface{}, slice is []type + if v.val == nil { + return ErrNotExist + } + rv := reflect.ValueOf(dst) + if rv.Kind() != reflect.Ptr || rv.Elem().Kind() != reflect.Slice { + return ErrDifferentTypes + } + el := rv.Elem() + // reset slice len to 0. + el.SetLen(0) + kind := el.Type().Elem().Kind() + src, ok := v.val.([]interface{}) + if !ok { + return ErrDifferentTypes + } + for _, s := range src { + if reflect.TypeOf(s).Kind() != kind { + return ErrTypeAssertion + } + el = reflect.Append(el, reflect.ValueOf(s)) + } + rv.Elem().Set(el) + return nil +} + +// Unmarshal is the interface implemented by an object that can unmarshal a textual representation of itself. +func (v *Value) Unmarshal(un encoding.TextUnmarshaler) error { + text, err := v.Raw() + if err != nil { + return err + } + return un.UnmarshalText([]byte(text)) +} + +// UnmarshalTOML unmarhsal toml to struct. +func (v *Value) UnmarshalTOML(dst interface{}) error { + text, err := v.Raw() + if err != nil { + return err + } + return toml.Unmarshal([]byte(text), dst) +} diff --git a/pkg/conf/paladin/value_test.go b/pkg/conf/paladin/value_test.go new file mode 100644 index 000000000..e1a742da8 --- /dev/null +++ b/pkg/conf/paladin/value_test.go @@ -0,0 +1,206 @@ +package paladin + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type testUnmarshler struct { + Text string + Int int +} + +func TestValueUnmarshal(t *testing.T) { + s := ` + int = 100 + text = "hello" + ` + v := Value{val: s, raw: s} + obj := new(testUnmarshler) + assert.Nil(t, v.UnmarshalTOML(obj)) + // error + v = Value{val: nil, raw: ""} + assert.NotNil(t, v.UnmarshalTOML(obj)) +} + +func TestValue(t *testing.T) { + var tests = []struct { + in interface{} + out interface{} + }{ + { + "text", + "text", + }, + { + time.Duration(time.Second * 10), + "10s", + }, + { + int64(100), + int64(100), + }, + { + float64(100.1), + float64(100.1), + }, + { + true, + true, + }, + { + nil, + nil, + }, + } + for _, test := range tests { + t.Run(fmt.Sprint(test.in), func(t *testing.T) { + v := Value{val: test.in, raw: fmt.Sprint(test.in)} + switch test.in.(type) { + case nil: + s, err := v.String() + assert.NotNil(t, err) + assert.Equal(t, s, "", test.in) + i, err := v.Int64() + assert.NotNil(t, err) + assert.Equal(t, i, int64(0), test.in) + f, err := v.Float64() + assert.NotNil(t, err) + assert.Equal(t, f, float64(0.0), test.in) + b, err := v.Bool() + assert.NotNil(t, err) + assert.Equal(t, b, false, test.in) + case string: + val, err := v.String() + assert.Nil(t, err) + assert.Equal(t, val, test.out.(string), test.in) + case int64: + val, err := v.Int() + assert.Nil(t, err) + assert.Equal(t, val, int(test.out.(int64)), test.in) + val32, err := v.Int32() + assert.Nil(t, err) + assert.Equal(t, val32, int32(test.out.(int64)), test.in) + val64, err := v.Int64() + assert.Nil(t, err) + assert.Equal(t, val64, test.out.(int64), test.in) + case float64: + val32, err := v.Float32() + assert.Nil(t, err) + assert.Equal(t, val32, float32(test.out.(float64)), test.in) + val64, err := v.Float64() + assert.Nil(t, err) + assert.Equal(t, val64, test.out.(float64), test.in) + case bool: + val, err := v.Bool() + assert.Nil(t, err) + assert.Equal(t, val, test.out.(bool), test.in) + case time.Duration: + v.val = test.out + val, err := v.Duration() + assert.Nil(t, err) + assert.Equal(t, val, test.in.(time.Duration), test.out) + } + }) + } +} + +func TestValueSlice(t *testing.T) { + var tests = []struct { + in interface{} + out interface{} + }{ + { + nil, + nil, + }, + { + []interface{}{"a", "b", "c"}, + []string{"a", "b", "c"}, + }, + { + []interface{}{1, 2, 3}, + []int64{1, 2, 3}, + }, + { + []interface{}{1.1, 1.2, 1.3}, + []float64{1.1, 1.2, 1.3}, + }, + { + []interface{}{true, false, true}, + []bool{true, false, true}, + }, + } + for _, test := range tests { + t.Run(fmt.Sprint(test.in), func(t *testing.T) { + v := Value{val: test.in, raw: fmt.Sprint(test.in)} + switch test.in.(type) { + case nil: + var s []string + assert.NotNil(t, v.Slice(&s)) + case []string: + var s []string + assert.Nil(t, v.Slice(&s)) + assert.Equal(t, s, test.out) + case []int64: + var s []int64 + assert.Nil(t, v.Slice(&s)) + assert.Equal(t, s, test.out) + case []float64: + var s []float64 + assert.Nil(t, v.Slice(&s)) + assert.Equal(t, s, test.out) + case []bool: + var s []bool + assert.Nil(t, v.Slice(&s)) + assert.Equal(t, s, test.out) + } + }) + } +} + +func BenchmarkValueInt(b *testing.B) { + v := &Value{val: int64(100), raw: "100"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + v.Int64() + } + }) +} +func BenchmarkValueFloat(b *testing.B) { + v := &Value{val: float64(100.1), raw: "100.1"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + v.Float64() + } + }) +} +func BenchmarkValueBool(b *testing.B) { + v := &Value{val: true, raw: "true"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + v.Bool() + } + }) +} +func BenchmarkValueString(b *testing.B) { + v := &Value{val: "text", raw: "text"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + v.String() + } + }) +} + +func BenchmarkValueSlice(b *testing.B) { + v := &Value{val: []interface{}{1, 2, 3}, raw: "100"} + b.RunParallel(func(pb *testing.PB) { + var slice []int64 + for pb.Next() { + v.Slice(&slice) + } + }) +}