From a8692e7ddefc2fdc0b5dc3473728c2ede6adeea4 Mon Sep 17 00:00:00 2001 From: Windfarer Date: Wed, 10 Nov 2021 20:10:28 +0800 Subject: [PATCH] feat(contrib/config): use key file extension as format & config load log (#1619) * add format --- config/config.go | 3 +++ config/config_test.go | 1 + contrib/config/consul/config.go | 8 +++--- contrib/config/consul/config_test.go | 32 ++++++++++++++++++++++++ contrib/config/consul/go.mod | 1 + contrib/config/etcd/config.go | 9 ++++--- contrib/config/etcd/config_test.go | 37 ++++++++++++++++++++++++++++ contrib/config/etcd/go.mod | 2 ++ contrib/config/kubernetes/config.go | 9 +++++-- contrib/config/nacos/config.go | 15 +++++++---- examples/config/t/a.yaml | 5 ++++ examples/config/t/b.yaml | 6 +++++ 12 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 examples/config/t/a.yaml create mode 100644 examples/config/t/b.yaml diff --git a/config/config.go b/config/config.go index c3ce750ba..5fed286bc 100644 --- a/config/config.go +++ b/config/config.go @@ -103,6 +103,9 @@ func (c *config) Load() error { if err != nil { return err } + for _, v := range kvs { + c.log.Infof("config loaded: %s format: %s", v.Key, v.Format) + } if err = c.reader.Merge(kvs...); err != nil { c.log.Errorf("failed to merge config source: %v", err) return err diff --git a/config/config_test.go b/config/config_test.go index 5f1f9474e..27f453c12 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -138,6 +138,7 @@ func TestConfig(t *testing.T) { cf := &config{} cf.opts = opts cf.reader = newReader(opts) + cf.log = log.NewHelper(opts.logger) err = cf.Load() assert.Nil(t, err) diff --git a/contrib/config/consul/config.go b/contrib/config/consul/config.go index ce4317db3..8025ec967 100644 --- a/contrib/config/consul/config.go +++ b/contrib/config/consul/config.go @@ -3,6 +3,7 @@ package consul import ( "context" "errors" + "path/filepath" "strings" "github.com/go-kratos/kratos/v2/config" @@ -67,12 +68,13 @@ func (s *source) Load() ([]*config.KeyValue, error) { if !strings.HasSuffix(s.options.path, "/") { pathPrefix = pathPrefix + "/" } - kvs := make([]*config.KeyValue, 0) for _, item := range kv { + k := strings.TrimPrefix(item.Key, pathPrefix) kvs = append(kvs, &config.KeyValue{ - Key: item.Key[len(pathPrefix):], - Value: item.Value, + Key: k, + Value: item.Value, + Format: strings.TrimPrefix(filepath.Ext(k), "."), }) } return kvs, nil diff --git a/contrib/config/consul/config_test.go b/contrib/config/consul/config_test.go index b47848fee..4f07aaa5c 100644 --- a/contrib/config/consul/config_test.go +++ b/contrib/config/consul/config_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/assert" ) const testPath = "kratos/test/config" @@ -60,3 +61,34 @@ func TestConfig(t *testing.T) { t.Error(err) } } + +func TestExtToFormat(t *testing.T) { + client, err := api.NewClient(&api.Config{ + Address: "127.0.0.1:8500", + }) + if err != nil { + t.Fatal(err) + } + tp := "kratos/test/ext" + tn := "a.bird.json" + tk := tp + "/" + tn + tc := `{"a":1}` + if _, err = client.KV().Put(&api.KVPair{Key: tk, Value: []byte(tc)}, nil); err != nil { + t.Fatal(err) + } + + source, err := New(client, WithPath(tp)) + if err != nil { + t.Fatal(err) + } + + kvs, err := source.Load() + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 1, len(kvs)) + assert.Equal(t, tn, kvs[0].Key) + assert.Equal(t, tc, string(kvs[0].Value)) + assert.Equal(t, "json", kvs[0].Format) +} diff --git a/contrib/config/consul/go.mod b/contrib/config/consul/go.mod index d2f0f0954..7bbed1a9f 100644 --- a/contrib/config/consul/go.mod +++ b/contrib/config/consul/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/go-kratos/kratos/v2 v2.0.5 github.com/hashicorp/consul/api v1.10.0 + github.com/stretchr/testify v1.7.0 ) replace github.com/go-kratos/kratos/v2 => ../../../ diff --git a/contrib/config/etcd/config.go b/contrib/config/etcd/config.go index a18f3dc20..ea027c2a9 100644 --- a/contrib/config/etcd/config.go +++ b/contrib/config/etcd/config.go @@ -3,6 +3,8 @@ package etcd import ( "context" "errors" + "path/filepath" + "strings" "github.com/go-kratos/kratos/v2/config" clientv3 "go.etcd.io/etcd/client/v3" @@ -75,12 +77,13 @@ func (s *source) Load() ([]*config.KeyValue, error) { if err != nil { return nil, err } - kvs := make([]*config.KeyValue, 0) for _, item := range rsp.Kvs { + k := string(item.Key) kvs = append(kvs, &config.KeyValue{ - Key: string(item.Key), - Value: item.Value, + Key: k, + Value: item.Value, + Format: strings.TrimPrefix(filepath.Ext(k), "."), }) } return kvs, nil diff --git a/contrib/config/etcd/config_test.go b/contrib/config/etcd/config_test.go index c83eb6903..f0205738d 100644 --- a/contrib/config/etcd/config_test.go +++ b/contrib/config/etcd/config_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" ) @@ -64,3 +65,39 @@ func TestConfig(t *testing.T) { t.Error(err) } } + +func TestExtToFormat(t *testing.T) { + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{"127.0.0.1:2379"}, + DialTimeout: time.Second, DialOptions: []grpc.DialOption{grpc.WithBlock()}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = client.Close() + }() + + tp := "/kratos/test/ext" + tn := "a.bird.json" + tk := tp + "/" + tn + tc := `{"a":1}` + if _, err = client.Put(context.Background(), tk, tc); err != nil { + t.Fatal(err) + } + + source, err := New(client, WithPath(tp), WithPrefix(true)) + if err != nil { + t.Fatal(err) + } + + kvs, err := source.Load() + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 1, len(kvs)) + assert.Equal(t, tk, kvs[0].Key) + assert.Equal(t, tc, string(kvs[0].Value)) + assert.Equal(t, "json", kvs[0].Format) +} diff --git a/contrib/config/etcd/go.mod b/contrib/config/etcd/go.mod index a94c8a736..047595ed3 100644 --- a/contrib/config/etcd/go.mod +++ b/contrib/config/etcd/go.mod @@ -3,7 +3,9 @@ module github.com/go-kratos/kratos/contrib/config/etcd/v2 go 1.16 require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-kratos/kratos/v2 v2.1.0 + github.com/stretchr/testify v1.7.0 go.etcd.io/etcd/client/v3 v3.5.0 google.golang.org/grpc v1.40.0 ) diff --git a/contrib/config/kubernetes/config.go b/contrib/config/kubernetes/config.go index 2e66fbc71..c66b837f5 100644 --- a/contrib/config/kubernetes/config.go +++ b/contrib/config/kubernetes/config.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "path/filepath" + "strings" "github.com/go-kratos/kratos/v2/config" v1 "k8s.io/api/core/v1" @@ -116,9 +118,12 @@ func (k *kube) load() (kvs []*config.KeyValue, err error) { func (k *kube) configMap(cm v1.ConfigMap) (kvs []*config.KeyValue) { for name, val := range cm.Data { + k := fmt.Sprintf("%s/%s/%s", k.opts.Namespace, cm.Name, name) + kvs = append(kvs, &config.KeyValue{ - Key: fmt.Sprintf("%s/%s/%s", k.opts.Namespace, cm.Name, name), - Value: []byte(val), + Key: k, + Value: []byte(val), + Format: strings.TrimPrefix(filepath.Ext(k), "."), }) } return kvs diff --git a/contrib/config/nacos/config.go b/contrib/config/nacos/config.go index 3d5cbf4b1..ee07e4110 100644 --- a/contrib/config/nacos/config.go +++ b/contrib/config/nacos/config.go @@ -2,6 +2,8 @@ package config import ( "context" + "path/filepath" + "strings" "time" "github.com/go-kratos/kratos/v2/config" @@ -89,11 +91,12 @@ func (c *Config) Load() ([]*config.KeyValue, error) { if err != nil { return nil, err } - + k := c.opts.dataID return []*config.KeyValue{ { - Key: c.opts.dataID, - Value: []byte(content), + Key: k, + Value: []byte(content), + Format: strings.TrimPrefix(filepath.Ext(k), "."), }, }, nil } @@ -144,10 +147,12 @@ func (w *Watcher) Next() ([]*config.KeyValue, error) { case <-w.Context.Done(): return nil, nil case content := <-w.content: + k := w.dataID return []*config.KeyValue{ { - Key: w.dataID, - Value: []byte(content), + Key: k, + Value: []byte(content), + Format: strings.TrimPrefix(filepath.Ext(k), "."), }, }, nil } diff --git a/examples/config/t/a.yaml b/examples/config/t/a.yaml new file mode 100644 index 000000000..91a2ebf73 --- /dev/null +++ b/examples/config/t/a.yaml @@ -0,0 +1,5 @@ +foo: + baz: "2" + biu: "example" +hello: + a: b \ No newline at end of file diff --git a/examples/config/t/b.yaml b/examples/config/t/b.yaml new file mode 100644 index 000000000..d51ff07a7 --- /dev/null +++ b/examples/config/t/b.yaml @@ -0,0 +1,6 @@ +foo: + bar: 3 + baz: aaaa +hey: + good: bad + qux: quux \ No newline at end of file