feat(contrib/config): use key file extension as format & config load log (#1619)

* add format

config/config.go

@@ -103,6 +103,9 @@ func (c *config) Load() error {
 		if err != nil {
 			return err
 		}
+		for _, v := range kvs {
+			c.log.Infof("config loaded: %s format: %s", v.Key, v.Format)
+		}
 		if err = c.reader.Merge(kvs...); err != nil {
 			c.log.Errorf("failed to merge config source: %v", err)
 			return err

config/config_test.go

@@ -138,6 +138,7 @@ func TestConfig(t *testing.T) {
 	cf := &config{}
 	cf.opts = opts
 	cf.reader = newReader(opts)
+	cf.log = log.NewHelper(opts.logger)
 
 	err = cf.Load()
 	assert.Nil(t, err)

contrib/config/consul/config.go

@@ -3,6 +3,7 @@ package consul
 import (
 	"context"
 	"errors"
+	"path/filepath"
 	"strings"
 
 	"github.com/go-kratos/kratos/v2/config"
@@ -67,12 +68,13 @@ func (s *source) Load() ([]*config.KeyValue, error) {
 	if !strings.HasSuffix(s.options.path, "/") {
 		pathPrefix = pathPrefix + "/"
 	}
-
 	kvs := make([]*config.KeyValue, 0)
 	for _, item := range kv {
+		k := strings.TrimPrefix(item.Key, pathPrefix)
 		kvs = append(kvs, &config.KeyValue{
-			Key:   item.Key[len(pathPrefix):],
+			Key:    k,
-			Value: item.Value,
+			Value:  item.Value,
+			Format: strings.TrimPrefix(filepath.Ext(k), "."),
 		})
 	}
 	return kvs, nil

contrib/config/consul/config_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul/api"
+	"github.com/stretchr/testify/assert"
 )
 
 const testPath = "kratos/test/config"
@@ -60,3 +61,34 @@ func TestConfig(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestExtToFormat(t *testing.T) {
+	client, err := api.NewClient(&api.Config{
+		Address: "127.0.0.1:8500",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	tp := "kratos/test/ext"
+	tn := "a.bird.json"
+	tk := tp + "/" + tn
+	tc := `{"a":1}`
+	if _, err = client.KV().Put(&api.KVPair{Key: tk, Value: []byte(tc)}, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	source, err := New(client, WithPath(tp))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	kvs, err := source.Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, 1, len(kvs))
+	assert.Equal(t, tn, kvs[0].Key)
+	assert.Equal(t, tc, string(kvs[0].Value))
+	assert.Equal(t, "json", kvs[0].Format)
+}

contrib/config/consul/go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/go-kratos/kratos/v2 v2.0.5
 	github.com/hashicorp/consul/api v1.10.0
+	github.com/stretchr/testify v1.7.0
 )
 
 replace github.com/go-kratos/kratos/v2 => ../../../

contrib/config/etcd/config.go

@@ -3,6 +3,8 @@ package etcd
 import (
 	"context"
 	"errors"
+	"path/filepath"
+	"strings"
 
 	"github.com/go-kratos/kratos/v2/config"
 	clientv3 "go.etcd.io/etcd/client/v3"
@@ -75,12 +77,13 @@ func (s *source) Load() ([]*config.KeyValue, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	kvs := make([]*config.KeyValue, 0)
 	for _, item := range rsp.Kvs {
+		k := string(item.Key)
 		kvs = append(kvs, &config.KeyValue{
-			Key:   string(item.Key),
+			Key:    k,
-			Value: item.Value,
+			Value:  item.Value,
+			Format: strings.TrimPrefix(filepath.Ext(k), "."),
 		})
 	}
 	return kvs, nil

contrib/config/etcd/config_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"google.golang.org/grpc"
 )
@@ -64,3 +65,39 @@ func TestConfig(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestExtToFormat(t *testing.T) {
+	client, err := clientv3.New(clientv3.Config{
+		Endpoints:   []string{"127.0.0.1:2379"},
+		DialTimeout: time.Second, DialOptions: []grpc.DialOption{grpc.WithBlock()},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = client.Close()
+	}()
+
+	tp := "/kratos/test/ext"
+	tn := "a.bird.json"
+	tk := tp + "/" + tn
+	tc := `{"a":1}`
+	if _, err = client.Put(context.Background(), tk, tc); err != nil {
+		t.Fatal(err)
+	}
+
+	source, err := New(client, WithPath(tp), WithPrefix(true))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	kvs, err := source.Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, 1, len(kvs))
+	assert.Equal(t, tk, kvs[0].Key)
+	assert.Equal(t, tc, string(kvs[0].Value))
+	assert.Equal(t, "json", kvs[0].Format)
+}

contrib/config/etcd/go.mod

@@ -3,7 +3,9 @@ module github.com/go-kratos/kratos/contrib/config/etcd/v2
 go 1.16
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-kratos/kratos/v2 v2.1.0
+	github.com/stretchr/testify v1.7.0
 	go.etcd.io/etcd/client/v3 v3.5.0
 	google.golang.org/grpc v1.40.0
 )

contrib/config/kubernetes/config.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path/filepath"
+	"strings"
 
 	"github.com/go-kratos/kratos/v2/config"
 	v1 "k8s.io/api/core/v1"
@@ -116,9 +118,12 @@ func (k *kube) load() (kvs []*config.KeyValue, err error) {
 
 func (k *kube) configMap(cm v1.ConfigMap) (kvs []*config.KeyValue) {
 	for name, val := range cm.Data {
+		k := fmt.Sprintf("%s/%s/%s", k.opts.Namespace, cm.Name, name)
+
 		kvs = append(kvs, &config.KeyValue{
-			Key:   fmt.Sprintf("%s/%s/%s", k.opts.Namespace, cm.Name, name),
+			Key:    k,
-			Value: []byte(val),
+			Value:  []byte(val),
+			Format: strings.TrimPrefix(filepath.Ext(k), "."),
 		})
 	}
 	return kvs

contrib/config/nacos/config.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"context"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/go-kratos/kratos/v2/config"
@@ -89,11 +91,12 @@ func (c *Config) Load() ([]*config.KeyValue, error) {
 	if err != nil {
 		return nil, err
 	}
-
+	k := c.opts.dataID
 	return []*config.KeyValue{
 		{
-			Key:   c.opts.dataID,
+			Key:    k,
-			Value: []byte(content),
+			Value:  []byte(content),
+			Format: strings.TrimPrefix(filepath.Ext(k), "."),
 		},
 	}, nil
 }
@@ -144,10 +147,12 @@ func (w *Watcher) Next() ([]*config.KeyValue, error) {
 	case <-w.Context.Done():
 		return nil, nil
 	case content := <-w.content:
+		k := w.dataID
 		return []*config.KeyValue{
 			{
-				Key:   w.dataID,
+				Key:    k,
-				Value: []byte(content),
+				Value:  []byte(content),
+				Format: strings.TrimPrefix(filepath.Ext(k), "."),
 			},
 		}, nil
 	}

examples/config/t/a.yaml

@@ -0,0 +1,5 @@
+foo:
+  baz: "2"
+  biu: "example"
+hello:
+  a: b

examples/config/t/b.yaml

@@ -0,0 +1,6 @@
+foo:
+  bar: 3
+  baz: aaaa
+hey:
+  good: bad
+  qux: quux