diff --git a/.travis.yml b/.travis.yml index 7db0c057c..b9bfc6413 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ language: go go: - 1.12.x + - 1.13.x + +services: + - docker # Only clone the most recent commit. git: @@ -16,6 +20,13 @@ env: - DEPLOY_ENV=dev - DISCOVERY_NODES=127.0.0.1:7171 - HTTP_PERF=tcp://0.0.0.0:0 + - DOCKER_COMPOSE_VERSION=1.24.1 + +before_install: + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin # Skip the install step. Don't `go get` dependencies. Only build with the code # in vendor/ diff --git a/doc/img/zipkin.jpg b/doc/img/zipkin.jpg new file mode 100644 index 000000000..8fca3794d Binary files /dev/null and b/doc/img/zipkin.jpg differ diff --git a/doc/wiki-cn/FAQ.md b/doc/wiki-cn/FAQ.md index 1260570c5..bdfca3499 100644 --- a/doc/wiki-cn/FAQ.md +++ b/doc/wiki-cn/FAQ.md @@ -9,7 +9,7 @@ go get -u github.com/bilibili/kratos/tool/kratos go: github.com/prometheus/client_model@v0.0.0-20190220174349-fd36f4220a90: parsing go.mod: missing module line go: github.com/remyoudompheng/bigfft@v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c: parsing go.mod: missing module line ``` -如果你使用了https://goproxy.io/ 代理,那你要使用其他代理来替换它，然后删除GOPATH目录下的mod缓存文件夹,然后重新执行安装命令 +如果你使用了https://goproxy.io/ 代理,那你要使用其他代理来替换它，然后删除GOPATH目录下的mod缓存文件夹（`go clean --modcache`）,然后重新执行安装命令 代理列表 diff --git a/doc/wiki-cn/breaker.md b/doc/wiki-cn/breaker.md index e69de29bb..cef56fbcd 100644 --- a/doc/wiki-cn/breaker.md +++ b/doc/wiki-cn/breaker.md @@ -0,0 +1,49 @@ +## 熔断器/Breaker +熔断器是为了当依赖的服务已经出现故障时，主动阻止对依赖服务的请求。保证自身服务的正常运行不受依赖服务影响，防止雪崩效应。 + +## kratos内置breaker的组件 +一般情况下直接使用kratos的组件时都自带了熔断逻辑，并且在提供了对应的breaker配置项。 +目前在kratos内集成熔断器的组件有: +- RPC client: pkg/net/rpc/warden/client +- Mysql client：pkg/database/sql +- Tidb client：pkg/database/tidb +- Http client：pkg/net/http/blademaster + +## 使用说明 +```go + //初始化熔断器组 + //一组熔断器公用同一个配置项，可从分组内取出单个熔断器使用。可用在比如mysql主从分离等场景。 + brkGroup := breaker.NewGroup(&breaker.Config{}) + //为每一个连接指定一个brekaker + //此处假设一个客户端连接对象实例为conn + //breakName定义熔断器名称 一般可以使用连接地址 + breakName = conn.Addr + conn.breaker = brkGroup.Get(breakName) + + //在连接发出请求前判断熔断器状态 + if err = conn.breaker.Allow(); err != nil { + return + } + + //连接执行成功或失败将结果告知braker + if(respErr != nil){ + conn.breaker.MarkFailed() + }else{ + conn.breaker.MarkSuccess() + } + +``` + +## 配置说明 +```go +type Config struct { + SwitchOff bool // 熔断器开关,默认关 false. + + K float64 //触发熔断的错误率（K = 1 - 1/错误率） + + Window xtime.Duration //统计桶窗口时间 + Bucket int //统计桶大小 + Request int64 //触发熔断的最少请求数量（请求少于该值时不会触发熔断） +} +``` + diff --git a/doc/wiki-cn/config-paladin.md b/doc/wiki-cn/config-paladin.md index 0becb02cf..28d8c1cc9 100644 --- a/doc/wiki-cn/config-paladin.md +++ b/doc/wiki-cn/config-paladin.md @@ -46,9 +46,10 @@ func TestMain(t *testing.M) { ### example main ```go -# main.go +// main.go func main() { - # 初始化paladin + flag.Parse() + // 初始化paladin if err := paladin.Init(); err != nil { panic(err) } @@ -58,28 +59,31 @@ func main() { ``` ### example HTTP/gRPC -```go +```toml # http.toml [server] addr = "0.0.0.0:9000" timeout = "1s" -# server.go +``` + +```go +// server.go func NewServer() { - # 默认配置用nil，这时读取HTTP/gRPC构架中的flag或者环境变量（可能是docker注入的环境变量，默认端口：8000/9000） - engine := bm.DefaultServer(nil) - - # 除非自己要替换了配置，用http.toml - var bc struct { - Server *bm.ServerConfig - } - if err = paladin.Get("http.toml").UnmarshalTOML("server", &bc); err != nil { - // 不存在时，将会为nil使用默认配置 - if err ！= paladin.ErrNotExist { - panic(err) - } - } - engine := bm.DefaultServer(conf) + // 默认配置用nil，这时读取HTTP/gRPC构架中的flag或者环境变量（可能是docker注入的环境变量，默认端口：8000/9000） + engine := bm.DefaultServer(nil) + + // 除非自己要替换了配置，用http.toml + var bc struct { + Server *bm.ServerConfig + } + if err := paladin.Get("http.toml").UnmarshalTOML(&bc); err != nil { + // 不存在时，将会为nil使用默认配置 + if err != paladin.ErrNotExist { + panic(err) + } + } + engine := bm.DefaultServer(bc.Server) } ``` @@ -87,25 +91,28 @@ func NewServer() { ```go # service.go type Service struct { - ac *paladin.Map + ac *paladin.Map } + func New() *Service { - # paladin.Map 通过atomic.Value支持自动热加载 - var ac = new(paladin.TOML) - if err := paladin.Watch("application.toml", ac); err != nil { - panic(err) - } - s := &Service{ - ac : ac; - } - return s + // paladin.Map 通过atomic.Value支持自动热加载 + var ac = new(paladin.TOML) + if err := paladin.Watch("application.toml", ac); err != nil { + panic(err) + } + s := &Service{ + ac: ac, + } + return s } + func (s *Service) Test() { - switch, err := s.ac.Bool("switch") - if err != nil { - // TODO - } - # or use default value - switch := paladin.Bool(s.ac.Value("switch"), false) + sw, err := s.ac.Get("switch").Bool() + if err != nil { + // TODO + } + + // or use default value + sw := paladin.Bool(s.ac.Get("switch"), false) } ``` diff --git a/doc/wiki-cn/dapper.md b/doc/wiki-cn/dapper.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/wiki-cn/ecode.md b/doc/wiki-cn/ecode.md new file mode 100644 index 000000000..d1cbf0817 --- /dev/null +++ b/doc/wiki-cn/ecode.md @@ -0,0 +1,102 @@ +# ecode + +## 背景 +错误码一般被用来进行异常传递，且需要具有携带`message`文案信息的能力。 + +## 错误码之Codes + +在`kratos`里，错误码被设计成`Codes`接口，声明如下[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/ecode/ecode.go)： + +```go +// Codes ecode error interface which has a code & message. +type Codes interface { + // sometimes Error return Code in string form + // NOTE: don't use Error in monitor report even it also work for now + Error() string + // Code get error code. + Code() int + // Message get code message. + Message() string + //Detail get error detail,it may be nil. + Details() []interface{} +} + +// A Code is an int error code spec. +type Code int +``` + +可以看到该接口一共有四个方法，且`type Code int`结构体实现了该接口。 + +### 注册message + +一个`Code`错误码可以对应一个`message`，默认实现会从全局变量`_messages`中获取，业务可以将自定义`Code`对应的`message`通过调用`Register`方法的方式传递进去，如： + +```go +cms := map[int]string{ + 0: "很好很强大！", + -304: "啥都没变啊~", + -404: "啥都没有啊~", +} +ecode.Register(cms) + +fmt.Println(ecode.OK.Message()) // 输出：很好很强大！ +``` + +注意：`map[int]string`类型并不是绝对，比如有业务要支持多语言的场景就可以扩展为类似`map[int]LangStruct`的结构，因为全局变量`_messages`是`atomic.Value`类型，只需要修改对应的`Message`方法实现即可。 + +### Details + +`Details`接口为`gRPC`预留，`gRPC`传递异常会将服务端的错误码pb序列化之后赋值给`Details`，客户端拿到之后反序列化得到，具体可阅读`status`的实现： +1. `ecode`包内的`Status`结构体实现了`Codes`接口[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/ecode/status.go) +2. `warden/internal/status`包内包装了`ecode.Status`和`grpc.Status`进行互相转换的方法[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/net/rpc/warden/internal/status/status.go) +3. `warden`的`client`和`server`则使用转换方法将`gRPC`底层返回的`error`最终转换为`ecode.Status` [代码位置](https://github.com/bilibili/kratos/blob/master/pkg/net/rpc/warden/client.go#L162) + +## 转换为ecode + +错误码转换有以下两种情况： +1. 因为框架传递错误是靠`ecode`错误码，比如bm框架返回的`code`字段默认就是数字，那么客户端接收到如`{"code":-404}`的话，可以使用`ec := ecode.Int(-404)`或`ec := ecode.String("-404")`来进行转换。 +2. 在项目中`dao`层返回一个错误码，往往返回参数类型建议为`error`而不是`ecode.Codes`，因为`error`更通用，那么上层`service`就可以使用`ec := ecode.Cause(err)`进行转换。 + +## 判断 + +错误码判断是否相等： +1. `ecode`与`ecode`判断使用：`ecode.Equal(ec1, ec2)` +2. `ecode`与`error`判断使用：`ecode.EqualError(ec, err)` + +## 使用工具生成 + +使用proto协议定义错误码，格式如下： + +```proto +// user.proto +syntax = "proto3"; + +package ecode; + +enum UserErrCode { + UserUndefined = 0; // 因protobuf协议限制必须存在！！！无意义的0，工具生成代码时会忽略该参数 + UserNotLogin = 123; // 正式错误码 +} +``` + +需要注意以下几点： + +1. 必须是enum类型，且名字规范必须以"ErrCode"结尾，如：UserErrCode +2. 因为protobuf协议限制，第一个enum值必须为无意义的0 + +使用`kratos tool protoc --ecode user.proto`进行生成，生成如下代码： + +```go +package ecode + +import ( + "github.com/bilibili/kratos/pkg/ecode" +) + +var _ ecode.Codes + +// UserErrCode +var ( + UserNotLogin = ecode.New(123); +) +``` diff --git a/doc/wiki-cn/summary.md b/doc/wiki-cn/summary.md index 4b6940647..dc0e4cb35 100644 --- a/doc/wiki-cn/summary.md +++ b/doc/wiki-cn/summary.md @@ -16,7 +16,8 @@ * [warden protobuf](warden-pb.md) * [config](config.md) * [paladin](config-paladin.md) -* [dapper trace](dapper.md) +* [ecode](ecode.md) +* [trace](dapper.md) * [log](logger.md) * [log-agent](log-agent.md) * [database](database.md) @@ -34,3 +35,7 @@ * [genbts](kratos-genbts.md) * [限流bbr](ratelimit.md) * [熔断breaker](breaker.md) +* [UT单元测试](ut.md) + * [testcli UT运行环境构建工具](ut-testcli.md) + * [testgen UT代码自动生成器](ut-testgen.md) + * [support UT周边辅助工具](ut-support.md) \ No newline at end of file diff --git a/doc/wiki-cn/trace.md b/doc/wiki-cn/trace.md new file mode 100644 index 000000000..6a0c5d555 --- /dev/null +++ b/doc/wiki-cn/trace.md @@ -0,0 +1,42 @@ +# 背景 + +当代的互联网的服务，通常都是用复杂的、大规模分布式集群来实现的。互联网应用构建在不同的软件模块集上，这些软件模块，有可能是由不同的团队开发、可能使用不同的编程语言来实现、有可能布在了几千台服务器，横跨多个不同的数据中心。因此，就需要一些可以帮助理解系统行为、用于分析性能问题的工具。 + +# 概览 + +* kratos内部的trace基于opentracing语义 +* 使用protobuf协议描述trace结构 +* 全链路支持（gRPC/HTTP/MySQL/Redis/Memcached等） + +## 参考文档 + +[opentracing](https://github.com/opentracing-contrib/opentracing-specification-zh/blob/master/specification.md) +[dapper](https://bigbully.github.io/Dapper-translation/) + +# 使用 + +kratos本身不提供整套`trace`数据方案，但在`net/trace/report.go`内声明了`repoter`接口，可以简单的集成现有开源系统，比如：`zipkin`和`jaeger`。 + +### zipkin使用 + +可以看[zipkin](https://github.com/bilibili/kratos/tree/master/pkg/net/trace/zipkin)的协议上报实现，具体使用方式如下： + +1. 前提是需要有一套自己搭建的`zipkin`集群 +2. 在业务代码的`main`函数内进行初始化，代码如下： + +```go +// 忽略其他代码 +import "github.com/bilibili/kratos/pkg/net/trace/zipkin" +// 忽略其他代码 +func main(){ + // 忽略其他代码 + zipkin.Init(&zipkin.Config{ + Endpoint: "http://localhost:9411/api/v2/spans", + }) + // 忽略其他代码 +} +``` + +### zipkin效果图 + +![zipkin](/doc/img/zipkin.jpg) diff --git a/doc/wiki-cn/ut-support.md b/doc/wiki-cn/ut-support.md new file mode 100644 index 000000000..b4b3dd707 --- /dev/null +++ b/doc/wiki-cn/ut-support.md @@ -0,0 +1,497 @@ +## 单元测试辅助工具 +在单元测试中，我们希望每个测试用例都是独立的。这时候就需要Stub, Mock, Fakes等工具来帮助我们进行用例和依赖之间的隔离。 + +同时通过对错误情况的 Mock 也可以帮我们检查代码多个分支结果，从而提高覆盖率。 + +以下工具已加入到 Kratos 框架 go modules，可以借助 testgen 代码生成器自动生成部分工具代码，请放心食用。更多使用方法还欢迎大家多多探索。 + +### GoConvey +GoConvey是一套针对golang语言的BDD类型的测试框架。提供了良好的管理和执行测试用例的方式，包含丰富的断言函数，而且同时有测试执行和报告Web界面的支持。 + +#### 使用特性 +为了更好的使用 GoConvey 来编写和组织测试用例，需要注意以下几点特性： + +1. Convey方法和So方法的使用 +> - Convey方法声明了一种规格的组织，每个组织内包含一句描述和一个方法。在方法内也可以嵌套其他Convey语句和So语句。 +```Go +// 顶层Convey方法，需引入*testing.T对象 +Convey(description string, t *testing.T, action func()) + +// 其他嵌套Convey方法，无需引入*testing.T对象 +Convey(description string, action func()) +``` +注：同一Scope下的Convey语句描述不可以相同！ +> - So方法是断言方法，用于对执行结果进行比对。GoConvey官方提供了大量断言，同时也可以自定义自己的断言（[戳这里了解官方文档](https://github.com/smartystreets/goconvey/wiki/Assertions)） +```Go +// A=B断言 +So(A, ShouldEqual, B) + +// A不为空断言 +So(A, ShouldNotBeNil) +``` + +2. 执行次序 +> 假设有以下Convey伪代码，执行次序将为A1B2A1C3。将Convey方法类比树的结点的话，整体执行类似树的遍历操作。 +> 所以Convey A部分可在组织测试用例时，充当“Setup”的方法。用于初始化等一些操作。 +```Go +Convey伪代码 +Convey A + So 1 + Convey B + So 2 + Convey C + So 3 +``` + +3. Reset方法 +> GoConvey提供了Reset方法来进行“Teardown”的操作。用于执行完测试用例后一些状态的回收，连接关闭等操作。Reset方法不可与顶层Convey语句在同层。 +```Go +// Reset +Reset func(action func()) +``` +假设有以下带有Reset方法的伪代码，同层Convey语句执行完后均会执行同层的Reset方法。执行次序为A1B2C3EA1D4E。 +```Go +Convey A + So 1 + Convey B + So 2 + Convey C + So 3 + Convey D + So 4 + Reset E +``` + +4. 自然语言逻辑到测试用例的转换 +> 在了解了Convey方法的特性和执行次序后，我们可以通过这些性质把对一个方法的测试用例按照日常逻辑组织起来。尤其建议使用Given-When-Then的形式来组织 +> - 比较直观的组织示例 +```Go +Convey("Top-level", t, func() { + + // Setup 工作，在本层内每个Convey方法执行前都会执行的部分： + db.Open() + db.Initialize() + + Convey("Test a query", func() { + db.Query() + // TODO: assertions here + }) + + Convey("Test inserts", func() { + db.Insert() + // TODO: assertions here + }) + + Reset(func() { + // Teardown工作，在本层内每个Convey方法执行完后都会执行的部分： + db.Close() + }) + +}) +``` +> - 定义单独的包含Setup和Teardown的帮助方法 +```Go +package main + +import ( + "database/sql" + "testing" + + _ "github.com/lib/pq" + . "github.com/smartystreets/goconvey/convey" +) + +// 帮助方法，将原先所需的处理方法以参数（f）形式传入 +func WithTransaction(db *sql.DB, f func(tx *sql.Tx)) func() { + return func() { + // Setup工作 + tx, err := db.Begin() + So(err, ShouldBeNil) + + Reset(func() { + // Teardown工作 + /* Verify that the transaction is alive by executing a command */ + _, err := tx.Exec("SELECT 1") + So(err, ShouldBeNil) + + tx.Rollback() + }) + + // 调用传入的闭包做实际的事务处理 + f(tx) + } +} + +func TestUsers(t *testing.T) { + db, err := sql.Open("postgres", "postgres://localhost?sslmode=disable") + if err != nil { + panic(err) + } + + Convey("Given a user in the database", t, WithTransaction(db, func(tx *sql.Tx) { + _, err := tx.Exec(`INSERT INTO "Users" ("id", "name") VALUES (1, 'Test User')`) + So(err, ShouldBeNil) + + Convey("Attempting to retrieve the user should return the user", func() { + var name string + + data := tx.QueryRow(`SELECT "name" FROM "Users" WHERE "id" = 1`) + err = data.Scan(&name) + + So(err, ShouldBeNil) + So(name, ShouldEqual, "Test User") + }) + })) +} +``` + +#### 使用建议 +强烈建议使用 [testgen](https://github.com/bilibili/kratos/blob/master/doc/wiki-cn/ut-testgen.md) 进行测试用例的生成，生成后每个方法将包含一个符合以下规范的正向用例。 + +用例规范： +1. 每个方法至少包含一个测试方法（命名为Test[PackageName][FunctionName]） +2. 每个测试方法包含一个顶层Convey语句，仅在此引入admin *testing.T类型的对象，在该层进行变量声明。 +3. 每个测试方法不同的用例用Convey方法组织 +4. 每个测试用例的一组断言用一个Convey方法组织 +5. 使用convey.C保持上下文一致 + +### MonkeyPatching + +#### 特性和使用条件 +1. Patch()对任何无接收者的方法均有效 +2. PatchInstanceMethod()对有接收者的包内/私有方法无法工作（因使用到了反射机制）。可以采用给私有方法的下一级打补丁，或改为无接收者的方法，或将方法转为公有 + +#### 适用场景（建议） +项目代码中上层对下层包依赖时，下层包方法Mock（例如service层对dao层方法依赖时） +基础库（MySql, Memcache, Redis）错误Mock +其他标准库，基础库以及第三方包方法Mock + +#### 使用示例 +1. 上层包对下层包依赖示例 +Service层对Dao层依赖： +```GO +// 原方法 +func (s *Service) realnameAlipayApply(c context.Context, mid int64) (info *model.RealnameAlipayApply, err error) { + if info, err = s.mbDao.RealnameAlipayApply(c, mid); err != nil { + return + } + ... + return +} + +// 测试方法 +func TestServicerealnameAlipayApply(t *testing.T) { + convey.Convey("realnameAlipayApply", t, func(ctx convey.C) { + ... + ctx.Convey("When everything goes positive", func(ctx convey.C) { + guard := monkey.PatchInstanceMethod(reflect.TypeOf(s.mbDao), "RealnameAlipayApply", func(_ *dao.Dao, _ context.Context, _ int64) (*model.RealnameAlipayApply, error) { + return nil, nil + }) + defer guard.Unpatch() + info, err := s.realnameAlipayApply(c, mid) + ctx.Convey("Then err should be nil,info should not be nil", func(ctx convey.C) { + ctx.So(info, convey.ShouldNotBeNil) + ctx.So(err, convey.ShouldBeNil) + }) + }) + }) +} +``` +2. 基础库错误Mock示例 +```Go + +// 原方法（部分） +func (d *Dao) BaseInfoCache(c context.Context, mid int64) (info *model.BaseInfo, err error) { + ... + conn := d.mc.Get(c) + defer conn.Close() + item, err := conn.Get(key) + if err != nil { + log.Error("conn.Get(%s) error(%v)", key, err) + return + } + ... + return +} + + +// 测试方法（错误Mock部分） +func TestDaoBaseInfoCache(t *testing.T) { + convey.Convey("BaseInfoCache", t, func(ctx convey.C) { + ... + Convey("When conn.Get gets error", func(ctx convey.C) { + guard := monkey.PatchInstanceMethod(reflect.TypeOf(d.mc), "Get", func(_ *memcache.Pool, _ context.Context) memcache.Conn { + return memcache.MockWith(memcache.ErrItemObject) + }) + defer guard.Unpatch() + _, err := d.BaseInfoCache(c, mid) + ctx.Convey("Error should be equal to memcache.ErrItemObject", func(ctx convey.C) { + ctx.So(err, convey.ShouldEqual, memcache.ErrItemObject) + }) + }) + }) +} +``` +#### 注意事项 +- Monkey非线程安全 +- Monkey无法针对Inline方法打补丁，在测试时可以使用go test -gcflags=-l来关闭inline编译的模式（一些简单的go inline介绍戳这里） +- Monkey在一些面向安全不允许内存页写和执行同时进行的操作系统上无法工作 +- 更多详情请戳：https://github.com/bouk/monkey + + + +### Gock——HTTP请求Mock工具 + +#### 特性和使用条件 + +#### 工作原理 +1. 截获任意通过 http.DefaultTransport或者自定义http.Transport对外的http.Client请求 +2. 以“先进先出”原则将对外需求和预定义好的HTTP Mock池中进行匹配 +3. 如果至少一个Mock被匹配，将按照2中顺序原则组成Mock的HTTP返回 +4. 如果没有Mock被匹配，若实际的网络可用，将进行实际的HTTP请求。否则将返回错误 + +#### 特性 +- 内建帮助工具实现JSON/XML简单Mock +- 支持持久的、易失的和TTL限制的Mock +- 支持HTTP Mock请求完整的正则表达式匹配 +- 可通过HTTP方法，URL参数，请求头和请求体匹配 +- 可扩展和可插件化的HTTP匹配规则 +- 具备在Mock和实际网络模式之间切换的能力 +- 具备过滤和映射HTTP请求到正确的Mock匹配的能力 +- 支持映射和过滤可以更简单的掌控Mock +- 通过使用http.RoundTripper接口广泛兼容HTTP拦截器 +- 可以在任意net/http兼容的Client上工作 +- 网络延迟模拟（beta版本） +- 无其他依赖 + +#### 适用场景（建议） +任何需要进行HTTP请求的操作，建议全部用Gock进行Mock，以减少对环境的依赖。 + +使用示例： +1. net/http 标准库 HTTP 请求Mock +```Go +import gock "gopkg.in/h2non/gock.v1" + +// 原方法 + func (d *Dao) Upload(c context.Context, fileName, fileType string, expire int64, body io.Reader) (location string, err error) { + ... + resp, err = d.bfsClient.Do(req) //d.bfsClient类型为*http.client + ... + if resp.StatusCode != http.StatusOK { + ... + } + header = resp.Header + code = header.Get("Code") + if code != strconv.Itoa(http.StatusOK) { + ... + } + ... + return +} + + +// 测试方法 +func TestDaoUpload(t *testing.T) { + convey.Convey("Upload", t, func(ctx convey.C) { + ... + // d.client 类型为 *http.client 根据Gock包描述需要设置http.Client的Transport情况。也可在TestMain中全局设置，则所有的HTTP请求均通过Gock来解决 + d.client.Transport = gock.DefaultTransport // ！注意：进行httpMock前需要对http 请求进行拦截，否则Mock失败 + // HTTP请求状态和Header都正确的Mock + ctx.Convey("When everything is correct", func(ctx convey.C) { + httpMock("PUT", url).Reply(200).SetHeaders(map[string]string{ + "Code": "200", + "Location": "SomePlace", + }) + location, err := d.Upload(c, fileName, fileType, expire, body) + ctx.Convey("Then err should be nil.location should not be nil.", func(ctx convey.C) { + ctx.So(err, convey.ShouldBeNil) + ctx.So(location, convey.ShouldNotBeNil) + }) + }) + ... + // HTTP请求状态错误Mock + ctx.Convey("When http request status != 200", func(ctx convey.C) { + d.client.Transport = gock.DefaultTransport + httpMock("PUT", url).Reply(404) + _, err := d.Upload(c, fileName, fileType, expire, body) + ctx.Convey("Then err should not be nil", func(ctx convey.C) { + ctx.So(err, convey.ShouldNotBeNil) + }) + }) + // HTTP请求Header中Code值错误Mock + ctx.Convey("When http request Code in header != 200", func(ctx convey.C) { + d.client.Transport = gock.DefaultTransport + httpMock("PUT", url).Reply(404).SetHeaders(map[string]string{ + "Code": "404", + "Location": "SomePlace", + }) + _, err := d.Upload(c, fileName, fileType, expire, body) + ctx.Convey("Then err should not be nil", func(ctx convey.C) { + ctx.So(err, convey.ShouldNotBeNil) + }) + }) + + // 由于同包内有其他进行实际HTTP请求的测试。所以再每次用例结束后，进行现场恢复（关闭Gock设置默认的Transport） + ctx.Reset(func() { + gock.OffAll() + d.client.Transport = http.DefaultClient.Transport + }) + + + }) +} + +func httpMock(method, url string) *gock.Request { + r := gock.New(url) + r.Method = strings.ToUpper(method) + return r +} +``` +2. blademaster库HTTP请求Mock +```Go +// 原方法 +func (d *Dao) SendWechatToGroup(c context.Context, chatid, msg string) (err error) { + ... + if err = d.client.Do(c, req, &res); err != nil { + ... + } + if res.Code != 0 { + ... + } + return +} + +// 测试方法 +func TestDaoSendWechatToGroup(t *testing.T) { + convey.Convey("SendWechatToGroup", t, func(ctx convey.C) { + ... + // 根据Gock包描述需要设置bm.Client的Transport情况。也可在TestMain中全局设置，则所有的HTTP请求均通过Gock来解决。 + // d.client 类型为 *bm.client + d.client.SetTransport(gock.DefaultTransport) // ！注意：进行httpMock前需要对http 请求进行拦截，否则Mock失败 + // HTTP请求状态和返回内容正常Mock + ctx.Convey("When everything gose postive", func(ctx convey.C) { + httpMock("POST", _sagaWechatURL+"/appchat/send").Reply(200).JSON(`{"code":0,"message":"0"}`) + err := d.SendWechatToGroup(c, d.c.WeChat.ChatID, msg) + ... + }) + // HTTP请求状态错误Mock + ctx.Convey("When http status != 200", func(ctx convey.C) { + httpMock("POST", _sagaWechatURL+"/appchat/send").Reply(404) + err := d.SendWechatToGroup(c, d.c.WeChat.ChatID, msg) + ... + }) + // HTTP请求返回值错误Mock + ctx.Convey("When http response code != 0", func(ctx convey.C) { + httpMock("POST", _sagaWechatURL+"/appchat/send").Reply(200).JSON(`{"code":-401,"message":"0"}`) + err := d.SendWechatToGroup(c, d.c.WeChat.ChatID, msg) + ... + }) + // 由于同包内有其他进行实际HTTP请求的测试。所以再每次用例结束后，进行现场恢复（关闭Gock设置默认的Transport）。 + ctx.Reset(func() { + gock.OffAll() + d.client.SetTransport(http.DefaultClient.Transport) + }) + }) +} + +func httpMock(method, url string) *gock.Request { + r := gock.New(url) + r.Method = strings.ToUpper(method) + return r +} +``` + +#### 注意事项 +- Gock不是完全线程安全的 +- 如果执行并发代码，在配置Gock和解释定制的HTTP clients时，要确保Mock已经事先声明好了来避免不需要的竞争机制 +- 更多详情请戳：https://github.com/h2non/gock + + +### GoMock + +#### 使用条件 +只能对公有接口（interface）定义的代码进行Mock，并仅能在测试过程中进行 + +#### 使用方法 +- 官方安装使用步骤 +```shell +## 获取GoMock包和自动生成Mock代码工具mockgen +go get github.com/golang/mock/gomock +go install github.com/golang/mock/mockgen + +## 生成mock文件 +## 方法1：生成对应文件下所有interface +mockgen -source=path/to/your/interface/file.go + +## 方法2：生成对应包内指定多个interface，并用逗号隔开 +mockgen database/sql/driver Conn,Driver + +## 示例： +mockgen -destination=$GOPATH/kratos/app/xxx/dao/dao_mock.go -package=dao kratos/app/xxx/dao DaoInterface +``` +- testgen 使用步骤（GoMock生成功能已集成在Creater工具中，无需额外安装步骤即可直接使用） +```shell +## 直接给出含有接口类型定义的包路径，生成Mock文件将放在包目录下一级mock/pkgName_mock.go中 +./creater --m mock absolute/path/to/your/pkg +``` +- 测试代码内使用方法 +```Go +// 测试用例内直接使用 +// 需引入的包 +import ( + ... + "github.com/otokaze/mock/gomock" + ... +) + +func TestPkgFoo(t *testing.T) { + convey.Convey("Foo", t, func(ctx convey.C) { + ... + ctx.Convey("Mock Interface to test", func(ctx convey.C) { + // 1. 使用gomock.NewController新增一个控制器 + mockCtrl := gomock.NewController(t) + // 2. 测试完成后关闭控制器 + defer mockCtrl.Finish() + // 3. 以控制器为参数生成Mock对象 + yourMock := mock.NewMockYourClient(mockCtrl) + // 4. 使用Mock对象替代原代码中的对象 + yourClient = yourMock + // 5. 使用EXPECT().方法名(方法参数).Return(返回值)来构造所需输入/输出 + yourMock.EXPECT().YourMethod(gomock.Any()).Return(nil) + res:= Foo(params) + ... + }) + ... + }) +} + +// 可以利用Convey执行顺序方式适当调整以简化代码 +func TestPkgFoo(t *testing.T) { + convey.Convey("Foo", t, func(ctx convey.C) { + ... + mockCtrl := gomock.NewController(t) + yourMock := mock.NewMockYourClient(mockCtrl) + ctx.Convey("Mock Interface to test1", func(ctx convey.C) { + yourMock.EXPECT().YourMethod(gomock.Any()).Return(nil) + ... + }) + ctx.Convey("Mock Interface to test2", func(ctx convey.C) { + yourMock.EXPECT().YourMethod(args).Return(res) + ... + }) + ... + ctx.Reset(func(){ + mockCtrl.Finish() + }) + }) +} +``` + +#### 适用场景（建议） +1. gRPC中的Client接口 +2. 也可改造现有代码构造Interface后使用（具体可配合Creater的功能进行Interface和Mock的生成） +3. 任何对接口中定义方法依赖的场景 + +#### 注意事项 +- 如有Mock文件在包内，在执行单元测试时Mock代码会被识别进行测试。请注意Mock文件的放置。 +- 更多详情请戳：https://github.com/golang/mock \ No newline at end of file diff --git a/doc/wiki-cn/ut-testcli.md b/doc/wiki-cn/ut-testcli.md new file mode 100644 index 000000000..f17e3d281 --- /dev/null +++ b/doc/wiki-cn/ut-testcli.md @@ -0,0 +1,154 @@ +## testcli UT运行环境构建工具 +基于 docker-compose 实现跨平台跨语言环境的容器依赖管理方案，以解决运行ut场景下的 (mysql, redis, mc)容器依赖问题。 + +*这个是testing/lich的二进制工具版本（Go请直接使用库版本：github.com/bilibili/kratos/pkg/testing/lich)* + +### 功能和特性 +- 自动读取 test 目录下的 yaml 并启动依赖 +- 自动导入 test 目录下的 DB 初始化 SQL +- 提供特定容器内的 healthcheck (mysql, mc, redis) +- 提供一站式解决 UT 服务依赖的工具版本 (testcli) + +### 编译安装 +*使用本工具/库需要前置安装好 docker & docker-compose@v1.24.1^* + +#### Method 1. With go get +```shell +go get -u github.com/bilibili/kratos/tool/testcli +$GOPATH/bin/testcli -h +``` +#### Method 2. Build with Go +```shell +cd github.com/bilibili/kratos/tool/testcli +go build -o $GOPATH/bin/testcli +$GOPATH/bin/testcli -h +``` +#### Method 3. Import with Kratos pkg +```Go +import "github.com/bilibili/kratos/pkg/testing/lich" +``` + +### 构建数据 +#### Step 1. create docker-compose.yml +创建依赖服务的 docker-compose.yml，并把它放在项目路径下的 test 文件夹下面。例如： +```shell +mkdir -p $YOUR_PROJECT/test +``` +```yaml +version: "3.7" + +services: + db: + image: mysql:5.6 + ports: + - 3306:3306 + environment: + - MYSQL_ROOT_PASSWORD=root + volumes: + - .:/docker-entrypoint-initdb.d + command: [ + '--character-set-server=utf8', + '--collation-server=utf8_unicode_ci' + ] + + redis: + image: redis + ports: + - 6379:6379 +``` +一般来讲，我们推荐在项目根目录创建 test 目录，里面存放描述服务的yml，以及需要初始化的数据（database.sql等）。 + +同时也需要注意，正确的对容器内服务进行健康检测，testcli会在容器的health状态执行UT，其实我们也内置了针对几个较为通用镜像（mysql mariadb mc redis）的健康检测，也就是不写也没事(^^;; + +#### Step 2. export database.sql +构造初始化的数据（database.sql等），当然也把它也在 test 文件夹里。 +```sql +CREATE DATABASE IF NOT EXISTS `YOUR_DATABASE_NAME`; + +SET NAMES 'utf8'; +USE `YOUR_DATABASE_NAME`; + +CREATE TABLE IF NOT EXISTS `YOUR_TABLE_NAME` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键'， + PRIMARY KEY (`id`), +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='YOUR_TABLE_NAME'; +``` +这里需要注意，在创建库/表的时候尽量加上 IF NOT EXISTS，以给予一定程度的容错，以及 SET NAMES 'utf8'; 用于解决客户端连接乱码问题。 + +#### Step 3. change your project mysql config +```toml +[mysql] + addr = "127.0.0.1:3306" + dsn = "root:root@tcp(127.0.0.1:3306)/YOUR_DATABASE?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8" + active = 20 + idle = 10 + idleTimeout ="1s" + queryTimeout = "1s" + execTimeout = "1s" + tranTimeout = "1s" +``` +在 *Step 1* 我们已经指定了服务对外暴露的端口为3306（这当然也可以是你指定的任何值），那理所应当的我们也要修改项目连接数据库的配置～ + +Great! 至此你已经完成了运行所需要用到的数据配置，接下来就来运行它。 + +### 运行 +开头也说过本工具支持两种运行方式：testcli 二进制工具版本和 go package 源码包，业务方可以根据需求场景进行选择。 +#### Method 1. With testcli tool +*已支持的 flag： -f，--nodown，down，run* +- -f，指定 docker-compose.yaml 文件路径，默认为当前目录下。 +- --nodown，指定是否在UT执行完成后保留容器，以供下次复用。 +- down，teardown 销毁当前项目下这个 compose 文件产生的容器。 +- run，运行你当前语言的单测执行命令（如：golang为 go test -v ./） + +example: +```shell +testcli -f ../../test/docker-compose.yaml run go test -v ./ +``` +#### Method 2. Import with Kratos pkg +- Step1. 在 Dao|Service 层中的 TestMain 单测主入口中，import "github.com/bilibili/kratos/pkg/testing/lich" 引入testcli工具的go库版本。 +- Step2. 使用 flag.Set("f", "../../test/docker-compose.yaml") 指定 docker-compose.yaml 文件的路径。 +- Step3. 在 flag.Parse() 后即可使用 lich.Setup() 安装依赖&初始化数据（注意测试用例执行结束后 lich.Teardown() 回收下～） +- Step4. 运行 `go test -v ./ `看看效果吧～ + +example: +```Go +package dao + + +import ( + "flag" + "os" + "strings" + "testing" + + "github.com/bilibili/kratos/pkg/conf/paladin" + "github.com/bilibili/kratos/pkg/testing/lich" + ) + +var ( + d *Dao +) + +func TestMain(m *testing.M) { + flag.Set("conf", "../../configs") + flag.Set("f", "../../test/docker-compose.yaml") + flag.Parse() + if err := paladin.Init(); err != nil { + panic(err) + } + if err := lich.Setup(); err != nil { + panic(err) + } + defer lich.Teardown() + d = New() + if code := m.Run(); code != 0 { + panic(code) + } +} + ``` +## 注意 +因为启动mysql容器较为缓慢，健康检测的机制会重试3次，每次暂留5秒钟，基本在10s内mysql就能从creating到服务正常启动！ + +当然你也可以在使用 testcli 时加上 --nodown，使其不用每次跑都新建容器，只在第一次跑的时候会初始化容器，后面都进行复用，这样速度会快很多。 + +成功启动后就欢乐奔放的玩耍吧～ Good Lucky! \ No newline at end of file diff --git a/doc/wiki-cn/ut-testgen.md b/doc/wiki-cn/ut-testgen.md new file mode 100644 index 000000000..3ec686e7f --- /dev/null +++ b/doc/wiki-cn/ut-testgen.md @@ -0,0 +1,52 @@ +## testgen UT代码自动生成器 +解放你的双手，让你的UT一步到位！ + +### 功能和特性 +- 支持生成 Dao|Service 层UT代码功能（每个方法包含一个正向用例） +- 支持生成 Dao|Service 层测试入口文件dao_test.go, service_test.go（用于控制初始化，控制测试流程等） +- 支持生成Mock代码（使用GoMock框架） +- 支持选择不同模式生成不同代码（使用"–m mode"指定） +- 生成单元测试代码时，同时支持传入目录或文件 +- 支持指定方法追加生成测试用例（使用"–func funcName"指定） + +### 编译安装 +#### Method 1. With go get +```shell +go get -u github.com/bilibili/kratos/tool/testgen +$GOPATH/bin/testgen -h +``` +#### Method 2. Build with Go +```shell +cd github.com/bilibili/kratos/tool/testgen +go build -o $GOPATH/bin/testgen +$GOPATH/bin/testgen -h +``` +### 运行 +#### 生成Dao/Service层单元UT +```shell +$GOPATH/bin/testgen YOUR_PROJECT/dao # default mode +$GOPATH/bin/testgen --m test path/to/your/pkg +$GOPATH/bin/testgen --func functionName path/to/your/pkg +``` + +#### 生成接口类型 +```shell +$GOPATH/bin/testgen --m interface YOUR_PROJECT/dao #当前仅支持传目录，如目录包含子目录也会做处理 +``` + +#### 生成Mock代码 + ```shell +$GOPATH/bin/testgen --m mock YOUR_PROJECT/dao #仅传入包路径即可 +``` + +#### 生成Monkey代码 +```shell +$GOPATH/bin/testgen --m monkey yourCodeDirPath #仅传入包路径即可 +``` +### 赋诗一首 +``` +莫生气 莫生气 +代码辣鸡非我意 +自己动手分田地 +谈笑风生活长命 +``` \ No newline at end of file diff --git a/doc/wiki-cn/ut.md b/doc/wiki-cn/ut.md new file mode 100644 index 000000000..9a73d15c6 --- /dev/null +++ b/doc/wiki-cn/ut.md @@ -0,0 +1,38 @@ +# 背景 +单元测试即对最小可测试单元进行检查和验证，它可以很好的让你的代码在上测试环境之前自己就能前置的发现问题，解决问题。当然每个语言都有原生支持的 UT 框架，不过在 Kratos 里面我们需要有一些配套设施以及周边工具来辅助我们构筑整个 UT 生态。 + +# 工具链 +- testgen UT代码自动生成器（README: tool/testgen/README.md） +- testcli UT运行环境构建工具（README: tool/testcli/README.md） + +# 测试框架选型 +golang 的单元测试，既可以用官方自带的 testing 包，也有开源的如 testify、goconvey 业内知名，使用非常多也很好用的框架。 + +根据一番调研和内部使用经验，我们确定： +> - testing 作为基础库测试框架（非常精简不过够用） +> - goconvey 作为业务程序的单元测试框架（因为涉及比较多的业务场景和流程控制判断，比如更丰富的res值判断、上下文嵌套支持、还有webUI等） + +# 单元测试标准 +1. 覆盖率，当前标准：60%（所有包均需达到） +尽量达到70%以上。当然覆盖率并不能完全说明单元测试的质量，开发者需要考虑关键的条件判断和预期的结果。复杂的代码是需要好好设计测试用例的。 +2. 通过率，当前标准：100%（所有用例中的断言必须通过） + +# 书写建议 +1. 结果验证 +> - 校验err是否为nil. err是go函数的标配了，也是最基础的判断，如果err不为nil，基本上函数返回值或者处理肯定是有问题了。 +> - 检验res值是否正确。res值的校验是非常重要的，也是很容易忽略的地方。比如返回结构体对象，要对结构体的成员进行判断，而有可能里面是0值。goconvey对res值的判断支持是非常友好的。 + +2. 逻辑验证 +> 业务代码经常是流程比较复杂的，而函数的执行结果也是有上下文的，比如有不同条件分支。goconvey就非常优雅的支持了这种情况，可以嵌套执行。单元测试要结合业务代码逻辑，才能尽量的减少线上bug。 + +3. 如何mock +主要分以下3块： +> - 基础组件，如mc、redis、mysql等，由 testcli(testing/lich) 起基础镜像支持（需要提供建表、INSERT语句）与本地开发环境一致，也保证了结果的一致性。 +> - rpc server，如 xxxx-service 需要定义 interface 供业务依赖方使用。所有rpc server 都必须要提供一个interface+mock代码(gomock)。 +> - http server则直接写mock代码gock。 + +# 注意 +单元测试极其重要，良好的单元测试习惯能很大程度上避免代码变更引起的bug！ +单元测试极其重要，良好的单元测试习惯能很大程度上避免代码变更引起的bug！ +单元测试极其重要，良好的单元测试习惯能很大程度上避免代码变更引起的bug！ +以为很重要所以重复 3 遍～ \ No newline at end of file diff --git a/example/blademaster/middleware/auth/example_test.go b/example/blademaster/middleware/auth/example_test.go index b97e22276..0d5b30b46 100644 --- a/example/blademaster/middleware/auth/example_test.go +++ b/example/blademaster/middleware/auth/example_test.go @@ -3,8 +3,8 @@ package auth_test import ( "fmt" - bm "github.com/bilibili/kratos/pkg/net/http/blademaster" "github.com/bilibili/kratos/example/blademaster/middleware/auth" + bm "github.com/bilibili/kratos/pkg/net/http/blademaster" "github.com/bilibili/kratos/pkg/net/metadata" ) diff --git a/example/protobuf/api.bm.go b/example/protobuf/api.bm.go index 4e29bad26..6e3ce12a7 100644 --- a/example/protobuf/api.bm.go +++ b/example/protobuf/api.bm.go @@ -9,6 +9,7 @@ import ( bm "github.com/bilibili/kratos/pkg/net/http/blademaster" "github.com/bilibili/kratos/pkg/net/http/blademaster/binding" ) +import google_protobuf1 "github.com/golang/protobuf/ptypes/empty" // to suppressed 'imported but not used warning' var _ *bm.Context @@ -16,10 +17,13 @@ var _ context.Context var _ binding.StructValidator var PathUserInfo = "/user.api.User/Info" +var PathUserCard = "/user.api.User/Card" // UserBMServer is the server API for User service. type UserBMServer interface { Info(ctx context.Context, req *UserReq) (resp *InfoReply, err error) + + Card(ctx context.Context, req *UserReq) (resp *google_protobuf1.Empty, err error) } var UserSvc UserBMServer @@ -33,8 +37,18 @@ func userInfo(c *bm.Context) { c.JSON(resp, err) } +func userCard(c *bm.Context) { + p := new(UserReq) + if err := c.BindWith(p, binding.Default(c.Request.Method, c.Request.Header.Get("Content-Type"))); err != nil { + return + } + resp, err := UserSvc.Card(c, p) + c.JSON(resp, err) +} + // RegisterUserBMServer Register the blademaster route func RegisterUserBMServer(e *bm.Engine, server UserBMServer) { UserSvc = server e.GET("/user.api.User/Info", userInfo) + e.GET("/user.api.User/Card", userCard) } diff --git a/example/protobuf/api.proto b/example/protobuf/api.proto index ef3d04abd..23f65d47d 100644 --- a/example/protobuf/api.proto +++ b/example/protobuf/api.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package user.api; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "google/protobuf/empty.proto"; option go_package = "api"; @@ -30,4 +31,5 @@ message InfoReply { service User { rpc Info(UserReq) returns (InfoReply); + rpc Card(UserReq) returns (google.protobuf.Empty); } diff --git a/go.mod b/go.mod index 29db6d7a3..95dcb20b3 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,9 @@ go 1.12 require ( github.com/BurntSushi/toml v0.3.1 - github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d // indirect - github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 // indirect + github.com/aristanetworks/goarista v0.0.0-20190912214011-b54698eaaca6 // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect @@ -16,24 +17,23 @@ require ( github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/go-sql-driver/mysql v1.4.1 - github.com/gogo/protobuf v1.2.1 + github.com/gogo/protobuf v1.3.0 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/mock v1.3.1 // indirect github.com/golang/protobuf v1.3.2 - github.com/google/btree v1.0.0 // indirect + github.com/google/uuid v1.1.1 // indirect github.com/gorilla/websocket v1.4.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.9.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/leodido/go-urn v1.1.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/montanaflynn/stats v0.5.0 - github.com/openzipkin/zipkin-go v0.2.0 + github.com/openzipkin/zipkin-go v0.2.1 + github.com/otokaze/mock v0.0.0-20190125081256-8282b7a7c7c3 + github.com/philchia/agollo v0.0.0-20190728085453-a95533fccea3 github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.0.0 - github.com/prometheus/client_model v0.0.0-20190220174349-fd36f4220a90 // indirect - github.com/prometheus/common v0.6.0 // indirect - github.com/prometheus/procfs v0.0.3 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c // indirect + github.com/prometheus/client_golang v1.1.0 + github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect github.com/shirou/gopsutil v2.19.6+incompatible github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 @@ -42,16 +42,16 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/tsuna/gohbase v0.0.0-20190502052937-24ffed0537aa github.com/urfave/cli v1.20.0 - go.etcd.io/etcd v0.0.0-20190720005121-fe86a786a4c3 + go.etcd.io/etcd v0.0.0-20190917205325-a14579fbfb1a go.uber.org/atomic v1.4.0 // indirect - go.uber.org/zap v1.10.0 // indirect + go.uber.org/multierr v1.2.0 // indirect golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect - golang.org/x/time v0.0.0-20190513212739-9d24e82272b4 // indirect + golang.org/x/net v0.0.0-20191011234655-491137f69257 + golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect + golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89 google.golang.org/appengine v1.6.1 // indirect - google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 - google.golang.org/grpc v1.22.0 + google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 + google.golang.org/grpc v1.24.0 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.29.1 gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index 6c202a5a2..d885b27d0 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,44 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d h1:VWP4o43LuzNbykZJzMUv5b9DWLgn0sn3GUj3RUyWMMQ= -github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0= -github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= +github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= +github.com/aristanetworks/goarista v0.0.0-20190912214011-b54698eaaca6 h1:6bZNnQcA2fkzH9AhZXbp2nDqbWa4bBqFeUb70Zq1HBM= +github.com/aristanetworks/goarista v0.0.0-20190912214011-b54698eaaca6/go.mod h1:Z4RTxGAuYhPzcq8+EdRM+R8M48Ssle2TsWtwRKa+vns= +github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238 h1:uNljlOxtOHrPnRoPPx+JanqjAGZpNiqAGVBfGskd/pg= -github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM= github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -48,7 +56,7 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -64,35 +72,35 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -102,30 +110,34 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0= -github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -136,11 +148,12 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= @@ -149,10 +162,20 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/openzipkin/zipkin-go v0.2.0 h1:33/f6xXB6YlOQ9tgTsXVOkdLCJsHTcZJnMy4DnSd6FU= -github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= +github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= +github.com/openzipkin/zipkin-go v0.2.1 h1:noL5/5Uf1HpVl3wNsfkZhIKbSWCVi5jgqkONNx8PXcA= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/otokaze/mock v0.0.0-20190125081256-8282b7a7c7c3 h1:zjmNboC3QFuMdJSaZJ7Qvi3HUxWXPdj7wb3rc4jH5HI= +github.com/otokaze/mock v0.0.0-20190125081256-8282b7a7c7c3/go.mod h1:pLR8n2aimFxvvDJ6n8JuQWthMGezCYMjuhlaTjPTZf0= +github.com/philchia/agollo v0.0.0-20190728085453-a95533fccea3 h1:e/WwwXpp+h9CtbiwSdDxrgq6ymvrvLH/P+kS+8qq4v8= +github.com/philchia/agollo v0.0.0-20190728085453-a95533fccea3/go.mod h1:EXNdWdQkS+QBi0nb/Xm+sBBuQ1PM7/NIPr1JDzOlt8A= +github.com/philchia/agollo v2.3.1+incompatible h1:C4zDDuOcP1Qynikz2rSJQSMjwexv4GfDpwBHJRinhPc= +github.com/philchia/agollo v2.3.1+incompatible/go.mod h1:EXNdWdQkS+QBi0nb/Xm+sBBuQ1PM7/NIPr1JDzOlt8A= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -160,37 +183,32 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190220174349-fd36f4220a90 h1:Cov9QkEXNhh8QeXoICvORjJ4RrpyvXmSf7rHSpS+ZfI= -github.com/prometheus/client_model v0.0.0-20190220174349-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c h1:eED6LswgZ3TfAl9fb+L2TfdSlXpYdg21iWZMdHuoSks= -github.com/remyoudompheng/bigfft v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec h1:6ncX5ko6B9LntYM0YBRXkiSaZMmLYeZ/NWcmeB43mMY= github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shirou/gopsutil v2.19.6+incompatible h1:49/Gru26Lne9Cl3IoAVDZVM09hvkSrUodgIIsCVRwbs= github.com/shirou/gopsutil v2.19.6+incompatible/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -205,6 +223,9 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= +github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= +github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/wc= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -212,23 +233,28 @@ github.com/tsuna/gohbase v0.0.0-20190502052937-24ffed0537aa h1:V/ABqiqsgqmpoIcLD github.com/tsuna/gohbase v0.0.0-20190502052937-24ffed0537aa/go.mod h1:3HfLQly3YNLGxNv/2YOfmz30vcjG9hbuME1GpxoLlGs= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= +github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20190720005121-fe86a786a4c3 h1:LlCFU/KJ9P/8QKB73kkd1z/zbm7ZJ2V4HEgEHI3N7gk= -go.etcd.io/etcd v0.0.0-20190720005121-fe86a786a4c3/go.mod h1:N0RPWo9FXJYZQI4BTkDtQylrstIigYHeR18ONnyTufk= +go.etcd.io/etcd v0.0.0-20190917205325-a14579fbfb1a h1:OpCyFK9+wUB3g4o1guENLYOUZhG4hswKiFbE+jC12Cc= +go.etcd.io/etcd v0.0.0-20190917205325-a14579fbfb1a/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4= +go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -246,8 +272,12 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc= +golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191011234655-491137f69257 h1:ry8e2D+cwaV6hk7lb3aRTjjZo24shrbK0e11QEOkTIg= +golang.org/x/net v0.0.0-20191011234655-491137f69257/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -260,51 +290,65 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8 h1:41hwlulw1prEMBxLQSlMSux1zxJf07B3WPsdjJlKZxE= +golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190513212739-9d24e82272b4 h1:RMGusaKverhgGR5KBERIKiTyWoWHRd84GCtsNlvLvIo= -golang.org/x/time v0.0.0-20190513212739-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b h1:mSUCVIwDx4hfXJfWsOPfdzEHxzb2Xjl6BQ8YgPnazQA= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89 h1:WiVZGyzQN7gPNLRkkpsNX3jC0Jx5j9GxadCZW/8eXw0= +golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0WCIICM8RkWvDw03lvB77hrhJnpxfU= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= +google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/pkg/cache/memcache/test/BUILD.bazel b/pkg/cache/memcache/test/BUILD.bazel deleted file mode 100644 index 0bf9680e6..000000000 --- a/pkg/cache/memcache/test/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) -load( - "@io_bazel_rules_go//proto:def.bzl", - "go_proto_library", -) - -go_library( - name = "go_default_library", - srcs = [], - embed = [":proto_go_proto"], - importpath = "go-common/library/cache/memcache/test", - tags = ["automanaged"], - visibility = ["//visibility:public"], - deps = ["@com_github_golang_protobuf//proto:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -proto_library( - name = "test_proto", - srcs = ["test.proto"], - import_prefix = "go-common/library/cache/memcache/test", - strip_import_prefix = "", - tags = ["automanaged"], -) - -go_proto_library( - name = "proto_go_proto", - compilers = ["@io_bazel_rules_go//proto:go_proto"], - importpath = "go-common/library/cache/memcache/test", - proto = ":test_proto", - tags = ["automanaged"], -) diff --git a/pkg/cache/memcache/test/docker-compose.yaml b/pkg/cache/memcache/test/docker-compose.yaml new file mode 100755 index 000000000..ace3ebedf --- /dev/null +++ b/pkg/cache/memcache/test/docker-compose.yaml @@ -0,0 +1,9 @@ +version: "3.7" + +services: + mc: + image: memcached:1 + ports: + - 11211:11211 + + diff --git a/pkg/cache/metrics.go b/pkg/cache/metrics.go index 9203b9bc3..649df52fd 100644 --- a/pkg/cache/metrics.go +++ b/pkg/cache/metrics.go @@ -4,6 +4,7 @@ import "github.com/bilibili/kratos/pkg/stat/metric" const _metricNamespace = "cache" +// be used in tool/kratos-gen-bts var ( MetricHits = metric.NewCounterVec(&metric.CounterVecOpts{ Namespace: _metricNamespace, diff --git a/pkg/cache/redis/commandinfo_test.go b/pkg/cache/redis/commandinfo_test.go new file mode 100644 index 000000000..d8f4e5214 --- /dev/null +++ b/pkg/cache/redis/commandinfo_test.go @@ -0,0 +1,27 @@ +package redis + +import "testing" + +func TestLookupCommandInfo(t *testing.T) { + for _, n := range []string{"watch", "WATCH", "wAtch"} { + if LookupCommandInfo(n) == (CommandInfo{}) { + t.Errorf("LookupCommandInfo(%q) = CommandInfo{}, expected non-zero value", n) + } + } +} + +func benchmarkLookupCommandInfo(b *testing.B, names ...string) { + for i := 0; i < b.N; i++ { + for _, c := range names { + LookupCommandInfo(c) + } + } +} + +func BenchmarkLookupCommandInfoCorrectCase(b *testing.B) { + benchmarkLookupCommandInfo(b, "watch", "WATCH", "monitor", "MONITOR") +} + +func BenchmarkLookupCommandInfoMixedCase(b *testing.B) { + benchmarkLookupCommandInfo(b, "wAtch", "WeTCH", "monItor", "MONiTOR") +} diff --git a/pkg/cache/redis/conn.go b/pkg/cache/redis/conn.go index 3f5a5a454..949a4bd55 100644 --- a/pkg/cache/redis/conn.go +++ b/pkg/cache/redis/conn.go @@ -30,6 +30,33 @@ import ( "github.com/pkg/errors" ) +// Conn represents a connection to a Redis server. +type Conn interface { + // Close closes the connection. + Close() error + + // Err returns a non-nil value if the connection is broken. The returned + // value is either the first non-nil value returned from the underlying + // network connection or a protocol parsing error. Applications should + // close broken connections. + Err() error + + // Do sends a command to the server and returns the received reply. + Do(commandName string, args ...interface{}) (reply interface{}, err error) + + // Send writes the command to the client's output buffer. + Send(commandName string, args ...interface{}) error + + // Flush flushes the output buffer to the Redis server. + Flush() error + + // Receive receives a single reply from the Redis server + Receive() (reply interface{}, err error) + + // WithContext returns Conn with the input ctx. + WithContext(ctx context.Context) Conn +} + // conn is the low-level implementation of Conn type conn struct { // Shared @@ -38,6 +65,8 @@ type conn struct { err error conn net.Conn + ctx context.Context + // Read readTimeout time.Duration br *bufio.Reader @@ -226,6 +255,7 @@ func NewConn(c *Config) (cn Conn, err error) { func (c *conn) Close() error { c.mu.Lock() + c.ctx = nil err := c.err if c.err == nil { c.err = errors.New("redigo: closed") @@ -295,7 +325,7 @@ func (c *conn) writeFloat64(n float64) error { func (c *conn) writeCommand(cmd string, args []interface{}) (err error) { if c.writeTimeout != 0 { - c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + c.conn.SetWriteDeadline(shrinkDeadline(c.ctx, c.writeTimeout)) } c.writeLen('*', 1+len(args)) err = c.writeString(cmd) @@ -478,7 +508,7 @@ func (c *conn) Send(cmd string, args ...interface{}) (err error) { func (c *conn) Flush() (err error) { if c.writeTimeout != 0 { - c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) + c.conn.SetWriteDeadline(shrinkDeadline(c.ctx, c.writeTimeout)) } if err = c.bw.Flush(); err != nil { c.fatal(err) @@ -488,7 +518,7 @@ func (c *conn) Flush() (err error) { func (c *conn) Receive() (reply interface{}, err error) { if c.readTimeout != 0 { - c.conn.SetReadDeadline(time.Now().Add(c.readTimeout)) + c.conn.SetReadDeadline(shrinkDeadline(c.ctx, c.readTimeout)) } if reply, err = c.readReply(); err != nil { return nil, c.fatal(err) @@ -511,7 +541,7 @@ func (c *conn) Receive() (reply interface{}, err error) { return } -func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { +func (c *conn) Do(cmd string, args ...interface{}) (reply interface{}, err error) { c.mu.Lock() pending := c.pending c.pending = 0 @@ -519,7 +549,7 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { if cmd == "" && pending == 0 { return nil, nil } - var err error + if cmd != "" { err = c.writeCommand(cmd, args) } @@ -530,7 +560,7 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { return nil, c.fatal(err) } if c.readTimeout != 0 { - c.conn.SetReadDeadline(time.Now().Add(c.readTimeout)) + c.conn.SetReadDeadline(shrinkDeadline(c.ctx, c.readTimeout)) } if cmd == "" { reply := make([]interface{}, pending) @@ -548,7 +578,6 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { return reply, nil } - var reply interface{} for i := 0; i <= pending; i++ { var e error if reply, e = c.readReply(); e != nil { @@ -561,5 +590,20 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) { return reply, err } -// WithContext FIXME: implement WithContext -func (c *conn) WithContext(ctx context.Context) Conn { return c } +func (c *conn) copy() *conn { + return &conn{ + pending: c.pending, + err: c.err, + conn: c.conn, + bw: c.bw, + br: c.br, + readTimeout: c.readTimeout, + writeTimeout: c.writeTimeout, + } +} + +func (c *conn) WithContext(ctx context.Context) Conn { + c2 := c.copy() + c2.ctx = ctx + return c2 +} diff --git a/pkg/cache/redis/conn_test.go b/pkg/cache/redis/conn_test.go new file mode 100644 index 000000000..3e37e882c --- /dev/null +++ b/pkg/cache/redis/conn_test.go @@ -0,0 +1,670 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "bytes" + "context" + "io" + "math" + "net" + "os" + "reflect" + "strings" + "testing" + "time" +) + +type tConn struct { + io.Reader + io.Writer +} + +func (*tConn) Close() error { return nil } +func (*tConn) LocalAddr() net.Addr { return nil } +func (*tConn) RemoteAddr() net.Addr { return nil } +func (*tConn) SetDeadline(t time.Time) error { return nil } +func (*tConn) SetReadDeadline(t time.Time) error { return nil } +func (*tConn) SetWriteDeadline(t time.Time) error { return nil } + +func dialTestConn(r io.Reader, w io.Writer) DialOption { + return DialNetDial(func(net, addr string) (net.Conn, error) { + return &tConn{Reader: r, Writer: w}, nil + }) +} + +var writeTests = []struct { + args []interface{} + expected string +}{ + { + []interface{}{"SET", "key", "value"}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n", + }, + { + []interface{}{"SET", "key", "value"}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n", + }, + { + []interface{}{"SET", "key", byte(100)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n", + }, + { + []interface{}{"SET", "key", 100}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n", + }, + { + []interface{}{"SET", "key", int64(math.MinInt64)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$20\r\n-9223372036854775808\r\n", + }, + { + []interface{}{"SET", "key", float64(1349673917.939762)}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$21\r\n1.349673917939762e+09\r\n", + }, + { + []interface{}{"SET", "key", ""}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n", + }, + { + []interface{}{"SET", "key", nil}, + "*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n", + }, + { + []interface{}{"ECHO", true, false}, + "*3\r\n$4\r\nECHO\r\n$1\r\n1\r\n$1\r\n0\r\n", + }, +} + +func TestWrite(t *testing.T) { + for _, tt := range writeTests { + var buf bytes.Buffer + c, _ := Dial("", "", dialTestConn(nil, &buf)) + err := c.Send(tt.args[0].(string), tt.args[1:]...) + if err != nil { + t.Errorf("Send(%v) returned error %v", tt.args, err) + continue + } + c.Flush() + actual := buf.String() + if actual != tt.expected { + t.Errorf("Send(%v) = %q, want %q", tt.args, actual, tt.expected) + } + } +} + +var errorSentinel = &struct{}{} + +var readTests = []struct { + reply string + expected interface{} +}{ + { + "+OK\r\n", + "OK", + }, + { + "+PONG\r\n", + "PONG", + }, + { + "@OK\r\n", + errorSentinel, + }, + { + "$6\r\nfoobar\r\n", + []byte("foobar"), + }, + { + "$-1\r\n", + nil, + }, + { + ":1\r\n", + int64(1), + }, + { + ":-2\r\n", + int64(-2), + }, + { + "*0\r\n", + []interface{}{}, + }, + { + "*-1\r\n", + nil, + }, + { + "*4\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$5\r\nHello\r\n$5\r\nWorld\r\n", + []interface{}{[]byte("foo"), []byte("bar"), []byte("Hello"), []byte("World")}, + }, + { + "*3\r\n$3\r\nfoo\r\n$-1\r\n$3\r\nbar\r\n", + []interface{}{[]byte("foo"), nil, []byte("bar")}, + }, + + { + // "x" is not a valid length + "$x\r\nfoobar\r\n", + errorSentinel, + }, + { + // -2 is not a valid length + "$-2\r\n", + errorSentinel, + }, + { + // "x" is not a valid integer + ":x\r\n", + errorSentinel, + }, + { + // missing \r\n following value + "$6\r\nfoobar", + errorSentinel, + }, + { + // short value + "$6\r\nxx", + errorSentinel, + }, + { + // long value + "$6\r\nfoobarx\r\n", + errorSentinel, + }, +} + +func TestRead(t *testing.T) { + for _, tt := range readTests { + c, _ := Dial("", "", dialTestConn(strings.NewReader(tt.reply), nil)) + actual, err := c.Receive() + if tt.expected == errorSentinel { + if err == nil { + t.Errorf("Receive(%q) did not return expected error", tt.reply) + } + } else { + if err != nil { + t.Errorf("Receive(%q) returned error %v", tt.reply, err) + continue + } + if !reflect.DeepEqual(actual, tt.expected) { + t.Errorf("Receive(%q) = %v, want %v", tt.reply, actual, tt.expected) + } + } + } +} + +var testCommands = []struct { + args []interface{} + expected interface{} +}{ + { + []interface{}{"PING"}, + "PONG", + }, + { + []interface{}{"SET", "foo", "bar"}, + "OK", + }, + { + []interface{}{"GET", "foo"}, + []byte("bar"), + }, + { + []interface{}{"GET", "nokey"}, + nil, + }, + { + []interface{}{"MGET", "nokey", "foo"}, + []interface{}{nil, []byte("bar")}, + }, + { + []interface{}{"INCR", "mycounter"}, + int64(1), + }, + { + []interface{}{"LPUSH", "mylist", "foo"}, + int64(1), + }, + { + []interface{}{"LPUSH", "mylist", "bar"}, + int64(2), + }, + { + []interface{}{"LRANGE", "mylist", 0, -1}, + []interface{}{[]byte("bar"), []byte("foo")}, + }, + { + []interface{}{"MULTI"}, + "OK", + }, + { + []interface{}{"LRANGE", "mylist", 0, -1}, + "QUEUED", + }, + { + []interface{}{"PING"}, + "QUEUED", + }, + { + []interface{}{"EXEC"}, + []interface{}{ + []interface{}{[]byte("bar"), []byte("foo")}, + "PONG", + }, + }, +} + +func TestDoCommands(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + for _, cmd := range testCommands { + actual, err := c.Do(cmd.args[0].(string), cmd.args[1:]...) + if err != nil { + t.Errorf("Do(%v) returned error %v", cmd.args, err) + continue + } + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Do(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } +} + +func TestPipelineCommands(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + for _, cmd := range testCommands { + if err := c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil { + t.Fatalf("Send(%v) returned error %v", cmd.args, err) + } + } + if err := c.Flush(); err != nil { + t.Errorf("Flush() returned error %v", err) + } + for _, cmd := range testCommands { + actual, err := c.Receive() + if err != nil { + t.Fatalf("Receive(%v) returned error %v", cmd.args, err) + } + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } +} + +func TestBlankCommmand(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + for _, cmd := range testCommands { + if err = c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil { + t.Fatalf("Send(%v) returned error %v", cmd.args, err) + } + } + reply, err := Values(c.Do("")) + if err != nil { + t.Fatalf("Do() returned error %v", err) + } + if len(reply) != len(testCommands) { + t.Fatalf("len(reply)=%d, want %d", len(reply), len(testCommands)) + } + for i, cmd := range testCommands { + actual := reply[i] + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } +} + +func TestRecvBeforeSend(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + done := make(chan struct{}) + go func() { + c.Receive() + close(done) + }() + time.Sleep(time.Millisecond) + c.Send("PING") + c.Flush() + <-done + _, err = c.Do("") + if err != nil { + t.Fatalf("error=%v", err) + } +} + +func TestError(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + c.Do("SET", "key", "val") + _, err = c.Do("HSET", "key", "fld", "val") + if err == nil { + t.Errorf("Expected err for HSET on string key.") + } + if c.Err() != nil { + t.Errorf("Conn has Err()=%v, expect nil", c.Err()) + } + _, err = c.Do("SET", "key", "val") + if err != nil { + t.Errorf("Do(SET, key, val) returned error %v, expected nil.", err) + } +} + +func TestReadTimeout(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen returned %v", err) + } + defer l.Close() + + go func() { + for { + c, err1 := l.Accept() + if err1 != nil { + return + } + go func() { + time.Sleep(time.Second) + c.Write([]byte("+OK\r\n")) + c.Close() + }() + } + }() + + // Do + + c1, err := Dial(l.Addr().Network(), l.Addr().String(), DialReadTimeout(time.Millisecond)) + if err != nil { + t.Fatalf("Dial returned %v", err) + } + defer c1.Close() + + _, err = c1.Do("PING") + if err == nil { + t.Fatalf("c1.Do() returned nil, expect error") + } + if c1.Err() == nil { + t.Fatalf("c1.Err() = nil, expect error") + } + + // Send/Flush/Receive + + c2, err := Dial(l.Addr().Network(), l.Addr().String(), DialReadTimeout(time.Millisecond)) + if err != nil { + t.Fatalf("Dial returned %v", err) + } + defer c2.Close() + + c2.Send("PING") + c2.Flush() + _, err = c2.Receive() + if err == nil { + t.Fatalf("c2.Receive() returned nil, expect error") + } + if c2.Err() == nil { + t.Fatalf("c2.Err() = nil, expect error") + } +} + +var dialErrors = []struct { + rawurl string + expectedError string +}{ + { + "localhost", + "invalid redis URL scheme", + }, + // The error message for invalid hosts is diffferent in different + // versions of Go, so just check that there is an error message. + { + "redis://weird url", + "", + }, + { + "redis://foo:bar:baz", + "", + }, + { + "http://www.google.com", + "invalid redis URL scheme: http", + }, + { + "redis://localhost:6379/abc123", + "invalid database: abc123", + }, +} + +func TestDialURLErrors(t *testing.T) { + for _, d := range dialErrors { + _, err := DialURL(d.rawurl) + if err == nil || !strings.Contains(err.Error(), d.expectedError) { + t.Errorf("DialURL did not return expected error (expected %v to contain %s)", err, d.expectedError) + } + } +} + +func TestDialURLPort(t *testing.T) { + checkPort := func(network, address string) (net.Conn, error) { + if address != "localhost:6379" { + t.Errorf("DialURL did not set port to 6379 by default (got %v)", address) + } + return nil, nil + } + _, err := DialURL("redis://localhost", DialNetDial(checkPort)) + if err != nil { + t.Error("dial error:", err) + } +} + +func TestDialURLHost(t *testing.T) { + checkHost := func(network, address string) (net.Conn, error) { + if address != "localhost:6379" { + t.Errorf("DialURL did not set host to localhost by default (got %v)", address) + } + return nil, nil + } + _, err := DialURL("redis://:6379", DialNetDial(checkHost)) + if err != nil { + t.Error("dial error:", err) + } +} + +func TestDialURLPassword(t *testing.T) { + var buf bytes.Buffer + _, err := DialURL("redis://x:abc123@localhost", dialTestConn(strings.NewReader("+OK\r\n"), &buf)) + if err != nil { + t.Error("dial error:", err) + } + expected := "*2\r\n$4\r\nAUTH\r\n$6\r\nabc123\r\n" + actual := buf.String() + if actual != expected { + t.Errorf("commands = %q, want %q", actual, expected) + } +} + +func TestDialURLDatabase(t *testing.T) { + var buf bytes.Buffer + _, err := DialURL("redis://localhost/3", dialTestConn(strings.NewReader("+OK\r\n"), &buf)) + if err != nil { + t.Error("dial error:", err) + } + expected := "*2\r\n$6\r\nSELECT\r\n$1\r\n3\r\n" + actual := buf.String() + if actual != expected { + t.Errorf("commands = %q, want %q", actual, expected) + } +} + +// Connect to local instance of Redis running on the default port. +func ExampleDial() { + c, err := Dial("tcp", ":6379") + if err != nil { + // handle error + } + defer c.Close() +} + +// Connect to remote instance of Redis using a URL. +func ExampleDialURL() { + c, err := DialURL(os.Getenv("REDIS_URL")) + if err != nil { + // handle connection error + } + defer c.Close() +} + +// TextExecError tests handling of errors in a transaction. See +// http://io/topics/transactions for information on how Redis handles +// errors in a transaction. +func TestExecError(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + // Execute commands that fail before EXEC is called. + + c.Do("DEL", "k0") + c.Do("ZADD", "k0", 0, 0) + c.Send("MULTI") + c.Send("NOTACOMMAND", "k0", 0, 0) + c.Send("ZINCRBY", "k0", 0, 0) + v, err := c.Do("EXEC") + if err == nil { + t.Fatalf("EXEC returned values %v, expected error", v) + } + + // Execute commands that fail after EXEC is called. The first command + // returns an error. + + c.Do("DEL", "k1") + c.Do("ZADD", "k1", 0, 0) + c.Send("MULTI") + c.Send("HSET", "k1", 0, 0) + c.Send("ZINCRBY", "k1", 0, 0) + v, err = c.Do("EXEC") + if err != nil { + t.Fatalf("EXEC returned error %v", err) + } + + vs, err := Values(v, nil) + if err != nil { + t.Fatalf("Values(v) returned error %v", err) + } + + if len(vs) != 2 { + t.Fatalf("len(vs) == %d, want 2", len(vs)) + } + + if _, ok := vs[0].(error); !ok { + t.Fatalf("first result is type %T, expected error", vs[0]) + } + + if _, ok := vs[1].([]byte); !ok { + t.Fatalf("second result is type %T, expected []byte", vs[1]) + } + + // Execute commands that fail after EXEC is called. The second command + // returns an error. + + c.Do("ZADD", "k2", 0, 0) + c.Send("MULTI") + c.Send("ZINCRBY", "k2", 0, 0) + c.Send("HSET", "k2", 0, 0) + v, err = c.Do("EXEC") + if err != nil { + t.Fatalf("EXEC returned error %v", err) + } + + vs, err = Values(v, nil) + if err != nil { + t.Fatalf("Values(v) returned error %v", err) + } + + if len(vs) != 2 { + t.Fatalf("len(vs) == %d, want 2", len(vs)) + } + + if _, ok := vs[0].([]byte); !ok { + t.Fatalf("first result is type %T, expected []byte", vs[0]) + } + + if _, ok := vs[1].(error); !ok { + t.Fatalf("second result is type %T, expected error", vs[2]) + } +} + +func BenchmarkDoEmpty(b *testing.B) { + c, err := DialDefaultServer() + if err != nil { + b.Fatal(err) + } + defer c.Close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := c.Do(""); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDoPing(b *testing.B) { + c, err := DialDefaultServer() + if err != nil { + b.Fatal(err) + } + defer c.Close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := c.Do("PING"); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkConn(b *testing.B) { + for i := 0; i < b.N; i++ { + c, err := DialDefaultServer() + if err != nil { + b.Fatal(err) + } + c2 := c.WithContext(context.TODO()) + if _, err := c2.Do("PING"); err != nil { + b.Fatal(err) + } + c2.Close() + } +} diff --git a/pkg/cache/redis/log.go b/pkg/cache/redis/log.go index 129b86d67..487a1408f 100644 --- a/pkg/cache/redis/log.go +++ b/pkg/cache/redis/log.go @@ -16,16 +16,18 @@ package redis import ( "bytes" + "context" "fmt" "log" ) // NewLoggingConn returns a logging wrapper around a connection. +// ATTENTION: ONLY use loggingConn in developing, DO NOT use this in production. func NewLoggingConn(conn Conn, logger *log.Logger, prefix string) Conn { if prefix != "" { prefix = prefix + "." } - return &loggingConn{conn, logger, prefix} + return &loggingConn{Conn: conn, logger: logger, prefix: prefix} } type loggingConn struct { @@ -98,16 +100,16 @@ func (c *loggingConn) print(method, commandName string, args []interface{}, repl c.logger.Output(3, buf.String()) } -func (c *loggingConn) Do(commandName string, args ...interface{}) (interface{}, error) { - reply, err := c.Conn.Do(commandName, args...) +func (c *loggingConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { + reply, err = c.Conn.Do(commandName, args...) c.print("Do", commandName, args, reply, err) return reply, err } -func (c *loggingConn) Send(commandName string, args ...interface{}) error { - err := c.Conn.Send(commandName, args...) +func (c *loggingConn) Send(commandName string, args ...interface{}) (err error) { + err = c.Conn.Send(commandName, args...) c.print("Send", commandName, args, nil, err) - return err + return } func (c *loggingConn) Receive() (interface{}, error) { @@ -115,3 +117,7 @@ func (c *loggingConn) Receive() (interface{}, error) { c.print("Receive", "", nil, reply, err) return reply, err } + +func (c *loggingConn) WithContext(ctx context.Context) Conn { + return c +} diff --git a/pkg/cache/redis/main_test.go b/pkg/cache/redis/main_test.go new file mode 100644 index 000000000..3164628f2 --- /dev/null +++ b/pkg/cache/redis/main_test.go @@ -0,0 +1,67 @@ +package redis + +import ( + "flag" + "os" + "testing" + "time" + + "github.com/bilibili/kratos/pkg/container/pool" + "github.com/bilibili/kratos/pkg/testing/lich" + xtime "github.com/bilibili/kratos/pkg/time" +) + +var ( + testRedisAddr string + testPool *Pool + testConfig *Config +) + +func setupTestConfig(addr string) { + c := getTestConfig(addr) + c.Config = &pool.Config{ + Active: 20, + Idle: 2, + IdleTimeout: xtime.Duration(90 * time.Second), + } + testConfig = c +} + +func getTestConfig(addr string) *Config { + return &Config{ + Name: "test", + Proto: "tcp", + Addr: addr, + DialTimeout: xtime.Duration(time.Second), + ReadTimeout: xtime.Duration(time.Second), + WriteTimeout: xtime.Duration(time.Second), + } +} + +func setupTestPool() { + testPool = NewPool(testConfig) +} + +// DialDefaultServer starts the test server if not already started and dials a +// connection to the server. +func DialDefaultServer() (Conn, error) { + c, err := Dial("tcp", testRedisAddr, DialReadTimeout(1*time.Second), DialWriteTimeout(1*time.Second)) + if err != nil { + return nil, err + } + c.Do("FLUSHDB") + return c, nil +} + +func TestMain(m *testing.M) { + flag.Set("f", "./test/docker-compose.yaml") + if err := lich.Setup(); err != nil { + panic(err) + } + defer lich.Teardown() + testRedisAddr = "localhost:6379" + setupTestConfig(testRedisAddr) + setupTestPool() + ret := m.Run() + os.Exit(ret) +} diff --git a/pkg/cache/redis/metrics.go b/pkg/cache/redis/metrics.go index e48795957..2037ce48a 100644 --- a/pkg/cache/redis/metrics.go +++ b/pkg/cache/redis/metrics.go @@ -1,6 +1,8 @@ package redis -import "github.com/bilibili/kratos/pkg/stat/metric" +import ( + "github.com/bilibili/kratos/pkg/stat/metric" +) const namespace = "redis_client" diff --git a/pkg/cache/redis/mock.go b/pkg/cache/redis/mock.go index da75817f3..fc9d5a3da 100644 --- a/pkg/cache/redis/mock.go +++ b/pkg/cache/redis/mock.go @@ -1,8 +1,6 @@ package redis -import ( - "context" -) +import "context" // MockErr for unit test. type MockErr struct { diff --git a/pkg/cache/redis/pipeline.go b/pkg/cache/redis/pipeline.go new file mode 100644 index 000000000..0a23205e5 --- /dev/null +++ b/pkg/cache/redis/pipeline.go @@ -0,0 +1,85 @@ +package redis + +import ( + "context" + "errors" +) + +type Pipeliner interface { + // Send writes the command to the client's output buffer. + Send(commandName string, args ...interface{}) + + // Exec executes all commands and get replies. + Exec(ctx context.Context) (rs *Replies, err error) +} + +var ( + ErrNoReply = errors.New("redis: no reply in result set") +) + +type pipeliner struct { + pool *Pool + cmds []*cmd +} + +type Replies struct { + replies []*reply +} + +type reply struct { + reply interface{} + err error +} + +func (rs *Replies) Next() bool { + return len(rs.replies) > 0 +} + +func (rs *Replies) Scan() (reply interface{}, err error) { + if !rs.Next() { + return nil, ErrNoReply + } + reply, err = rs.replies[0].reply, rs.replies[0].err + rs.replies = rs.replies[1:] + return +} + +type cmd struct { + commandName string + args []interface{} +} + +func (p *pipeliner) Send(commandName string, args ...interface{}) { + p.cmds = append(p.cmds, &cmd{commandName: commandName, args: args}) + return +} + +func (p *pipeliner) Exec(ctx context.Context) (rs *Replies, err error) { + n := len(p.cmds) + if n == 0 { + return &Replies{}, nil + } + c := p.pool.Get(ctx) + defer c.Close() + for len(p.cmds) > 0 { + cmd := p.cmds[0] + p.cmds = p.cmds[1:] + if err := c.Send(cmd.commandName, cmd.args...); err != nil { + p.cmds = p.cmds[:0] + return nil, err + } + } + if err = c.Flush(); err != nil { + p.cmds = p.cmds[:0] + return nil, err + } + rps := make([]*reply, 0, n) + for i := 0; i < n; i++ { + rp, err := c.Receive() + rps = append(rps, &reply{reply: rp, err: err}) + } + rs = &Replies{ + replies: rps, + } + return +} diff --git a/pkg/cache/redis/pipeline_test.go b/pkg/cache/redis/pipeline_test.go new file mode 100644 index 000000000..d3d95d260 --- /dev/null +++ b/pkg/cache/redis/pipeline_test.go @@ -0,0 +1,96 @@ +package redis + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + "github.com/bilibili/kratos/pkg/container/pool" + xtime "github.com/bilibili/kratos/pkg/time" +) + +func TestRedis_Pipeline(t *testing.T) { + conf := &Config{ + Name: "test", + Proto: "tcp", + Addr: testRedisAddr, + DialTimeout: xtime.Duration(1 * time.Second), + ReadTimeout: xtime.Duration(1 * time.Second), + WriteTimeout: xtime.Duration(1 * time.Second), + } + conf.Config = &pool.Config{ + Active: 10, + Idle: 2, + IdleTimeout: xtime.Duration(90 * time.Second), + } + + r := NewRedis(conf) + r.Do(context.TODO(), "FLUSHDB") + + p := r.Pipeline() + + for _, cmd := range testCommands { + p.Send(cmd.args[0].(string), cmd.args[1:]...) + } + + replies, err := p.Exec(context.TODO()) + + i := 0 + for replies.Next() { + cmd := testCommands[i] + actual, err := replies.Scan() + if err != nil { + t.Fatalf("Receive(%v) returned error %v", cmd.args, err) + } + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + i++ + } + err = r.Close() + if err != nil { + t.Errorf("Close() error %v", err) + } +} + +func ExamplePipeliner() { + r := NewRedis(testConfig) + defer r.Close() + + pip := r.Pipeline() + pip.Send("SET", "hello", "world") + pip.Send("GET", "hello") + replies, err := pip.Exec(context.TODO()) + if err != nil { + fmt.Printf("%#v\n", err) + } + for replies.Next() { + s, err := String(replies.Scan()) + if err != nil { + fmt.Printf("err %#v\n", err) + } + fmt.Printf("%#v\n", s) + } + // Output: + // "OK" + // "world" +} + +func BenchmarkRedisPipelineExec(b *testing.B) { + r := NewRedis(testConfig) + defer r.Close() + + r.Do(context.TODO(), "SET", "abcde", "fghiasdfasdf") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + p := r.Pipeline() + p.Send("GET", "abcde") + _, err := p.Exec(context.TODO()) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/pkg/cache/redis/pool.go b/pkg/cache/redis/pool.go index ad1beef35..cd64de6a5 100644 --- a/pkg/cache/redis/pool.go +++ b/pkg/cache/redis/pool.go @@ -45,24 +45,14 @@ type Pool struct { statfunc func(name, addr, cmd string, t time.Time, err error) func() } -// Config client settings. -type Config struct { - *pool.Config - - Name string // redis name, for trace - Proto string - Addr string - Auth string - DialTimeout xtime.Duration - ReadTimeout xtime.Duration - WriteTimeout xtime.Duration -} - // NewPool creates a new pool. func NewPool(c *Config, options ...DialOption) (p *Pool) { if c.DialTimeout <= 0 || c.ReadTimeout <= 0 || c.WriteTimeout <= 0 { panic("must config redis timeout") } + if c.SlowLog <= 0 { + c.SlowLog = xtime.Duration(250 * time.Millisecond) + } ops := []DialOption{ DialConnectTimeout(time.Duration(c.DialTimeout)), DialReadTimeout(time.Duration(c.ReadTimeout)), @@ -71,12 +61,18 @@ func NewPool(c *Config, options ...DialOption) (p *Pool) { } ops = append(ops, options...) p1 := pool.NewSlice(c.Config) + + // new pool p1.New = func(ctx context.Context) (io.Closer, error) { conn, err := Dial(c.Proto, c.Addr, ops...) if err != nil { return nil, err } - return &traceConn{Conn: conn, connTags: []trace.Tag{trace.TagString(trace.TagPeerAddress, c.Addr)}}, nil + return &traceConn{ + Conn: conn, + connTags: []trace.Tag{trace.TagString(trace.TagPeerAddress, c.Addr)}, + slowLogThreshold: time.Duration(c.SlowLog), + }, nil } p = &Pool{Slice: p1, c: c, statfunc: pstat} return @@ -93,7 +89,7 @@ func (p *Pool) Get(ctx context.Context) Conn { return errorConnection{err} } c1, _ := c.(Conn) - return &pooledConnection{p: p, c: c1.WithContext(ctx), ctx: ctx, now: beginTime} + return &pooledConnection{p: p, c: c1.WithContext(ctx), rc: c1, now: beginTime} } // Close releases the resources used by the pool. @@ -103,12 +99,12 @@ func (p *Pool) Close() error { type pooledConnection struct { p *Pool + rc Conn c Conn state int now time.Time cmds []string - ctx context.Context } var ( @@ -180,7 +176,7 @@ func (pc *pooledConnection) Close() error { } } _, err := c.Do("") - pc.p.Slice.Put(context.Background(), c, pc.state != 0 || c.Err() != nil) + pc.p.Slice.Put(context.Background(), pc.rc, pc.state != 0 || c.Err() != nil) return err } @@ -193,7 +189,9 @@ func (pc *pooledConnection) Do(commandName string, args ...interface{}) (reply i ci := LookupCommandInfo(commandName) pc.state = (pc.state | ci.Set) &^ ci.Clear reply, err = pc.c.Do(commandName, args...) - pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, commandName, now, err)() + if pc.p.statfunc != nil { + pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, commandName, now, err)() + } return } @@ -217,13 +215,14 @@ func (pc *pooledConnection) Receive() (reply interface{}, err error) { if len(pc.cmds) > 0 { cmd := pc.cmds[0] pc.cmds = pc.cmds[1:] - pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, cmd, pc.now, err)() + if pc.p.statfunc != nil { + pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, cmd, pc.now, err)() + } } return } func (pc *pooledConnection) WithContext(ctx context.Context) Conn { - pc.ctx = ctx return pc } diff --git a/pkg/cache/redis/pool_test.go b/pkg/cache/redis/pool_test.go new file mode 100644 index 000000000..fdea2a337 --- /dev/null +++ b/pkg/cache/redis/pool_test.go @@ -0,0 +1,540 @@ +// Copyright 2011 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "context" + "errors" + "io" + "reflect" + "sync" + "testing" + "time" + + "github.com/bilibili/kratos/pkg/container/pool" +) + +type poolTestConn struct { + d *poolDialer + err error + c Conn + ctx context.Context +} + +func (c *poolTestConn) Flush() error { + return c.c.Flush() +} + +func (c *poolTestConn) Receive() (reply interface{}, err error) { + return c.c.Receive() +} + +func (c *poolTestConn) WithContext(ctx context.Context) Conn { + c.c.WithContext(ctx) + c.ctx = ctx + return c +} + +func (c *poolTestConn) Close() error { + c.d.mu.Lock() + c.d.open-- + c.d.mu.Unlock() + return c.c.Close() +} + +func (c *poolTestConn) Err() error { return c.err } + +func (c *poolTestConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { + if commandName == "ERR" { + c.err = args[0].(error) + commandName = "PING" + } + if commandName != "" { + c.d.commands = append(c.d.commands, commandName) + } + return c.c.Do(commandName, args...) +} + +func (c *poolTestConn) Send(commandName string, args ...interface{}) error { + c.d.commands = append(c.d.commands, commandName) + return c.c.Send(commandName, args...) +} + +type poolDialer struct { + mu sync.Mutex + t *testing.T + dialed int + open int + commands []string + dialErr error +} + +func (d *poolDialer) dial() (Conn, error) { + d.mu.Lock() + d.dialed += 1 + dialErr := d.dialErr + d.mu.Unlock() + if dialErr != nil { + return nil, d.dialErr + } + c, err := DialDefaultServer() + if err != nil { + return nil, err + } + d.mu.Lock() + d.open += 1 + d.mu.Unlock() + return &poolTestConn{d: d, c: c}, nil +} + +func (d *poolDialer) check(message string, p *Pool, dialed, open int) { + d.mu.Lock() + if d.dialed != dialed { + d.t.Errorf("%s: dialed=%d, want %d", message, d.dialed, dialed) + } + if d.open != open { + d.t.Errorf("%s: open=%d, want %d", message, d.open, open) + } + // if active := p.ActiveCount(); active != open { + // d.t.Errorf("%s: active=%d, want %d", message, active, open) + // } + d.mu.Unlock() +} + +func TestPoolReuse(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + var err error + + for i := 0; i < 10; i++ { + c1 := p.Get(context.TODO()) + c1.Do("PING") + c2 := p.Get(context.TODO()) + c2.Do("PING") + c1.Close() + c2.Close() + + } + + d.check("before close", p, 2, 2) + err = p.Close() + if err != nil { + t.Fatal(err) + } + d.check("after close", p, 2, 0) +} + +func TestPoolMaxIdle(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + for i := 0; i < 10; i++ { + c1 := p.Get(context.TODO()) + c1.Do("PING") + c2 := p.Get(context.TODO()) + c2.Do("PING") + c3 := p.Get(context.TODO()) + c3.Do("PING") + c1.Close() + c2.Close() + c3.Close() + } + d.check("before close", p, 12, 2) + p.Close() + d.check("after close", p, 12, 0) +} + +func TestPoolError(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + c := p.Get(context.TODO()) + c.Do("ERR", io.EOF) + if c.Err() == nil { + t.Errorf("expected c.Err() != nil") + } + c.Close() + + c = p.Get(context.TODO()) + c.Do("ERR", io.EOF) + c.Close() + + d.check(".", p, 2, 0) +} + +func TestPoolClose(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + c1 := p.Get(context.TODO()) + c1.Do("PING") + c2 := p.Get(context.TODO()) + c2.Do("PING") + c3 := p.Get(context.TODO()) + c3.Do("PING") + + c1.Close() + if _, err := c1.Do("PING"); err == nil { + t.Errorf("expected error after connection closed") + } + + c2.Close() + c2.Close() + + p.Close() + + d.check("after pool close", p, 3, 1) + + if _, err := c1.Do("PING"); err == nil { + t.Errorf("expected error after connection and pool closed") + } + + c3.Close() + + d.check("after conn close", p, 3, 0) + + c1 = p.Get(context.TODO()) + if _, err := c1.Do("PING"); err == nil { + t.Errorf("expected error after pool closed") + } +} + +func TestPoolConcurrenSendReceive(t *testing.T) { + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return DialDefaultServer() + } + defer p.Close() + + c := p.Get(context.TODO()) + done := make(chan error, 1) + go func() { + _, err := c.Receive() + done <- err + }() + c.Send("PING") + c.Flush() + err := <-done + if err != nil { + t.Fatalf("Receive() returned error %v", err) + } + _, err = c.Do("") + if err != nil { + t.Fatalf("Do() returned error %v", err) + } + c.Close() +} + +func TestPoolMaxActive(t *testing.T) { + d := poolDialer{t: t} + conf := getTestConfig(testRedisAddr) + conf.Config = &pool.Config{ + Active: 2, + Idle: 2, + } + p := NewPool(conf) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + c1 := p.Get(context.TODO()) + c1.Do("PING") + c2 := p.Get(context.TODO()) + c2.Do("PING") + + d.check("1", p, 2, 2) + + c3 := p.Get(context.TODO()) + if _, err := c3.Do("PING"); err != pool.ErrPoolExhausted { + t.Errorf("expected pool exhausted") + } + + c3.Close() + d.check("2", p, 2, 2) + c2.Close() + d.check("3", p, 2, 2) + + c3 = p.Get(context.TODO()) + if _, err := c3.Do("PING"); err != nil { + t.Errorf("expected good channel, err=%v", err) + } + c3.Close() + + d.check("4", p, 2, 2) +} + +func TestPoolMonitorCleanup(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + c := p.Get(context.TODO()) + c.Send("MONITOR") + c.Close() + + d.check("", p, 1, 0) +} + +func TestPoolPubSubCleanup(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + c := p.Get(context.TODO()) + c.Send("SUBSCRIBE", "x") + c.Close() + + want := []string{"SUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get(context.TODO()) + c.Send("PSUBSCRIBE", "x*") + c.Close() + + want = []string{"PSUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil +} + +func TestPoolTransactionCleanup(t *testing.T) { + d := poolDialer{t: t} + p := NewPool(testConfig) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + c := p.Get(context.TODO()) + c.Do("WATCH", "key") + c.Do("PING") + c.Close() + + want := []string{"WATCH", "PING", "UNWATCH"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get(context.TODO()) + c.Do("WATCH", "key") + c.Do("UNWATCH") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "UNWATCH", "PING"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get(context.TODO()) + c.Do("WATCH", "key") + c.Do("MULTI") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "MULTI", "PING", "DISCARD"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get(context.TODO()) + c.Do("WATCH", "key") + c.Do("MULTI") + c.Do("DISCARD") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "MULTI", "DISCARD", "PING"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil + + c = p.Get(context.TODO()) + c.Do("WATCH", "key") + c.Do("MULTI") + c.Do("EXEC") + c.Do("PING") + c.Close() + + want = []string{"WATCH", "MULTI", "EXEC", "PING"} + if !reflect.DeepEqual(d.commands, want) { + t.Errorf("got commands %v, want %v", d.commands, want) + } + d.commands = nil +} + +func startGoroutines(p *Pool, cmd string, args ...interface{}) chan error { + errs := make(chan error, 10) + for i := 0; i < cap(errs); i++ { + go func() { + c := p.Get(context.TODO()) + _, err := c.Do(cmd, args...) + errs <- err + c.Close() + }() + } + + // Wait for goroutines to block. + time.Sleep(time.Second / 4) + + return errs +} + +func TestWaitPoolDialError(t *testing.T) { + testErr := errors.New("test") + d := poolDialer{t: t} + config1 := testConfig + config1.Config = &pool.Config{ + Active: 1, + Idle: 1, + Wait: true, + } + p := NewPool(config1) + p.Slice.New = func(ctx context.Context) (io.Closer, error) { + return d.dial() + } + defer p.Close() + + c := p.Get(context.TODO()) + errs := startGoroutines(p, "ERR", testErr) + d.check("before close", p, 1, 1) + + d.dialErr = errors.New("dial") + c.Close() + + nilCount := 0 + errCount := 0 + timeout := time.After(2 * time.Second) + for i := 0; i < cap(errs); i++ { + select { + case err := <-errs: + switch err { + case nil: + nilCount++ + case d.dialErr: + errCount++ + default: + t.Fatalf("expected dial error or nil, got %v", err) + } + case <-timeout: + t.Logf("Wait all the time and timeout %d", i) + return + } + } + if nilCount != 1 { + t.Errorf("expected one nil error, got %d", nilCount) + } + if errCount != cap(errs)-1 { + t.Errorf("expected %d dial erors, got %d", cap(errs)-1, errCount) + } + d.check("done", p, cap(errs), 0) +} + +func BenchmarkPoolGet(b *testing.B) { + b.StopTimer() + p := NewPool(testConfig) + c := p.Get(context.Background()) + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + defer p.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + c := p.Get(context.Background()) + c.Close() + } +} + +func BenchmarkPoolGetErr(b *testing.B) { + b.StopTimer() + p := NewPool(testConfig) + c := p.Get(context.Background()) + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + defer p.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + c = p.Get(context.Background()) + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + } +} + +func BenchmarkPoolGetPing(b *testing.B) { + b.StopTimer() + p := NewPool(testConfig) + c := p.Get(context.Background()) + if err := c.Err(); err != nil { + b.Fatal(err) + } + c.Close() + defer p.Close() + b.StartTimer() + for i := 0; i < b.N; i++ { + c := p.Get(context.Background()) + if _, err := c.Do("PING"); err != nil { + b.Fatal(err) + } + c.Close() + } +} + +func BenchmarkPooledConn(b *testing.B) { + p := NewPool(testConfig) + defer p.Close() + for i := 0; i < b.N; i++ { + ctx := context.TODO() + c := p.Get(ctx) + c2 := c.WithContext(context.TODO()) + if _, err := c2.Do("PING"); err != nil { + b.Fatal(err) + } + c2.Close() + } +} diff --git a/pkg/cache/redis/pubsub_test.go b/pkg/cache/redis/pubsub_test.go new file mode 100644 index 000000000..69c66ffd8 --- /dev/null +++ b/pkg/cache/redis/pubsub_test.go @@ -0,0 +1,146 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "fmt" + "reflect" + "sync" + "testing" +) + +func publish(channel, value interface{}) { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + c.Do("PUBLISH", channel, value) +} + +// Applications can receive pushed messages from one goroutine and manage subscriptions from another goroutine. +func ExamplePubSubConn() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + var wg sync.WaitGroup + wg.Add(2) + + psc := PubSubConn{Conn: c} + + // This goroutine receives and prints pushed notifications from the server. + // The goroutine exits when the connection is unsubscribed from all + // channels or there is an error. + go func() { + defer wg.Done() + for { + switch n := psc.Receive().(type) { + case Message: + fmt.Printf("Message: %s %s\n", n.Channel, n.Data) + case PMessage: + fmt.Printf("PMessage: %s %s %s\n", n.Pattern, n.Channel, n.Data) + case Subscription: + fmt.Printf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count) + if n.Count == 0 { + return + } + case error: + fmt.Printf("error: %v\n", n) + return + } + } + }() + + // This goroutine manages subscriptions for the connection. + go func() { + defer wg.Done() + + psc.Subscribe("example") + psc.PSubscribe("p*") + + // The following function calls publish a message using another + // connection to the Redis server. + publish("example", "hello") + publish("example", "world") + publish("pexample", "foo") + publish("pexample", "bar") + + // Unsubscribe from all connections. This will cause the receiving + // goroutine to exit. + psc.Unsubscribe() + psc.PUnsubscribe() + }() + + wg.Wait() + + // Output: + // Subscription: subscribe example 1 + // Subscription: psubscribe p* 2 + // Message: example hello + // Message: example world + // PMessage: p* pexample foo + // PMessage: p* pexample bar + // Subscription: unsubscribe example 1 + // Subscription: punsubscribe p* 0 +} + +func expectPushed(t *testing.T, c PubSubConn, message string, expected interface{}) { + actual := c.Receive() + if !reflect.DeepEqual(actual, expected) { + t.Errorf("%s = %v, want %v", message, actual, expected) + } +} + +func TestPushed(t *testing.T) { + pc, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer pc.Close() + + sc, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer sc.Close() + + c := PubSubConn{Conn: sc} + + c.Subscribe("c1") + expectPushed(t, c, "Subscribe(c1)", Subscription{Kind: "subscribe", Channel: "c1", Count: 1}) + c.Subscribe("c2") + expectPushed(t, c, "Subscribe(c2)", Subscription{Kind: "subscribe", Channel: "c2", Count: 2}) + c.PSubscribe("p1") + expectPushed(t, c, "PSubscribe(p1)", Subscription{Kind: "psubscribe", Channel: "p1", Count: 3}) + c.PSubscribe("p2") + expectPushed(t, c, "PSubscribe(p2)", Subscription{Kind: "psubscribe", Channel: "p2", Count: 4}) + c.PUnsubscribe() + expectPushed(t, c, "Punsubscribe(p1)", Subscription{Kind: "punsubscribe", Channel: "p1", Count: 3}) + expectPushed(t, c, "Punsubscribe()", Subscription{Kind: "punsubscribe", Channel: "p2", Count: 2}) + + pc.Do("PUBLISH", "c1", "hello") + expectPushed(t, c, "PUBLISH c1 hello", Message{Channel: "c1", Data: []byte("hello")}) + + c.Ping("hello") + expectPushed(t, c, `Ping("hello")`, Pong{"hello"}) + + c.Conn.Send("PING") + c.Conn.Flush() + expectPushed(t, c, `Send("PING")`, Pong{}) +} diff --git a/pkg/cache/redis/redis.go b/pkg/cache/redis/redis.go index 638cd9ab2..f912372b0 100644 --- a/pkg/cache/redis/redis.go +++ b/pkg/cache/redis/redis.go @@ -16,6 +16,9 @@ package redis import ( "context" + + "github.com/bilibili/kratos/pkg/container/pool" + xtime "github.com/bilibili/kratos/pkg/time" ) // Error represents an error returned in a command reply. @@ -23,29 +26,53 @@ type Error string func (err Error) Error() string { return string(err) } -// Conn represents a connection to a Redis server. -type Conn interface { - // Close closes the connection. - Close() error +// Config client settings. +type Config struct { + *pool.Config + + Name string // redis name, for trace + Proto string + Addr string + Auth string + DialTimeout xtime.Duration + ReadTimeout xtime.Duration + WriteTimeout xtime.Duration + SlowLog xtime.Duration +} - // Err returns a non-nil value if the connection is broken. The returned - // value is either the first non-nil value returned from the underlying - // network connection or a protocol parsing error. Applications should - // close broken connections. - Err() error +type Redis struct { + pool *Pool + conf *Config +} - // Do sends a command to the server and returns the received reply. - Do(commandName string, args ...interface{}) (reply interface{}, err error) +func NewRedis(c *Config, options ...DialOption) *Redis { + return &Redis{ + pool: NewPool(c, options...), + conf: c, + } +} - // Send writes the command to the client's output buffer. - Send(commandName string, args ...interface{}) error +// Do gets a new conn from pool, then execute Do with this conn, finally close this conn. +// ATTENTION: Don't use this method with transaction command like MULTI etc. Because every Do will close conn automatically, use r.Conn to get a raw conn for this situation. +func (r *Redis) Do(ctx context.Context, commandName string, args ...interface{}) (reply interface{}, err error) { + conn := r.pool.Get(ctx) + defer conn.Close() + reply, err = conn.Do(commandName, args...) + return +} - // Flush flushes the output buffer to the Redis server. - Flush() error +// Close closes connection pool +func (r *Redis) Close() error { + return r.pool.Close() +} - // Receive receives a single reply from the Redis server - Receive() (reply interface{}, err error) +// Conn direct gets a connection +func (r *Redis) Conn(ctx context.Context) Conn { + return r.pool.Get(ctx) +} - // WithContext - WithContext(ctx context.Context) Conn +func (r *Redis) Pipeline() (p Pipeliner) { + return &pipeliner{ + pool: r.pool, + } } diff --git a/pkg/cache/redis/redis_test.go b/pkg/cache/redis/redis_test.go new file mode 100644 index 000000000..464cbff8f --- /dev/null +++ b/pkg/cache/redis/redis_test.go @@ -0,0 +1,324 @@ +package redis + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/bilibili/kratos/pkg/container/pool" + xtime "github.com/bilibili/kratos/pkg/time" +) + +func TestRedis(t *testing.T) { + testSet(t, testPool) + testSend(t, testPool) + testGet(t, testPool) + testErr(t, testPool) + if err := testPool.Close(); err != nil { + t.Errorf("redis: close error(%v)", err) + } + conn, err := NewConn(testConfig) + if err != nil { + t.Errorf("redis: new conn error(%v)", err) + } + if err := conn.Close(); err != nil { + t.Errorf("redis: close error(%v)", err) + } +} + +func testSet(t *testing.T, p *Pool) { + var ( + key = "test" + value = "test" + conn = p.Get(context.TODO()) + ) + defer conn.Close() + if reply, err := conn.Do("set", key, value); err != nil { + t.Errorf("redis: conn.Do(SET, %s, %s) error(%v)", key, value, err) + } else { + t.Logf("redis: set status: %s", reply) + } +} + +func testSend(t *testing.T, p *Pool) { + var ( + key = "test" + value = "test" + expire = 1000 + conn = p.Get(context.TODO()) + ) + defer conn.Close() + if err := conn.Send("SET", key, value); err != nil { + t.Errorf("redis: conn.Send(SET, %s, %s) error(%v)", key, value, err) + } + if err := conn.Send("EXPIRE", key, expire); err != nil { + t.Errorf("redis: conn.Send(EXPIRE key(%s) expire(%d)) error(%v)", key, expire, err) + } + if err := conn.Flush(); err != nil { + t.Errorf("redis: conn.Flush error(%v)", err) + } + for i := 0; i < 2; i++ { + if _, err := conn.Receive(); err != nil { + t.Errorf("redis: conn.Receive error(%v)", err) + return + } + } + t.Logf("redis: set value: %s", value) +} + +func testGet(t *testing.T, p *Pool) { + var ( + key = "test" + conn = p.Get(context.TODO()) + ) + defer conn.Close() + if reply, err := conn.Do("GET", key); err != nil { + t.Errorf("redis: conn.Do(GET, %s) error(%v)", key, err) + } else { + t.Logf("redis: get value: %s", reply) + } +} + +func testErr(t *testing.T, p *Pool) { + conn := p.Get(context.TODO()) + if err := conn.Close(); err != nil { + t.Errorf("redis: close error(%v)", err) + } + if err := conn.Err(); err == nil { + t.Errorf("redis: err not nil") + } else { + t.Logf("redis: err: %v", err) + } +} + +func BenchmarkRedis(b *testing.B) { + conf := &Config{ + Name: "test", + Proto: "tcp", + Addr: testRedisAddr, + DialTimeout: xtime.Duration(time.Second), + ReadTimeout: xtime.Duration(time.Second), + WriteTimeout: xtime.Duration(time.Second), + } + conf.Config = &pool.Config{ + Active: 10, + Idle: 5, + IdleTimeout: xtime.Duration(90 * time.Second), + } + benchmarkPool := NewPool(conf) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + conn := benchmarkPool.Get(context.TODO()) + if err := conn.Close(); err != nil { + b.Errorf("redis: close error(%v)", err) + } + } + }) + if err := benchmarkPool.Close(); err != nil { + b.Errorf("redis: close error(%v)", err) + } +} + +var testRedisCommands = []struct { + args []interface{} + expected interface{} +}{ + { + []interface{}{"PING"}, + "PONG", + }, + { + []interface{}{"SET", "foo", "bar"}, + "OK", + }, + { + []interface{}{"GET", "foo"}, + []byte("bar"), + }, + { + []interface{}{"GET", "nokey"}, + nil, + }, + { + []interface{}{"MGET", "nokey", "foo"}, + []interface{}{nil, []byte("bar")}, + }, + { + []interface{}{"INCR", "mycounter"}, + int64(1), + }, + { + []interface{}{"LPUSH", "mylist", "foo"}, + int64(1), + }, + { + []interface{}{"LPUSH", "mylist", "bar"}, + int64(2), + }, + { + []interface{}{"LRANGE", "mylist", 0, -1}, + []interface{}{[]byte("bar"), []byte("foo")}, + }, +} + +func TestNewRedis(t *testing.T) { + type args struct { + c *Config + options []DialOption + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + "new_redis", + args{ + testConfig, + make([]DialOption, 0), + }, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := NewRedis(tt.args.c, tt.args.options...) + if r == nil { + t.Errorf("NewRedis() error, got nil") + return + } + err := r.Close() + if err != nil { + t.Errorf("Close() error %v", err) + } + }) + } +} + +func TestRedis_Do(t *testing.T) { + r := NewRedis(testConfig) + r.Do(context.TODO(), "FLUSHDB") + + for _, cmd := range testRedisCommands { + actual, err := r.Do(context.TODO(), cmd.args[0].(string), cmd.args[1:]...) + if err != nil { + t.Errorf("Do(%v) returned error %v", cmd.args, err) + continue + } + if !reflect.DeepEqual(actual, cmd.expected) { + t.Errorf("Do(%v) = %v, want %v", cmd.args, actual, cmd.expected) + } + } + err := r.Close() + if err != nil { + t.Errorf("Close() error %v", err) + } +} + +func TestRedis_Conn(t *testing.T) { + + type args struct { + ctx context.Context + } + tests := []struct { + name string + p *Redis + args args + wantErr bool + g int + c int + }{ + { + "Close", + NewRedis(&Config{ + Config: &pool.Config{ + Active: 1, + Idle: 1, + }, + Name: "test_get", + Proto: "tcp", + Addr: testRedisAddr, + DialTimeout: xtime.Duration(time.Second), + ReadTimeout: xtime.Duration(time.Second), + WriteTimeout: xtime.Duration(time.Second), + }), + args{context.TODO()}, + false, + 3, + 3, + }, + { + "CloseExceededPoolSize", + NewRedis(&Config{ + Config: &pool.Config{ + Active: 1, + Idle: 1, + }, + Name: "test_get_out", + Proto: "tcp", + Addr: testRedisAddr, + DialTimeout: xtime.Duration(time.Second), + ReadTimeout: xtime.Duration(time.Second), + WriteTimeout: xtime.Duration(time.Second), + }), + args{context.TODO()}, + true, + 5, + 3, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i := 1; i <= tt.g; i++ { + got := tt.p.Conn(tt.args.ctx) + if err := got.Close(); err != nil { + if !tt.wantErr { + t.Error(err) + } + } + if i <= tt.c { + if err := got.Close(); err != nil { + t.Error(err) + } + } + } + }) + } +} + +func BenchmarkRedisDoPing(b *testing.B) { + r := NewRedis(testConfig) + defer r.Close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := r.Do(context.Background(), "PING"); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkRedisDoSET(b *testing.B) { + r := NewRedis(testConfig) + defer r.Close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := r.Do(context.Background(), "SET", "a", "b"); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkRedisDoGET(b *testing.B) { + r := NewRedis(testConfig) + defer r.Close() + r.Do(context.Background(), "SET", "a", "b") + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := r.Do(context.Background(), "GET", "b"); err != nil { + b.Fatal(err) + } + } +} diff --git a/pkg/cache/redis/reply_test.go b/pkg/cache/redis/reply_test.go new file mode 100644 index 000000000..d3b1b9551 --- /dev/null +++ b/pkg/cache/redis/reply_test.go @@ -0,0 +1,179 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "fmt" + "reflect" + "testing" + + "github.com/pkg/errors" +) + +type valueError struct { + v interface{} + err error +} + +func ve(v interface{}, err error) valueError { + return valueError{v, err} +} + +var replyTests = []struct { + name interface{} + actual valueError + expected valueError +}{ + { + "ints([v1, v2])", + ve(Ints([]interface{}{[]byte("4"), []byte("5")}, nil)), + ve([]int{4, 5}, nil), + }, + { + "ints(nil)", + ve(Ints(nil, nil)), + ve([]int(nil), ErrNil), + }, + { + "strings([v1, v2])", + ve(Strings([]interface{}{[]byte("v1"), []byte("v2")}, nil)), + ve([]string{"v1", "v2"}, nil), + }, + { + "strings(nil)", + ve(Strings(nil, nil)), + ve([]string(nil), ErrNil), + }, + { + "byteslices([v1, v2])", + ve(ByteSlices([]interface{}{[]byte("v1"), []byte("v2")}, nil)), + ve([][]byte{[]byte("v1"), []byte("v2")}, nil), + }, + { + "byteslices(nil)", + ve(ByteSlices(nil, nil)), + ve([][]byte(nil), ErrNil), + }, + { + "values([v1, v2])", + ve(Values([]interface{}{[]byte("v1"), []byte("v2")}, nil)), + ve([]interface{}{[]byte("v1"), []byte("v2")}, nil), + }, + { + "values(nil)", + ve(Values(nil, nil)), + ve([]interface{}(nil), ErrNil), + }, + { + "float64(1.0)", + ve(Float64([]byte("1.0"), nil)), + ve(float64(1.0), nil), + }, + { + "float64(nil)", + ve(Float64(nil, nil)), + ve(float64(0.0), ErrNil), + }, + { + "uint64(1)", + ve(Uint64(int64(1), nil)), + ve(uint64(1), nil), + }, + { + "uint64(-1)", + ve(Uint64(int64(-1), nil)), + ve(uint64(0), errNegativeInt), + }, +} + +func TestReply(t *testing.T) { + for _, rt := range replyTests { + if errors.Cause(rt.actual.err) != rt.expected.err { + t.Errorf("%s returned err %v, want %v", rt.name, rt.actual.err, rt.expected.err) + continue + } + if !reflect.DeepEqual(rt.actual.v, rt.expected.v) { + t.Errorf("%s=%+v, want %+v", rt.name, rt.actual.v, rt.expected.v) + } + } +} + +// dial wraps DialDefaultServer() with a more suitable function name for examples. +func dial() (Conn, error) { + return DialDefaultServer() +} + +func ExampleBool() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SET", "foo", 1) + exists, _ := Bool(c.Do("EXISTS", "foo")) + fmt.Printf("%#v\n", exists) + // Output: + // true +} + +func ExampleInt() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SET", "k1", 1) + n, _ := Int(c.Do("GET", "k1")) + fmt.Printf("%#v\n", n) + n, _ = Int(c.Do("INCR", "k1")) + fmt.Printf("%#v\n", n) + // Output: + // 1 + // 2 +} + +func ExampleInts() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SADD", "set_with_integers", 4, 5, 6) + ints, _ := Ints(c.Do("SMEMBERS", "set_with_integers")) + fmt.Printf("%#v\n", ints) + // Output: + // []int{4, 5, 6} +} + +func ExampleString() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Do("SET", "hello", "world") + s, _ := String(c.Do("GET", "hello")) + fmt.Printf("%#v", s) + // Output: + // "world" +} diff --git a/pkg/cache/redis/scan_test.go b/pkg/cache/redis/scan_test.go new file mode 100644 index 000000000..fba605d77 --- /dev/null +++ b/pkg/cache/redis/scan_test.go @@ -0,0 +1,438 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "fmt" + "math" + "reflect" + "testing" +) + +var scanConversionTests = []struct { + src interface{} + dest interface{} +}{ + {[]byte("-inf"), math.Inf(-1)}, + {[]byte("+inf"), math.Inf(1)}, + {[]byte("0"), float64(0)}, + {[]byte("3.14159"), float64(3.14159)}, + {[]byte("3.14"), float32(3.14)}, + {[]byte("-100"), int(-100)}, + {[]byte("101"), int(101)}, + {int64(102), int(102)}, + {[]byte("103"), uint(103)}, + {int64(104), uint(104)}, + {[]byte("105"), int8(105)}, + {int64(106), int8(106)}, + {[]byte("107"), uint8(107)}, + {int64(108), uint8(108)}, + {[]byte("0"), false}, + {int64(0), false}, + {[]byte("f"), false}, + {[]byte("1"), true}, + {int64(1), true}, + {[]byte("t"), true}, + {"hello", "hello"}, + {[]byte("hello"), "hello"}, + {[]byte("world"), []byte("world")}, + {[]interface{}{[]byte("foo")}, []interface{}{[]byte("foo")}}, + {[]interface{}{[]byte("foo")}, []string{"foo"}}, + {[]interface{}{[]byte("hello"), []byte("world")}, []string{"hello", "world"}}, + {[]interface{}{[]byte("bar")}, [][]byte{[]byte("bar")}}, + {[]interface{}{[]byte("1")}, []int{1}}, + {[]interface{}{[]byte("1"), []byte("2")}, []int{1, 2}}, + {[]interface{}{[]byte("1"), []byte("2")}, []float64{1, 2}}, + {[]interface{}{[]byte("1")}, []byte{1}}, + {[]interface{}{[]byte("1")}, []bool{true}}, +} + +func TestScanConversion(t *testing.T) { + for _, tt := range scanConversionTests { + values := []interface{}{tt.src} + dest := reflect.New(reflect.TypeOf(tt.dest)) + values, err := Scan(values, dest.Interface()) + if err != nil { + t.Errorf("Scan(%v) returned error %v", tt, err) + continue + } + if !reflect.DeepEqual(tt.dest, dest.Elem().Interface()) { + t.Errorf("Scan(%v) returned %v values: %v, want %v", tt, dest.Elem().Interface(), values, tt.dest) + } + } +} + +var scanConversionErrorTests = []struct { + src interface{} + dest interface{} +}{ + {[]byte("1234"), byte(0)}, + {int64(1234), byte(0)}, + {[]byte("-1"), byte(0)}, + {int64(-1), byte(0)}, + {[]byte("junk"), false}, + {Error("blah"), false}, +} + +func TestScanConversionError(t *testing.T) { + for _, tt := range scanConversionErrorTests { + values := []interface{}{tt.src} + dest := reflect.New(reflect.TypeOf(tt.dest)) + values, err := Scan(values, dest.Interface()) + if err == nil { + t.Errorf("Scan(%v) did not return error values: %v", tt, values) + } + } +} + +func ExampleScan() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Send("HMSET", "album:1", "title", "Red", "rating", 5) + c.Send("HMSET", "album:2", "title", "Earthbound", "rating", 1) + c.Send("HMSET", "album:3", "title", "Beat") + c.Send("LPUSH", "albums", "1") + c.Send("LPUSH", "albums", "2") + c.Send("LPUSH", "albums", "3") + values, err := Values(c.Do("SORT", "albums", + "BY", "album:*->rating", + "GET", "album:*->title", + "GET", "album:*->rating")) + if err != nil { + fmt.Println(err) + return + } + + for len(values) > 0 { + var title string + rating := -1 // initialize to illegal value to detect nil. + values, err = Scan(values, &title, &rating) + if err != nil { + fmt.Println(err) + return + } + if rating == -1 { + fmt.Println(title, "not-rated") + } else { + fmt.Println(title, rating) + } + } + // Output: + // Beat not-rated + // Earthbound 1 + // Red 5 +} + +type s0 struct { + X int + Y int `redis:"y"` + Bt bool +} + +type s1 struct { + X int `redis:"-"` + I int `redis:"i"` + U uint `redis:"u"` + S string `redis:"s"` + P []byte `redis:"p"` + B bool `redis:"b"` + Bt bool + Bf bool + s0 +} + +var scanStructTests = []struct { + title string + reply []string + value interface{} +}{ + {"basic", + []string{"i", "-1234", "u", "5678", "s", "hello", "p", "world", "b", "t", "Bt", "1", "Bf", "0", "X", "123", "y", "456"}, + &s1{I: -1234, U: 5678, S: "hello", P: []byte("world"), B: true, Bt: true, Bf: false, s0: s0{X: 123, Y: 456}}, + }, +} + +func TestScanStruct(t *testing.T) { + for _, tt := range scanStructTests { + + var reply []interface{} + for _, v := range tt.reply { + reply = append(reply, []byte(v)) + } + + value := reflect.New(reflect.ValueOf(tt.value).Type().Elem()) + + if err := ScanStruct(reply, value.Interface()); err != nil { + t.Fatalf("ScanStruct(%s) returned error %v", tt.title, err) + } + + if !reflect.DeepEqual(value.Interface(), tt.value) { + t.Fatalf("ScanStruct(%s) returned %v, want %v", tt.title, value.Interface(), tt.value) + } + } +} + +func TestBadScanStructArgs(t *testing.T) { + x := []interface{}{"A", "b"} + test := func(v interface{}) { + if err := ScanStruct(x, v); err == nil { + t.Errorf("Expect error for ScanStruct(%T, %T)", x, v) + } + } + + test(nil) + + var v0 *struct{} + test(v0) + + var v1 int + test(&v1) + + x = x[:1] + v2 := struct{ A string }{} + test(&v2) +} + +var scanSliceTests = []struct { + src []interface{} + fieldNames []string + ok bool + dest interface{} +}{ + { + []interface{}{[]byte("1"), nil, []byte("-1")}, + nil, + true, + []int{1, 0, -1}, + }, + { + []interface{}{[]byte("1"), nil, []byte("2")}, + nil, + true, + []uint{1, 0, 2}, + }, + { + []interface{}{[]byte("-1")}, + nil, + false, + []uint{1}, + }, + { + []interface{}{[]byte("hello"), nil, []byte("world")}, + nil, + true, + [][]byte{[]byte("hello"), nil, []byte("world")}, + }, + { + []interface{}{[]byte("hello"), nil, []byte("world")}, + nil, + true, + []string{"hello", "", "world"}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + nil, + true, + []struct{ A, B string }{{"a1", "b1"}, {"a2", "b2"}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1")}, + nil, + false, + []struct{ A, B, C string }{{"a1", "b1", ""}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + nil, + true, + []*struct{ A, B string }{{"a1", "b1"}, {"a2", "b2"}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + []string{"A", "B"}, + true, + []struct{ A, C, B string }{{"a1", "", "b1"}, {"a2", "", "b2"}}, + }, + { + []interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")}, + nil, + false, + []struct{}{}, + }, +} + +func TestScanSlice(t *testing.T) { + for _, tt := range scanSliceTests { + + typ := reflect.ValueOf(tt.dest).Type() + dest := reflect.New(typ) + + err := ScanSlice(tt.src, dest.Interface(), tt.fieldNames...) + if tt.ok != (err == nil) { + t.Errorf("ScanSlice(%v, []%s, %v) returned error %v", tt.src, typ, tt.fieldNames, err) + continue + } + if tt.ok && !reflect.DeepEqual(dest.Elem().Interface(), tt.dest) { + t.Errorf("ScanSlice(src, []%s) returned %#v, want %#v", typ, dest.Elem().Interface(), tt.dest) + } + } +} + +func ExampleScanSlice() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + c.Send("HMSET", "album:1", "title", "Red", "rating", 5) + c.Send("HMSET", "album:2", "title", "Earthbound", "rating", 1) + c.Send("HMSET", "album:3", "title", "Beat", "rating", 4) + c.Send("LPUSH", "albums", "1") + c.Send("LPUSH", "albums", "2") + c.Send("LPUSH", "albums", "3") + values, err := Values(c.Do("SORT", "albums", + "BY", "album:*->rating", + "GET", "album:*->title", + "GET", "album:*->rating")) + if err != nil { + fmt.Println(err) + return + } + + var albums []struct { + Title string + Rating int + } + if err := ScanSlice(values, &albums); err != nil { + fmt.Println(err) + return + } + fmt.Printf("%v\n", albums) + // Output: + // [{Earthbound 1} {Beat 4} {Red 5}] +} + +var argsTests = []struct { + title string + actual Args + expected Args +}{ + {"struct ptr", + Args{}.AddFlat(&struct { + I int `redis:"i"` + U uint `redis:"u"` + S string `redis:"s"` + P []byte `redis:"p"` + M map[string]string `redis:"m"` + Bt bool + Bf bool + }{ + -1234, 5678, "hello", []byte("world"), map[string]string{"hello": "world"}, true, false, + }), + Args{"i", int(-1234), "u", uint(5678), "s", "hello", "p", []byte("world"), "m", map[string]string{"hello": "world"}, "Bt", true, "Bf", false}, + }, + {"struct", + Args{}.AddFlat(struct{ I int }{123}), + Args{"I", 123}, + }, + {"slice", + Args{}.Add(1).AddFlat([]string{"a", "b", "c"}).Add(2), + Args{1, "a", "b", "c", 2}, + }, + {"struct omitempty", + Args{}.AddFlat(&struct { + I int `redis:"i,omitempty"` + U uint `redis:"u,omitempty"` + S string `redis:"s,omitempty"` + P []byte `redis:"p,omitempty"` + M map[string]string `redis:"m,omitempty"` + Bt bool `redis:"Bt,omitempty"` + Bf bool `redis:"Bf,omitempty"` + }{ + 0, 0, "", []byte{}, map[string]string{}, true, false, + }), + Args{"Bt", true}, + }, +} + +func TestArgs(t *testing.T) { + for _, tt := range argsTests { + if !reflect.DeepEqual(tt.actual, tt.expected) { + t.Fatalf("%s is %v, want %v", tt.title, tt.actual, tt.expected) + } + } +} + +func ExampleArgs() { + c, err := dial() + if err != nil { + fmt.Println(err) + return + } + defer c.Close() + + var p1, p2 struct { + Title string `redis:"title"` + Author string `redis:"author"` + Body string `redis:"body"` + } + + p1.Title = "Example" + p1.Author = "Gary" + p1.Body = "Hello" + + if _, err := c.Do("HMSET", Args{}.Add("id1").AddFlat(&p1)...); err != nil { + fmt.Println(err) + return + } + + m := map[string]string{ + "title": "Example2", + "author": "Steve", + "body": "Map", + } + + if _, err := c.Do("HMSET", Args{}.Add("id2").AddFlat(m)...); err != nil { + fmt.Println(err) + return + } + + for _, id := range []string{"id1", "id2"} { + + v, err := Values(c.Do("HGETALL", id)) + if err != nil { + fmt.Println(err) + return + } + + if err := ScanStruct(v, &p2); err != nil { + fmt.Println(err) + return + } + + fmt.Printf("%+v\n", p2) + } + + // Output: + // {Title:Example Author:Gary Body:Hello} + // {Title:Example2 Author:Steve Body:Map} +} diff --git a/pkg/cache/redis/script_test.go b/pkg/cache/redis/script_test.go new file mode 100644 index 000000000..405a33128 --- /dev/null +++ b/pkg/cache/redis/script_test.go @@ -0,0 +1,103 @@ +// Copyright 2012 Gary Burd +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package redis + +import ( + "fmt" + "reflect" + "testing" + "time" +) + +func ExampleScript() { + c, err := Dial("tcp", ":6379") + if err != nil { + // handle error + } + defer c.Close() + // Initialize a package-level variable with a script. + var getScript = NewScript(1, `return call('get', KEYS[1])`) + + // In a function, use the script Do method to evaluate the script. The Do + // method optimistically uses the EVALSHA command. If the script is not + // loaded, then the Do method falls back to the EVAL command. + if _, err = getScript.Do(c, "foo"); err != nil { + // handle error + } +} + +func TestScript(t *testing.T) { + c, err := DialDefaultServer() + if err != nil { + t.Fatalf("error connection to database, %v", err) + } + defer c.Close() + + // To test fall back in Do, we make script unique by adding comment with current time. + script := fmt.Sprintf("--%d\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", time.Now().UnixNano()) + s := NewScript(2, script) + reply := []interface{}{[]byte("key1"), []byte("key2"), []byte("arg1"), []byte("arg2")} + + v, err := s.Do(c, "key1", "key2", "arg1", "arg2") + if err != nil { + t.Errorf("s.Do(c, ...) returned %v", err) + } + + if !reflect.DeepEqual(v, reply) { + t.Errorf("s.Do(c, ..); = %v, want %v", v, reply) + } + + err = s.Load(c) + if err != nil { + t.Errorf("s.Load(c) returned %v", err) + } + + err = s.SendHash(c, "key1", "key2", "arg1", "arg2") + if err != nil { + t.Errorf("s.SendHash(c, ...) returned %v", err) + } + + err = c.Flush() + if err != nil { + t.Errorf("c.Flush() returned %v", err) + } + + v, err = c.Receive() + if err != nil { + t.Errorf("c.Receive() returned %v", err) + } + if !reflect.DeepEqual(v, reply) { + t.Errorf("s.SendHash(c, ..); c.Receive() = %v, want %v", v, reply) + } + + err = s.Send(c, "key1", "key2", "arg1", "arg2") + if err != nil { + t.Errorf("s.Send(c, ...) returned %v", err) + } + + err = c.Flush() + if err != nil { + t.Errorf("c.Flush() returned %v", err) + } + + v, err = c.Receive() + if err != nil { + t.Errorf("c.Receive() returned %v", err) + } + if !reflect.DeepEqual(v, reply) { + t.Errorf("s.Send(c, ..); c.Receive() = %v, want %v", v, reply) + } + +} diff --git a/pkg/cache/redis/test/docker-compose.yaml b/pkg/cache/redis/test/docker-compose.yaml new file mode 100644 index 000000000..4bb1f4552 --- /dev/null +++ b/pkg/cache/redis/test/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3.7" + +services: + redis: + image: redis + ports: + - 6379:6379 + healthcheck: + test: ["CMD", "redis-cli","ping"] + interval: 20s + timeout: 1s + retries: 20 \ No newline at end of file diff --git a/pkg/cache/redis/trace.go b/pkg/cache/redis/trace.go index 3804f937d..ed0b2a62d 100644 --- a/pkg/cache/redis/trace.go +++ b/pkg/cache/redis/trace.go @@ -10,10 +10,9 @@ import ( ) const ( - _traceComponentName = "pkg/cache/redis" + _traceComponentName = "library/cache/redis" _tracePeerService = "redis" _traceSpanKind = "client" - _slowLogDuration = time.Millisecond * 250 ) var _internalTags = []trace.Tag{ @@ -24,26 +23,28 @@ var _internalTags = []trace.Tag{ type traceConn struct { // tr for pipeline, if tr != nil meaning on pipeline - tr trace.Trace - ctx context.Context + tr trace.Trace // connTag include e.g. ip,port connTags []trace.Tag + ctx context.Context + // origin redis conn Conn pending int + // TODO: split slow log from trace. + slowLogThreshold time.Duration } func (t *traceConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { statement := getStatement(commandName, args...) - defer slowLog(statement, time.Now()) - root, ok := trace.FromContext(t.ctx) + defer t.slowLog(statement, time.Now()) // NOTE: ignored empty commandName // current sdk will Do empty command after pipeline finished - if !ok || commandName == "" { + if t.tr == nil || commandName == "" { return t.Conn.Do(commandName, args...) } - tr := root.Fork("", "Redis:"+commandName) + tr := t.tr.Fork("", "Redis:"+commandName) tr.SetTag(_internalTags...) tr.SetTag(t.connTags...) tr.SetTag(trace.TagString(trace.TagDBStatement, statement)) @@ -52,16 +53,15 @@ func (t *traceConn) Do(commandName string, args ...interface{}) (reply interface return } -func (t *traceConn) Send(commandName string, args ...interface{}) error { +func (t *traceConn) Send(commandName string, args ...interface{}) (err error) { statement := getStatement(commandName, args...) - defer slowLog(statement, time.Now()) + defer t.slowLog(statement, time.Now()) t.pending++ - root, ok := trace.FromContext(t.ctx) - if !ok { + if t.tr == nil { return t.Conn.Send(commandName, args...) } - if t.tr == nil { - t.tr = root.Fork("", "Redis:Pipeline") + if t.pending == 1 { + t.tr = t.tr.Fork("", "Redis:Pipeline") t.tr.SetTag(_internalTags...) t.tr.SetTag(t.connTags...) } @@ -69,8 +69,7 @@ func (t *traceConn) Send(commandName string, args ...interface{}) error { trace.Log(trace.LogEvent, "Send"), trace.Log("db.statement", statement), ) - err := t.Conn.Send(commandName, args...) - if err != nil { + if err = t.Conn.Send(commandName, args...); err != nil { t.tr.SetTag(trace.TagBool(trace.TagError, true)) t.tr.SetLog( trace.Log(trace.LogEvent, "Send Fail"), @@ -81,7 +80,7 @@ func (t *traceConn) Send(commandName string, args ...interface{}) error { } func (t *traceConn) Flush() error { - defer slowLog("Flush", time.Now()) + defer t.slowLog("Flush", time.Now()) if t.tr == nil { return t.Conn.Flush() } @@ -98,7 +97,7 @@ func (t *traceConn) Flush() error { } func (t *traceConn) Receive() (reply interface{}, err error) { - defer slowLog("Receive", time.Now()) + defer t.slowLog("Receive", time.Now()) if t.tr == nil { return t.Conn.Receive() } @@ -122,13 +121,14 @@ func (t *traceConn) Receive() (reply interface{}, err error) { } func (t *traceConn) WithContext(ctx context.Context) Conn { - t.ctx = ctx + t.Conn = t.Conn.WithContext(ctx) + t.tr, _ = trace.FromContext(ctx) return t } -func slowLog(statement string, now time.Time) { +func (t *traceConn) slowLog(statement string, now time.Time) { du := time.Since(now) - if du > _slowLogDuration { + if du > t.slowLogThreshold { log.Warn("%s slow log statement: %s time: %v", _tracePeerService, statement, du) } } diff --git a/pkg/cache/redis/trace_test.go b/pkg/cache/redis/trace_test.go new file mode 100644 index 000000000..181910342 --- /dev/null +++ b/pkg/cache/redis/trace_test.go @@ -0,0 +1,192 @@ +package redis + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/bilibili/kratos/pkg/net/trace" + "github.com/stretchr/testify/assert" +) + +const testTraceSlowLogThreshold = time.Duration(250 * time.Millisecond) + +type mockTrace struct { + tags []trace.Tag + logs []trace.LogField + perr *error + operationName string + finished bool +} + +func (m *mockTrace) Fork(serviceName string, operationName string) trace.Trace { + m.operationName = operationName + return m +} +func (m *mockTrace) Follow(serviceName string, operationName string) trace.Trace { + panic("not implemented") +} +func (m *mockTrace) Finish(err *error) { + m.perr = err + m.finished = true +} +func (m *mockTrace) SetTag(tags ...trace.Tag) trace.Trace { + m.tags = append(m.tags, tags...) + return m +} +func (m *mockTrace) SetLog(logs ...trace.LogField) trace.Trace { + m.logs = append(m.logs, logs...) + return m +} +func (m *mockTrace) Visit(fn func(k, v string)) {} +func (m *mockTrace) SetTitle(title string) {} +func (m *mockTrace) TraceID() string { return "" } + +type mockConn struct{} + +func (c *mockConn) Close() error { return nil } +func (c *mockConn) Err() error { return nil } +func (c *mockConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) { + return nil, nil +} +func (c *mockConn) Send(commandName string, args ...interface{}) error { return nil } +func (c *mockConn) Flush() error { return nil } +func (c *mockConn) Receive() (reply interface{}, err error) { return nil, nil } +func (c *mockConn) WithContext(context.Context) Conn { return c } + +func TestTraceDo(t *testing.T) { + tr := &mockTrace{} + ctx := trace.NewContext(context.Background(), tr) + tc := &traceConn{Conn: &mockConn{}, slowLogThreshold: testTraceSlowLogThreshold} + conn := tc.WithContext(ctx) + + conn.Do("GET", "test") + + assert.Equal(t, "Redis:GET", tr.operationName) + assert.NotEmpty(t, tr.tags) + assert.True(t, tr.finished) +} + +func TestTraceDoErr(t *testing.T) { + tr := &mockTrace{} + ctx := trace.NewContext(context.Background(), tr) + tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hhhhhhh")}, + slowLogThreshold: testTraceSlowLogThreshold} + conn := tc.WithContext(ctx) + + conn.Do("GET", "test") + + assert.Equal(t, "Redis:GET", tr.operationName) + assert.True(t, tr.finished) + assert.NotNil(t, *tr.perr) +} + +func TestTracePipeline(t *testing.T) { + tr := &mockTrace{} + ctx := trace.NewContext(context.Background(), tr) + tc := &traceConn{Conn: &mockConn{}, slowLogThreshold: testTraceSlowLogThreshold} + conn := tc.WithContext(ctx) + + N := 2 + for i := 0; i < N; i++ { + conn.Send("GET", "hello, world") + } + conn.Flush() + for i := 0; i < N; i++ { + conn.Receive() + } + + assert.Equal(t, "Redis:Pipeline", tr.operationName) + assert.NotEmpty(t, tr.tags) + assert.NotEmpty(t, tr.logs) + assert.True(t, tr.finished) +} + +func TestTracePipelineErr(t *testing.T) { + tr := &mockTrace{} + ctx := trace.NewContext(context.Background(), tr) + tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hahah")}, + slowLogThreshold: testTraceSlowLogThreshold} + conn := tc.WithContext(ctx) + + N := 2 + for i := 0; i < N; i++ { + conn.Send("GET", "hello, world") + } + conn.Flush() + for i := 0; i < N; i++ { + conn.Receive() + } + + assert.Equal(t, "Redis:Pipeline", tr.operationName) + assert.NotEmpty(t, tr.tags) + assert.NotEmpty(t, tr.logs) + assert.True(t, tr.finished) + var isError bool + for _, tag := range tr.tags { + if tag.Key == "error" { + isError = true + } + } + assert.True(t, isError) +} + +func TestSendStatement(t *testing.T) { + tr := &mockTrace{} + ctx := trace.NewContext(context.Background(), tr) + tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hahah")}, + slowLogThreshold: testTraceSlowLogThreshold} + conn := tc.WithContext(ctx) + conn.Send("SET", "hello", "test") + conn.Flush() + conn.Receive() + + assert.Equal(t, "Redis:Pipeline", tr.operationName) + assert.NotEmpty(t, tr.tags) + assert.NotEmpty(t, tr.logs) + assert.Equal(t, "event", tr.logs[0].Key) + assert.Equal(t, "Send", tr.logs[0].Value) + assert.Equal(t, "db.statement", tr.logs[1].Key) + assert.Equal(t, "SET hello", tr.logs[1].Value) + assert.True(t, tr.finished) + var isError bool + for _, tag := range tr.tags { + if tag.Key == "error" { + isError = true + } + } + assert.True(t, isError) +} + +func TestDoStatement(t *testing.T) { + tr := &mockTrace{} + ctx := trace.NewContext(context.Background(), tr) + tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hahah")}, + slowLogThreshold: testTraceSlowLogThreshold} + conn := tc.WithContext(ctx) + conn.Do("SET", "hello", "test") + + assert.Equal(t, "Redis:SET", tr.operationName) + assert.Equal(t, "SET hello", tr.tags[len(tr.tags)-1].Value) + assert.True(t, tr.finished) +} + +func BenchmarkTraceConn(b *testing.B) { + for i := 0; i < b.N; i++ { + c, err := DialDefaultServer() + if err != nil { + b.Fatal(err) + } + t := &traceConn{ + Conn: c, + connTags: []trace.Tag{trace.TagString(trace.TagPeerAddress, "abc")}, + slowLogThreshold: time.Duration(1 * time.Second), + } + c2 := t.WithContext(context.TODO()) + if _, err := c2.Do("PING"); err != nil { + b.Fatal(err) + } + c2.Close() + } +} diff --git a/pkg/cache/redis/util.go b/pkg/cache/redis/util.go new file mode 100644 index 000000000..aa52597bb --- /dev/null +++ b/pkg/cache/redis/util.go @@ -0,0 +1,17 @@ +package redis + +import ( + "context" + "time" +) + +func shrinkDeadline(ctx context.Context, timeout time.Duration) time.Time { + var timeoutTime = time.Now().Add(timeout) + if ctx == nil { + return timeoutTime + } + if deadline, ok := ctx.Deadline(); ok && timeoutTime.After(deadline) { + return deadline + } + return timeoutTime +} diff --git a/pkg/cache/redis/util_test.go b/pkg/cache/redis/util_test.go new file mode 100644 index 000000000..748b8423e --- /dev/null +++ b/pkg/cache/redis/util_test.go @@ -0,0 +1,37 @@ +package redis + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestShrinkDeadline(t *testing.T) { + t.Run("test not deadline", func(t *testing.T) { + timeout := time.Second + timeoutTime := time.Now().Add(timeout) + tm := shrinkDeadline(context.Background(), timeout) + assert.True(t, tm.After(timeoutTime)) + }) + t.Run("test big deadline", func(t *testing.T) { + timeout := time.Second + timeoutTime := time.Now().Add(timeout) + deadlineTime := time.Now().Add(2 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + tm := shrinkDeadline(ctx, timeout) + assert.True(t, tm.After(timeoutTime) && tm.Before(deadlineTime)) + }) + t.Run("test small deadline", func(t *testing.T) { + timeout := time.Second + deadlineTime := time.Now().Add(500 * time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + + tm := shrinkDeadline(ctx, timeout) + assert.True(t, tm.After(deadlineTime) && tm.Before(time.Now().Add(timeout))) + }) +} diff --git a/pkg/conf/paladin/README.md b/pkg/conf/paladin/README.md index 6db087389..a49a330ba 100644 --- a/pkg/conf/paladin/README.md +++ b/pkg/conf/paladin/README.md @@ -2,17 +2,18 @@ ##### 项目简介 -paladin 是一个config SDK客户端，包括了file、mock几个抽象功能，方便使用本地文件或者sven配置中心，并且集成了对象自动reload功能。 - +paladin 是一个config SDK客户端，包括了file、mock几个抽象功能，方便使用本地文件或者sven\apollo配置中心，并且集成了对象自动reload功能。 local files: ``` demo -conf=/data/conf/app/msm-servie.toml // or dir demo -conf=/data/conf/app/ - ``` -example: + +*注：使用远程配置中心的用户在执行应用，如这里的`demo`时务必**不要**带上`-conf`参数，具体见下文远程配置中心的例子* + +local file example: ``` type exampleConf struct { Bool bool @@ -65,6 +66,71 @@ func ExampleClient() { } ``` +remote config center example: +``` +type exampleConf struct { + Bool bool + Int int64 + Float float64 + String string +} + +func (e *exampleConf) Set(text string) error { + var ec exampleConf + if err := yaml.Unmarshal([]byte(text), &ec); err != nil { + return err + } + *e = ec + return nil +} + +func ExampleApolloClient() { + /* + pass flags or set envs that apollo needs, for example: + + ``` + export APOLLO_APP_ID=SampleApp + export APOLLO_CLUSTER=default + export APOLLO_CACHE_DIR=/tmp + export APOLLO_META_ADDR=localhost:8080 + export APOLLO_NAMESPACES=example.yml + ``` + */ + + if err := paladin.Init(apollo.PaladinDriverApollo); err != nil { + panic(err) + } + var ( + ec exampleConf + eo exampleConf + m paladin.Map + strs []string + ) + // config unmarshal + if err := paladin.Get("example.yml").UnmarshalYAML(&ec); err != nil { + panic(err) + } + // config setter + if err := paladin.Watch("example.yml", &ec); err != nil { + panic(err) + } + // paladin map + if err := paladin.Watch("example.yml", &m); err != nil { + panic(err) + } + s, err := m.Value("key").String() + b, err := m.Value("key").Bool() + i, err := m.Value("key").Int64() + f, err := m.Value("key").Float64() + // value slice + err = m.Value("strings").Slice(&strs) + // watch key + for event := range paladin.WatchEvent(context.TODO(), "key") { + fmt.Println(event) + } +} +``` + ##### 编译环境 - **请只用 Golang v1.12.x 以上版本编译执行** diff --git a/pkg/conf/paladin/apollo/apollo.go b/pkg/conf/paladin/apollo/apollo.go new file mode 100644 index 000000000..0ce83f9de --- /dev/null +++ b/pkg/conf/paladin/apollo/apollo.go @@ -0,0 +1,273 @@ +package apollo + +import ( + "context" + "errors" + "flag" + "log" + "os" + "strings" + "sync" + "time" + + "github.com/philchia/agollo" + + "github.com/bilibili/kratos/pkg/conf/paladin" +) + +var ( + _ paladin.Client = &apollo{} + defaultValue = "" +) + +type apolloWatcher struct { + keys []string // in apollo, they're called namespaces + C chan paladin.Event +} + +func newApolloWatcher(keys []string) *apolloWatcher { + return &apolloWatcher{keys: keys, C: make(chan paladin.Event, 5)} +} + +func (aw *apolloWatcher) HasKey(key string) bool { + if len(aw.keys) == 0 { + return true + } + for _, k := range aw.keys { + if k == key { + return true + } + } + return false +} + +func (aw *apolloWatcher) Handle(event paladin.Event) { + select { + case aw.C <- event: + default: + log.Printf("paladin: event channel full discard ns %s update event", event.Key) + } +} + +// apollo is apollo config client. +type apollo struct { + client *agollo.Client + values *paladin.Map + wmu sync.RWMutex + watchers map[*apolloWatcher]struct{} +} + +// Config is apollo config client config. +type Config struct { + AppID string `json:"app_id"` + Cluster string `json:"cluster"` + CacheDir string `json:"cache_dir"` + MetaAddr string `json:"meta_addr"` + Namespaces []string `json:"namespaces"` +} + +type apolloDriver struct{} + +var ( + confAppID, confCluster, confCacheDir, confMetaAddr, confNamespaces string +) + +func init() { + addApolloFlags() + paladin.Register(PaladinDriverApollo, &apolloDriver{}) +} + +func addApolloFlags() { + flag.StringVar(&confAppID, "apollo.appid", "", "apollo app id") + flag.StringVar(&confCluster, "apollo.cluster", "", "apollo cluster") + flag.StringVar(&confCacheDir, "apollo.cachedir", "/tmp", "apollo cache dir") + flag.StringVar(&confMetaAddr, "apollo.metaaddr", "", "apollo meta server addr, e.g. localhost:8080") + flag.StringVar(&confNamespaces, "apollo.namespaces", "", "subscribed apollo namespaces, comma separated, e.g. app.yml,mysql.yml") +} + +func buildConfigForApollo() (c *Config, err error) { + if appidFromEnv := os.Getenv("APOLLO_APP_ID"); appidFromEnv != "" { + confAppID = appidFromEnv + } + if confAppID == "" { + err = errors.New("invalid apollo appid, pass it via APOLLO_APP_ID=xxx with env or --apollo.appid=xxx with flag") + return + } + if clusterFromEnv := os.Getenv("APOLLO_CLUSTER"); clusterFromEnv != "" { + confCluster = clusterFromEnv + } + if confAppID == "" { + err = errors.New("invalid apollo cluster, pass it via APOLLO_CLUSTER=xxx with env or --apollo.cluster=xxx with flag") + return + } + if cacheDirFromEnv := os.Getenv("APOLLO_CACHE_DIR"); cacheDirFromEnv != "" { + confCacheDir = cacheDirFromEnv + } + if metaAddrFromEnv := os.Getenv("APOLLO_META_ADDR"); metaAddrFromEnv != "" { + confMetaAddr = metaAddrFromEnv + } + if confMetaAddr == "" { + err = errors.New("invalid apollo meta addr, pass it via APOLLO_META_ADDR=xxx with env or --apollo.metaaddr=xxx with flag") + return + } + if namespacesFromEnv := os.Getenv("APOLLO_NAMESPACES"); namespacesFromEnv != "" { + confNamespaces = namespacesFromEnv + } + namespaceNames := strings.Split(confNamespaces, ",") + if len(namespaceNames) == 0 { + err = errors.New("invalid apollo namespaces, pass it via APOLLO_NAMESPACES=xxx with env or --apollo.namespaces=xxx with flag") + return + } + c = &Config{ + AppID: confAppID, + Cluster: confCluster, + CacheDir: confCacheDir, + MetaAddr: confMetaAddr, + Namespaces: namespaceNames, + } + return +} + +// New new an apollo config client. +// it watches apollo namespaces changes and updates local cache. +// BTW, in our context, namespaces in apollo means keys in paladin. +func (ad *apolloDriver) New() (paladin.Client, error) { + c, err := buildConfigForApollo() + if err != nil { + return nil, err + } + return ad.new(c) +} + +func (ad *apolloDriver) new(conf *Config) (paladin.Client, error) { + if conf == nil { + err := errors.New("invalid apollo conf") + return nil, err + } + client := agollo.NewClient(&agollo.Conf{ + AppID: conf.AppID, + Cluster: conf.Cluster, + NameSpaceNames: conf.Namespaces, // these namespaces will be subscribed at init + CacheDir: conf.CacheDir, + IP: conf.MetaAddr, + }) + err := client.Start() + if err != nil { + return nil, err + } + a := &apollo{ + client: client, + values: new(paladin.Map), + watchers: make(map[*apolloWatcher]struct{}), + } + raws, err := a.loadValues(conf.Namespaces) + if err != nil { + return nil, err + } + a.values.Store(raws) + // watch namespaces by default. + a.WatchEvent(context.TODO(), conf.Namespaces...) + go a.watchproc(conf.Namespaces) + return a, nil +} + +// loadValues load values from apollo namespaces to values +func (a *apollo) loadValues(keys []string) (values map[string]*paladin.Value, err error) { + values = make(map[string]*paladin.Value, len(keys)) + for _, k := range keys { + if values[k], err = a.loadValue(k); err != nil { + return + } + } + return +} + +// loadValue load value from apollo namespace content to value +func (a *apollo) loadValue(key string) (*paladin.Value, error) { + content := a.client.GetNameSpaceContent(key, defaultValue) + return paladin.NewValue(content, content), nil +} + +// reloadValue reload value by key and send event +func (a *apollo) reloadValue(key string) (err error) { + // NOTE: in some case immediately read content from client after receive event + // will get old content due to cache, sleep 100ms make sure get correct content. + time.Sleep(100 * time.Millisecond) + var ( + value *paladin.Value + rawValue string + ) + value, err = a.loadValue(key) + if err != nil { + return + } + rawValue, err = value.Raw() + if err != nil { + return + } + raws := a.values.Load() + raws[key] = value + a.values.Store(raws) + a.wmu.RLock() + n := 0 + for w := range a.watchers { + if w.HasKey(key) { + n++ + // FIXME(Colstuwjx): check change event and send detail type like EventAdd\Update\Delete. + w.Handle(paladin.Event{Event: paladin.EventUpdate, Key: key, Value: rawValue}) + } + } + a.wmu.RUnlock() + log.Printf("paladin: reload config: %s events: %d\n", key, n) + return +} + +// apollo config daemon to watch remote apollo notifications +func (a *apollo) watchproc(keys []string) { + events := a.client.WatchUpdate() + for { + select { + case event := <-events: + if err := a.reloadValue(event.Namespace); err != nil { + log.Printf("paladin: load key: %s error: %s, skipped", event.Namespace, err) + } + } + } +} + +// Get return value by key. +func (a *apollo) Get(key string) *paladin.Value { + return a.values.Get(key) +} + +// GetAll return value map. +func (a *apollo) GetAll() *paladin.Map { + return a.values +} + +// WatchEvent watch with the specified keys. +func (a *apollo) WatchEvent(ctx context.Context, keys ...string) <-chan paladin.Event { + aw := newApolloWatcher(keys) + err := a.client.SubscribeToNamespaces(keys...) + if err != nil { + log.Printf("subscribe namespaces %v failed, %v", keys, err) + return aw.C + } + a.wmu.Lock() + a.watchers[aw] = struct{}{} + a.wmu.Unlock() + return aw.C +} + +// Close close watcher. +func (a *apollo) Close() (err error) { + if err = a.client.Stop(); err != nil { + return + } + a.wmu.RLock() + for w := range a.watchers { + close(w.C) + } + a.wmu.RUnlock() + return +} diff --git a/pkg/conf/paladin/apollo/apollo_test.go b/pkg/conf/paladin/apollo/apollo_test.go new file mode 100644 index 000000000..cef79cb10 --- /dev/null +++ b/pkg/conf/paladin/apollo/apollo_test.go @@ -0,0 +1,73 @@ +package apollo + +import ( + "context" + "fmt" + "log" + "os" + "testing" + "time" + + "github.com/bilibili/kratos/pkg/conf/paladin/apollo/internal/mockserver" +) + +func TestMain(m *testing.M) { + setup() + code := m.Run() + teardown() + os.Exit(code) +} + +func setup() { + go func() { + if err := mockserver.Run(); err != nil { + log.Fatal(err) + } + }() + // wait for mock server to run + time.Sleep(time.Millisecond * 500) +} + +func teardown() { + mockserver.Close() +} + +func TestApollo(t *testing.T) { + var ( + testAppYAML = "app.yml" + testAppYAMLContent1 = "test: test12234\ntest2: test333" + testAppYAMLContent2 = "test: 1111" + testClientJSON = "client.json" + testClientJSONContent = `{"name":"agollo"}` + ) + os.Setenv("APOLLO_APP_ID", "SampleApp") + os.Setenv("APOLLO_CLUSTER", "default") + os.Setenv("APOLLO_CACHE_DIR", "/tmp") + os.Setenv("APOLLO_META_ADDR", "localhost:8080") + os.Setenv("APOLLO_NAMESPACES", fmt.Sprintf("%s,%s", testAppYAML, testClientJSON)) + mockserver.Set(testAppYAML, "content", testAppYAMLContent1) + mockserver.Set(testClientJSON, "content", testClientJSONContent) + ad := &apolloDriver{} + apollo, err := ad.New() + if err != nil { + t.Fatalf("new apollo error, %v", err) + } + value := apollo.Get(testAppYAML) + if content, _ := value.String(); content != testAppYAMLContent1 { + t.Fatalf("got app.yml unexpected value %s", content) + } + value = apollo.Get(testClientJSON) + if content, _ := value.String(); content != testClientJSONContent { + t.Fatalf("got app.yml unexpected value %s", content) + } + mockserver.Set(testAppYAML, "content", testAppYAMLContent2) + updates := apollo.WatchEvent(context.TODO(), testAppYAML) + select { + case <-updates: + case <-time.After(time.Millisecond * 30000): + } + value = apollo.Get(testAppYAML) + if content, _ := value.String(); content != testAppYAMLContent2 { + t.Fatalf("got app.yml unexpected updated value %s", content) + } +} diff --git a/pkg/conf/paladin/apollo/const.go b/pkg/conf/paladin/apollo/const.go new file mode 100644 index 000000000..1aac397fe --- /dev/null +++ b/pkg/conf/paladin/apollo/const.go @@ -0,0 +1,6 @@ +package apollo + +const ( + // PaladinDriverApollo ... + PaladinDriverApollo = "apollo" +) diff --git a/pkg/conf/paladin/apollo/internal/mockserver/mockserver.go b/pkg/conf/paladin/apollo/internal/mockserver/mockserver.go new file mode 100644 index 000000000..5ed0bc2ba --- /dev/null +++ b/pkg/conf/paladin/apollo/internal/mockserver/mockserver.go @@ -0,0 +1,149 @@ +package mockserver + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "sync" + "time" +) + +type notification struct { + NamespaceName string `json:"namespaceName,omitempty"` + NotificationID int `json:"notificationId,omitempty"` +} + +type result struct { + // AppID string `json:"appId"` + // Cluster string `json:"cluster"` + NamespaceName string `json:"namespaceName"` + Configurations map[string]string `json:"configurations"` + ReleaseKey string `json:"releaseKey"` +} + +type mockServer struct { + server http.Server + + lock sync.Mutex + notifications map[string]int + config map[string]map[string]string +} + +func (s *mockServer) NotificationHandler(rw http.ResponseWriter, req *http.Request) { + s.lock.Lock() + defer s.lock.Unlock() + req.ParseForm() + var notifications []notification + if err := json.Unmarshal([]byte(req.FormValue("notifications")), ¬ifications); err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + var changes []notification + for _, noti := range notifications { + if currentID := s.notifications[noti.NamespaceName]; currentID != noti.NotificationID { + changes = append(changes, notification{NamespaceName: noti.NamespaceName, NotificationID: currentID}) + } + } + + if len(changes) == 0 { + rw.WriteHeader(http.StatusNotModified) + return + } + bts, err := json.Marshal(&changes) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + rw.Write(bts) +} + +func (s *mockServer) ConfigHandler(rw http.ResponseWriter, req *http.Request) { + req.ParseForm() + + strs := strings.Split(req.RequestURI, "/") + var namespace, releaseKey = strings.Split(strs[4], "?")[0], req.FormValue("releaseKey") + config := s.Get(namespace) + + var result = result{NamespaceName: namespace, Configurations: config, ReleaseKey: releaseKey} + bts, err := json.Marshal(&result) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + rw.Write(bts) +} + +var server *mockServer + +func (s *mockServer) Set(namespace, key, value string) { + server.lock.Lock() + defer server.lock.Unlock() + + notificationID := s.notifications[namespace] + notificationID++ + s.notifications[namespace] = notificationID + + if kv, ok := s.config[namespace]; ok { + kv[key] = value + return + } + kv := map[string]string{key: value} + s.config[namespace] = kv +} + +func (s *mockServer) Get(namespace string) map[string]string { + server.lock.Lock() + defer server.lock.Unlock() + + return s.config[namespace] +} + +func (s *mockServer) Delete(namespace, key string) { + server.lock.Lock() + defer server.lock.Unlock() + + if kv, ok := s.config[namespace]; ok { + delete(kv, key) + } + + notificationID := s.notifications[namespace] + notificationID++ + s.notifications[namespace] = notificationID +} + +// Set namespace's key value +func Set(namespace, key, value string) { + server.Set(namespace, key, value) +} + +// Delete namespace's key +func Delete(namespace, key string) { + server.Delete(namespace, key) +} + +// Run mock server +func Run() error { + initServer() + return server.server.ListenAndServe() +} + +func initServer() { + server = &mockServer{ + notifications: map[string]int{}, + config: map[string]map[string]string{}, + } + mux := http.NewServeMux() + mux.Handle("/notifications/", http.HandlerFunc(server.NotificationHandler)) + mux.Handle("/configs/", http.HandlerFunc(server.ConfigHandler)) + server.server.Handler = mux + server.server.Addr = ":8080" +} + +// Close mock server +func Close() error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second)) + defer cancel() + + return server.server.Shutdown(ctx) +} diff --git a/pkg/conf/paladin/default.go b/pkg/conf/paladin/default.go index 566c83720..a9410f563 100644 --- a/pkg/conf/paladin/default.go +++ b/pkg/conf/paladin/default.go @@ -2,6 +2,7 @@ package paladin import ( "context" + "errors" "flag" ) @@ -16,12 +17,30 @@ func init() { } // Init init config client. -func Init() (err error) { +// If confPath is set, it inits file client by default +// Otherwise we could pass args to init remote client +// args[0]: driver name, string type +func Init(args ...interface{}) (err error) { if confPath != "" { DefaultClient, err = NewFile(confPath) } else { - // TODO: Get the configuration from the remote service - panic("Please specify a file or dir name by -conf flag.") + var ( + driver Driver + ) + argsLackErr := errors.New("lack of remote config center args") + if len(args) == 0 { + panic(argsLackErr.Error()) + } + argsInvalidErr := errors.New("invalid remote config center args") + driverName, ok := args[0].(string) + if !ok { + panic(argsInvalidErr.Error()) + } + driver, err = GetDriver(driverName) + if err != nil { + return + } + DefaultClient, err = driver.New() } if err != nil { return diff --git a/pkg/conf/paladin/driver.go b/pkg/conf/paladin/driver.go new file mode 100644 index 000000000..9f2151e4c --- /dev/null +++ b/pkg/conf/paladin/driver.go @@ -0,0 +1,9 @@ +package paladin + +// Driver defined paladin remote client impl +// each remote config center driver must do +// 1. implements `New` method +// 2. call `Register` to register itself +type Driver interface { + New() (Client, error) +} diff --git a/pkg/conf/paladin/example_test.go b/pkg/conf/paladin/example_test.go index d3d365457..110e402b8 100644 --- a/pkg/conf/paladin/example_test.go +++ b/pkg/conf/paladin/example_test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/bilibili/kratos/pkg/conf/paladin" + "github.com/bilibili/kratos/pkg/conf/paladin/apollo" "github.com/BurntSushi/toml" ) @@ -26,7 +27,7 @@ func (e *exampleConf) Set(text string) error { return nil } -// ExampleClient is a example client usage. +// ExampleClient is an example client usage. // exmaple.toml: /* bool = true @@ -56,7 +57,41 @@ func ExampleClient() { }() } -// ExampleMap is a example map usage. +// ExampleApolloClient is an example client for apollo driver usage. +func ExampleApolloClient() { + /* + pass flags or set envs that apollo needs, for example: + + ``` + export APOLLO_APP_ID=SampleApp + export APOLLO_CLUSTER=default + export APOLLO_CACHE_DIR=/tmp + export APOLLO_META_ADDR=localhost:8080 + export APOLLO_NAMESPACES=example.yml + ``` + */ + + if err := paladin.Init(apollo.PaladinDriverApollo); err != nil { + panic(err) + } + var ec exampleConf + // var setter + if err := paladin.Watch("example.yml", &ec); err != nil { + panic(err) + } + if err := paladin.Get("example.yml").UnmarshalYAML(&ec); err != nil { + panic(err) + } + // use exampleConf + // watch event key + go func() { + for event := range paladin.WatchEvent(context.TODO(), "key") { + fmt.Println(event) + } + }() +} + +// ExampleMap is an example map usage. // exmaple.toml: /* bool = true diff --git a/pkg/conf/paladin/file.go b/pkg/conf/paladin/file.go index 0995cf227..09fecf3b1 100644 --- a/pkg/conf/paladin/file.go +++ b/pkg/conf/paladin/file.go @@ -31,7 +31,7 @@ func (w *watcher) HasKey(key string) bool { return true } for _, k := range w.keys { - if keyNamed(k) == key { + if KeyNamed(k) == key { return true } } @@ -138,7 +138,7 @@ func (f *file) reloadFile(fpath string) (err error) { if err != nil { return } - key := keyNamed(path.Base(fpath)) + key := KeyNamed(path.Base(fpath)) raws := f.values.Load() raws[key] = value f.values.Store(raws) @@ -167,7 +167,7 @@ func loadValues(base string) (map[string]*Value, error) { return nil, fmt.Errorf("paladin: read dir %s error: %s", base, err) } for _, file := range files { - if !file.IsDir() { + if !file.IsDir() && (file.Mode()&os.ModeSymlink) != os.ModeSymlink { paths = append(paths, path.Join(base, file.Name())) } } diff --git a/pkg/conf/paladin/map.go b/pkg/conf/paladin/map.go index c262d4050..fa43dc116 100644 --- a/pkg/conf/paladin/map.go +++ b/pkg/conf/paladin/map.go @@ -5,8 +5,8 @@ import ( "sync/atomic" ) -// keyNamed key naming to lower case. -func keyNamed(key string) string { +// KeyNamed key naming to lower case. +func KeyNamed(key string) string { return strings.ToLower(key) } @@ -19,7 +19,7 @@ type Map struct { func (m *Map) Store(values map[string]*Value) { dst := make(map[string]*Value, len(values)) for k, v := range values { - dst[keyNamed(k)] = v + dst[KeyNamed(k)] = v } m.values.Store(dst) } @@ -36,13 +36,13 @@ func (m *Map) Load() map[string]*Value { // Exist check if values map exist a key. func (m *Map) Exist(key string) bool { - _, ok := m.Load()[keyNamed(key)] + _, ok := m.Load()[KeyNamed(key)] return ok } // Get return get value by key. func (m *Map) Get(key string) *Value { - v, ok := m.Load()[keyNamed(key)] + v, ok := m.Load()[KeyNamed(key)] if ok { return v } diff --git a/pkg/conf/paladin/mock.go b/pkg/conf/paladin/mock.go index 1792d95bd..4e705c1de 100644 --- a/pkg/conf/paladin/mock.go +++ b/pkg/conf/paladin/mock.go @@ -13,7 +13,7 @@ type Mock struct { } // NewMock new a config mock client. -func NewMock(vs map[string]string) *Mock { +func NewMock(vs map[string]string) Client { values := make(map[string]*Value, len(vs)) for k, v := range vs { values[k] = &Value{val: v, raw: v} diff --git a/pkg/conf/paladin/register.go b/pkg/conf/paladin/register.go new file mode 100644 index 000000000..400497745 --- /dev/null +++ b/pkg/conf/paladin/register.go @@ -0,0 +1,55 @@ +package paladin + +import ( + "fmt" + "sort" + "sync" +) + +var ( + driversMu sync.RWMutex + drivers = make(map[string]Driver) +) + +// Register makes a paladin driver available by the provided name. +// If Register is called twice with the same name or if driver is nil, +// it panics. +func Register(name string, driver Driver) { + driversMu.Lock() + defer driversMu.Unlock() + + if driver == nil { + panic("paladin: driver is nil") + } + + if _, dup := drivers[name]; dup { + panic("paladin: Register called twice for driver " + name) + } + + drivers[name] = driver +} + +// Drivers returns a sorted list of the names of the registered paladin driver. +func Drivers() []string { + driversMu.RLock() + defer driversMu.RUnlock() + + var list []string + for name := range drivers { + list = append(list, name) + } + + sort.Strings(list) + return list +} + +// GetDriver returns a driver implement by name. +func GetDriver(name string) (Driver, error) { + driversMu.RLock() + driveri, ok := drivers[name] + driversMu.RUnlock() + if !ok { + return nil, fmt.Errorf("paladin: unknown driver %q (forgotten import?)", name) + } + return driveri, nil +} diff --git a/pkg/conf/paladin/toml.go b/pkg/conf/paladin/toml.go index 87f92771c..09595fb0b 100644 --- a/pkg/conf/paladin/toml.go +++ b/pkg/conf/paladin/toml.go @@ -28,7 +28,7 @@ func (m *TOML) UnmarshalText(text []byte) error { } values := map[string]*Value{} for k, v := range raws { - k = keyNamed(k) + k = KeyNamed(k) rv := reflect.ValueOf(v) switch rv.Kind() { case reflect.Map: diff --git a/pkg/conf/paladin/value.go b/pkg/conf/paladin/value.go index bafb86b99..733db7df7 100644 --- a/pkg/conf/paladin/value.go +++ b/pkg/conf/paladin/value.go @@ -25,6 +25,14 @@ type Value struct { raw string } +// NewValue new a value +func NewValue(val interface{}, raw string) *Value { + return &Value{ + val: val, + raw: raw, + } +} + // Bool return bool value. func (v *Value) Bool() (bool, error) { if v.val == nil { @@ -112,7 +120,7 @@ func (v *Value) Raw() (string, error) { return v.raw, nil } -// Slice scan a slcie interface, if slice has element it will be discard. +// Slice scan a slice interface, if slice has element it will be discard. func (v *Value) Slice(dst interface{}) error { // NOTE: val is []interface{}, slice is []type if v.val == nil { @@ -167,6 +175,7 @@ func (v *Value) UnmarshalJSON(dst interface{}) error { return json.Unmarshal([]byte(text), dst) } +// UnmarshalYAML unmarshal yaml to struct. func (v *Value) UnmarshalYAML(dst interface{}) error { text, err := v.Raw() if err != nil { diff --git a/pkg/database/hbase/metrics.go b/pkg/database/hbase/metrics.go index 4867271d5..a29f82e3f 100644 --- a/pkg/database/hbase/metrics.go +++ b/pkg/database/hbase/metrics.go @@ -41,8 +41,8 @@ func codeFromErr(err error) string { code = "connot_find_region" case gohbase.TableNotFound: code = "table_not_found" - //case gohbase.ErrRegionUnavailable: - // code = "region_unavailable" + //case gohbase.ErrRegionUnavailable: + // code = "region_unavailable" } return code } diff --git a/pkg/ecode/status.go b/pkg/ecode/status.go index e690e79d0..8268f26ef 100644 --- a/pkg/ecode/status.go +++ b/pkg/ecode/status.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/bilibili/kratos/pkg/ecode/types" + "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" ) @@ -81,13 +82,13 @@ func (s *Status) Proto() *types.Status { // FromCode create status from ecode func FromCode(code Code) *Status { - return &Status{s: &types.Status{Code: int32(code)}} + return &Status{s: &types.Status{Code: int32(code), Message: code.Message()}} } // FromProto new status from grpc detail func FromProto(pbMsg proto.Message) Codes { if msg, ok := pbMsg.(*types.Status); ok { - if msg.Message == "" { + if msg.Message == "" || msg.Message == strconv.FormatInt(int64(msg.Code), 10) { // NOTE: if message is empty convert to pure Code, will get message from config center. return Code(msg.Code) } diff --git a/pkg/naming/discovery/discovery.go b/pkg/naming/discovery/discovery.go index 635229ce4..9876315a1 100644 --- a/pkg/naming/discovery/discovery.go +++ b/pkg/naming/discovery/discovery.go @@ -102,7 +102,7 @@ type appInfo struct { } func fixConfig(c *Config) error { - if len(c.Nodes) == 0 { + if len(c.Nodes) == 0 && env.DiscoveryNodes != "" { c.Nodes = strings.Split(env.DiscoveryNodes, ",") } if c.Region == "" { @@ -370,9 +370,15 @@ func (d *Discovery) register(ctx context.Context, ins *naming.Instance) (err err uri := fmt.Sprintf(_registerURL, d.pickNode()) params := d.newParams(c) params.Set("appid", ins.AppID) - params.Set("addrs", strings.Join(ins.Addrs, ",")) + for _, addr := range ins.Addrs { + params.Add("addrs", addr) + } params.Set("version", ins.Version) - params.Set("status", _statusUP) + if ins.Status == 0 { + params.Set("status", _statusUP) + } else { + params.Set("status", strconv.FormatInt(ins.Status, 10)) + } params.Set("metadata", string(metadata)) if err = d.httpClient.Post(ctx, uri, "", params, &res); err != nil { d.switchNode() @@ -469,7 +475,7 @@ func (d *Discovery) set(ctx context.Context, ins *naming.Instance) (err error) { params := d.newParams(conf) params.Set("appid", ins.AppID) params.Set("version", ins.Version) - params.Set("status", _statusUP) + params.Set("status", strconv.FormatInt(ins.Status, 10)) if ins.Metadata != nil { var metadata []byte if metadata, err = json.Marshal(ins.Metadata); err != nil { diff --git a/pkg/naming/etcd/etcd_test.go b/pkg/naming/etcd/etcd_test.go index 72d15aa15..5dfd2e539 100644 --- a/pkg/naming/etcd/etcd_test.go +++ b/pkg/naming/etcd/etcd_test.go @@ -3,11 +3,11 @@ package etcd import ( "context" "fmt" - "testing" - "time" "github.com/bilibili/kratos/pkg/naming" "go.etcd.io/etcd/clientv3" "google.golang.org/grpc" + "testing" + "time" ) func TestNew(t *testing.T) { diff --git a/pkg/naming/naming.go b/pkg/naming/naming.go index e67d9352a..5e0d76348 100644 --- a/pkg/naming/naming.go +++ b/pkg/naming/naming.go @@ -35,6 +35,8 @@ type Instance struct { // Metadata is the information associated with Addr, which may be used // to make load balancing decision. Metadata map[string]string `json:"metadata"` + // Status instance status, eg: 1UP 2Waiting + Status int64 `json:"status"` } // Resolver resolve naming service diff --git a/pkg/net/http/blademaster/context.go b/pkg/net/http/blademaster/context.go index d9957b19e..d660ecac9 100644 --- a/pkg/net/http/blademaster/context.go +++ b/pkg/net/http/blademaster/context.go @@ -49,7 +49,6 @@ type Context struct { RoutePath string Params Params - } /************************************/ @@ -67,7 +66,6 @@ func (c *Context) Next() { } } - // Abort prevents pending handlers from being called. Note that this will not stop the current handler. // Let's say you have an authorization middleware that validates that the current request is authorized. // If the authorization fails (ex: the password does not match), call Abort to ensure the remaining handlers @@ -276,9 +274,17 @@ func (c *Context) BindWith(obj interface{}, b binding.Binding) error { return c.mustBindWith(obj, b) } -// Bind bind req arg with defult form binding. +// Bind checks the Content-Type to select a binding engine automatically, +// Depending the "Content-Type" header different bindings are used: +// "application/json" --> JSON binding +// "application/xml" --> XML binding +// otherwise --> returns an error. +// It parses the request's body as JSON if Content-Type == "application/json" using JSON or XML as a JSON input. +// It decodes the json payload into the struct specified as a pointer. +// It writes a 400 error and sets Content-Type header "text/plain" in the response if input is not valid. func (c *Context) Bind(obj interface{}) error { - return c.mustBindWith(obj, binding.Form) + b := binding.Default(c.Request.Method, c.Request.Header.Get("Content-Type")) + return c.mustBindWith(obj, b) } // mustBindWith binds the passed struct pointer using the specified binding engine. diff --git a/pkg/net/http/blademaster/logger.go b/pkg/net/http/blademaster/logger.go index 89143fab0..b4c353fe0 100644 --- a/pkg/net/http/blademaster/logger.go +++ b/pkg/net/http/blademaster/logger.go @@ -34,8 +34,10 @@ func Logger() HandlerFunc { caller = noUser } - _metricServerReqCodeTotal.Inc(c.RoutePath[1:], caller, strconv.FormatInt(int64(cerr.Code()), 10)) - _metricServerReqDur.Observe(int64(dt/time.Millisecond), c.RoutePath[1:], caller) + if len(c.RoutePath) > 0 { + _metricServerReqCodeTotal.Inc(c.RoutePath[1:], caller, strconv.FormatInt(int64(cerr.Code()), 10)) + _metricServerReqDur.Observe(int64(dt/time.Millisecond), c.RoutePath[1:], caller) + } lf := log.Infov errmsg := "" diff --git a/pkg/net/http/blademaster/perf.go b/pkg/net/http/blademaster/perf.go index a74e22ecc..ad6c1cbb9 100644 --- a/pkg/net/http/blademaster/perf.go +++ b/pkg/net/http/blademaster/perf.go @@ -19,28 +19,45 @@ var ( func init() { v := os.Getenv("HTTP_PERF") - if v == "" { - v = "tcp://0.0.0.0:2333" - } flag.StringVar(&_perfDSN, "http.perf", v, "listen http perf dsn, or use HTTP_PERF env variable.") } -func startPerf() { +func startPerf(engine *Engine) { _perfOnce.Do(func() { - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + if os.Getenv("HTTP_PERF") == "" { + prefixRouter := engine.Group("/debug/pprof") + { + prefixRouter.GET("/", pprofHandler(pprof.Index)) + prefixRouter.GET("/cmdline", pprofHandler(pprof.Cmdline)) + prefixRouter.GET("/profile", pprofHandler(pprof.Profile)) + prefixRouter.POST("/symbol", pprofHandler(pprof.Symbol)) + prefixRouter.GET("/symbol", pprofHandler(pprof.Symbol)) + prefixRouter.GET("/trace", pprofHandler(pprof.Trace)) + prefixRouter.GET("/allocs", pprofHandler(pprof.Handler("allocs").ServeHTTP)) + prefixRouter.GET("/block", pprofHandler(pprof.Handler("block").ServeHTTP)) + prefixRouter.GET("/goroutine", pprofHandler(pprof.Handler("goroutine").ServeHTTP)) + prefixRouter.GET("/heap", pprofHandler(pprof.Handler("heap").ServeHTTP)) + prefixRouter.GET("/mutex", pprofHandler(pprof.Handler("mutex").ServeHTTP)) + prefixRouter.GET("/threadcreate", pprofHandler(pprof.Handler("threadcreate").ServeHTTP)) + } + return + } go func() { d, err := dsn.Parse(_perfDSN) if err != nil { panic(errors.Errorf("blademaster: http perf dsn must be tcp://$host:port, %s:error(%v)", _perfDSN, err)) } - if err := http.ListenAndServe(d.Host, mux); err != nil { + if err := http.ListenAndServe(d.Host, nil); err != nil { panic(errors.Errorf("blademaster: listen %s: error(%v)", d.Host, err)) } }() }) } + +func pprofHandler(h http.HandlerFunc) HandlerFunc { + handler := http.HandlerFunc(h) + return func(c *Context) { + handler.ServeHTTP(c.Writer, c.Request) + } +} diff --git a/pkg/net/http/blademaster/server.go b/pkg/net/http/blademaster/server.go index 29b47f2eb..6fa12ac56 100644 --- a/pkg/net/http/blademaster/server.go +++ b/pkg/net/http/blademaster/server.go @@ -195,7 +195,7 @@ func NewServer(conf *ServerConfig) *Engine { c.Bytes(405, "text/plain", []byte(http.StatusText(405))) c.Abort() }) - startPerf() + startPerf(engine) return engine } diff --git a/pkg/net/ip/ip.go b/pkg/net/ip/ip.go index 386d7e23f..0966ccc5e 100644 --- a/pkg/net/ip/ip.go +++ b/pkg/net/ip/ip.go @@ -68,7 +68,7 @@ func InternalIP() string { return "" } -// isUp Interface is up +// isUp Interface is up func isUp(v net.Flags) bool { - return v&net.FlagUp == net.FlagUp + return v&net.FlagUp == net.FlagUp } diff --git a/pkg/net/netutil/breaker/README.md b/pkg/net/netutil/breaker/README.md index f700157f5..d4294a9e2 100644 --- a/pkg/net/netutil/breaker/README.md +++ b/pkg/net/netutil/breaker/README.md @@ -11,10 +11,9 @@ > 4. 默认配置如下所示： _conf = &Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(100 * time.Millisecond), Bucket: 10, - Ratio: 0.5, Request: 100, + K:1.5, } ##### 测试 diff --git a/pkg/net/netutil/breaker/breaker.go b/pkg/net/netutil/breaker/breaker.go index f7b681ac3..2d4d6ecbd 100644 --- a/pkg/net/netutil/breaker/breaker.go +++ b/pkg/net/netutil/breaker/breaker.go @@ -11,10 +11,6 @@ import ( type Config struct { SwitchOff bool // breaker switch,default off. - // Hystrix - Ratio float32 - Sleep xtime.Duration - // Google K float64 @@ -30,12 +26,6 @@ func (conf *Config) fix() { if conf.Request == 0 { conf.Request = 100 } - if conf.Ratio == 0 { - conf.Ratio = 0.5 - } - if conf.Sleep == 0 { - conf.Sleep = xtime.Duration(500 * time.Millisecond) - } if conf.Bucket == 0 { conf.Bucket = 10 } @@ -84,8 +74,6 @@ var ( Bucket: 10, Request: 100, - Sleep: xtime.Duration(500 * time.Millisecond), - Ratio: 0.5, // Percentage of failures must be lower than 33.33% K: 1.5, diff --git a/pkg/net/netutil/breaker/breaker_test.go b/pkg/net/netutil/breaker/breaker_test.go index 4e024251f..6f4b2fc43 100644 --- a/pkg/net/netutil/breaker/breaker_test.go +++ b/pkg/net/netutil/breaker/breaker_test.go @@ -28,9 +28,7 @@ func TestGroup(t *testing.T) { g := NewGroup(_conf) c := &Config{ Window: xtime.Duration(1 * time.Second), - Sleep: xtime.Duration(100 * time.Millisecond), Bucket: 10, - Ratio: 0.5, Request: 100, SwitchOff: !_conf.SwitchOff, } @@ -44,9 +42,7 @@ func TestInit(t *testing.T) { switchOff := _conf.SwitchOff c := &Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(100 * time.Millisecond), Bucket: 10, - Ratio: 0.5, Request: 100, SwitchOff: !switchOff, } @@ -69,9 +65,7 @@ func TestGo(t *testing.T) { _group.Reload(&Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(100 * time.Millisecond), Bucket: 10, - Ratio: 0.5, Request: 100, SwitchOff: true, }) diff --git a/pkg/net/netutil/breaker/example_test.go b/pkg/net/netutil/breaker/example_test.go index f0f386c93..9afe49d30 100644 --- a/pkg/net/netutil/breaker/example_test.go +++ b/pkg/net/netutil/breaker/example_test.go @@ -12,9 +12,8 @@ import ( func ExampleGroup() { c := &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(100 * time.Millisecond), + K: 1.5, Bucket: 10, - Ratio: 0.5, Request: 100, } // init default config diff --git a/pkg/net/rpc/warden/CHANGELOG.md b/pkg/net/rpc/warden/CHANGELOG.md deleted file mode 100644 index 10965198c..000000000 --- a/pkg/net/rpc/warden/CHANGELOG.md +++ /dev/null @@ -1,91 +0,0 @@ -### net/rpc/warden - -##### Version 1.1.21 -1. fix resolver bug - -##### Version 1.1.20 -1. client增加timeoutCallOpt强制覆盖每次请求的timeout - -##### Version 1.1.19 -1. 升级grpc至1.22.0 -2. client增加keepAlive选项 - -##### Version 1.1.18 -1. 修复resolver过滤导致的子集bug - -##### Version 1.1.17 -1. 移除 bbr feature flag，默认开启自适应限流 - -##### Version 1.1.16 -1. 使用 flag(grpc.bbr) 绑定 BBR 限流 - -##### Version 1.1.15 -1. warden使用 metadata.Range 方法 - -##### Version 1.1.14 -1. 为 server log 添加选项 - -##### Version 1.1.13 -1. 为 client log 添加选项 - -##### Version 1.1.12 -1. 设置 caller 为 no_user 如果 user 不存在 - -##### Version 1.1.12 -1. warden支持mirror传递 - -##### Version 1.1.11 -1. Validate RequestErr支持详细报错信息 - -##### Version 1.1.10 -1. 默认读取环境中的color - -##### Version 1.1.9 -1. 增加NonBlock模式 - -##### Version 1.1.8 -1. 新增appid mock - -##### Version 1.1.7 -1. 兼容cpu为0和wrr dt为0的情况 - -##### Version 1.1.6 -1. 修改caller传递和获取方式 -2. 添加error detail example - -##### Version 1.1.5 -1. 增加server端json格式支持 - -##### Version 1.1.4 -1. 判断reosvler.builder为nil之后再注册 - -##### Version 1.1.3 -1. 支持zone和clusters - -##### Version 1.1.2 -1. 业务错误日志记为 WARN - -##### Version 1.1.1 -1. server实现了返回cpu信息 - -##### Version 1.1.0 -1. 增加ErrorDetail -2. 修复日志打印error信息丢失问题 - -##### Version 1.0.3 -1. 给server增加keepalive参数 - -##### Version 1.0.2 - -1. 替代默认的timoue，使用durtaion.Shrink()来传递context -2. 修复peer.Addr为nil时会panic的问题 - -##### Version 1.0.1 - -1. 去除timeout的手动传递，改为使用grpc默认自带的grpc-timeout -2. 获取server address改为使用call option的方式，去除对balancer的依赖 - -##### Version 1.0.0 - -1. 使用NewClient来新建一个RPC客户端，并默认集成trace、log、recovery、moniter拦截器 -2. 使用NewServer来新建一个RPC服务端，并默认集成trace、log、recovery、moniter拦截器 diff --git a/pkg/net/rpc/warden/balancer/wrr/wrr_test.go b/pkg/net/rpc/warden/balancer/wrr/wrr_test.go index f9e21f36e..c0b26262f 100644 --- a/pkg/net/rpc/warden/balancer/wrr/wrr_test.go +++ b/pkg/net/rpc/warden/balancer/wrr/wrr_test.go @@ -137,8 +137,8 @@ func TestBalancerDone(t *testing.T) { latency, count := picker.(*wrrPicker).subConns[0].latencySummary() expectLatency := float64(100*time.Millisecond) / 1e5 - if !(expectLatency < latency && latency < (expectLatency+100)) { - t.Fatalf("latency is less than 100ms or greter than 100ms, %f", latency) + if latency < expectLatency || latency > (expectLatency+500) { + t.Fatalf("latency is less than 100ms or greater than 150ms, %f", latency) } assert.Equal(t, int64(1), count) diff --git a/pkg/net/rpc/warden/exapmle_test.go b/pkg/net/rpc/warden/exapmle_test.go index 5461e7b2e..10c037958 100644 --- a/pkg/net/rpc/warden/exapmle_test.go +++ b/pkg/net/rpc/warden/exapmle_test.go @@ -60,9 +60,8 @@ func ExampleClient() { Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), Bucket: 10, - Ratio: 0.3, + K: 1.5, Request: 20, }, }) diff --git a/pkg/net/rpc/warden/internal/benchmark/bench/client/client.go b/pkg/net/rpc/warden/internal/benchmark/bench/client/client.go index 137e90b82..62710092a 100644 --- a/pkg/net/rpc/warden/internal/benchmark/bench/client/client.go +++ b/pkg/net/rpc/warden/internal/benchmark/bench/client/client.go @@ -39,10 +39,9 @@ func wardenCli() proto.HelloClient { Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), Bucket: 10, - Ratio: 0.3, Request: 20, + K: 1.5, }, }, grpc.WithInitialWindowSize(iws), diff --git a/pkg/net/rpc/warden/internal/benchmark/helloworld/client/greeter_client.go b/pkg/net/rpc/warden/internal/benchmark/helloworld/client/greeter_client.go index ada604806..d47c50e43 100644 --- a/pkg/net/rpc/warden/internal/benchmark/helloworld/client/greeter_client.go +++ b/pkg/net/rpc/warden/internal/benchmark/helloworld/client/greeter_client.go @@ -21,10 +21,9 @@ var ( Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), Bucket: 10, - Ratio: 0.3, Request: 20, + K: 1.5, }, } cli pb.GreeterClient diff --git a/pkg/net/rpc/warden/internal/pb/ecode.go b/pkg/net/rpc/warden/internal/pb/ecode.go deleted file mode 100644 index 803e4fb01..000000000 --- a/pkg/net/rpc/warden/internal/pb/ecode.go +++ /dev/null @@ -1,48 +0,0 @@ -package pb - -import ( - "strconv" - - "github.com/bilibili/kratos/pkg/ecode" - - any "github.com/golang/protobuf/ptypes/any" -) - -func (e *Error) Error() string { - return strconv.FormatInt(int64(e.GetErrCode()), 10) -} - -// Code is the code of error. -func (e *Error) Code() int { - return int(e.GetErrCode()) -} - -// Message is error message. -func (e *Error) Message() string { - return e.GetErrMessage() -} - -// Equal compare whether two errors are equal. -func (e *Error) Equal(ec error) bool { - return ecode.Cause(ec).Code() == e.Code() -} - -// Details return error details. -func (e *Error) Details() []interface{} { - return []interface{}{e.GetErrDetail()} -} - -// From will convert ecode.Codes to pb.Error. -// -// Deprecated: please use ecode.Error -func From(ec ecode.Codes) *Error { - var detail *any.Any - if details := ec.Details(); len(details) > 0 { - detail, _ = details[0].(*any.Any) - } - return &Error{ - ErrCode: int32(ec.Code()), - ErrMessage: ec.Message(), - ErrDetail: detail, - } -} diff --git a/pkg/net/rpc/warden/internal/pb/ecode.pb.go b/pkg/net/rpc/warden/internal/pb/ecode.pb.go deleted file mode 100644 index 50146f804..000000000 --- a/pkg/net/rpc/warden/internal/pb/ecode.pb.go +++ /dev/null @@ -1,96 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: error.proto - -package pb - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import any "github.com/golang/protobuf/ptypes/any" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -// Deprecated: please use ecode.Error -type Error struct { - ErrCode int32 `protobuf:"varint,1,opt,name=err_code,json=errCode,proto3" json:"err_code,omitempty"` - ErrMessage string `protobuf:"bytes,2,opt,name=err_message,json=errMessage,proto3" json:"err_message,omitempty"` - ErrDetail *any.Any `protobuf:"bytes,3,opt,name=err_detail,json=errDetail,proto3" json:"err_detail,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Error) Reset() { *m = Error{} } -func (m *Error) String() string { return proto.CompactTextString(m) } -func (*Error) ProtoMessage() {} -func (*Error) Descriptor() ([]byte, []int) { - return fileDescriptor_error_28aad86a4e53115b, []int{0} -} -func (m *Error) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Error.Unmarshal(m, b) -} -func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Error.Marshal(b, m, deterministic) -} -func (dst *Error) XXX_Merge(src proto.Message) { - xxx_messageInfo_Error.Merge(dst, src) -} -func (m *Error) XXX_Size() int { - return xxx_messageInfo_Error.Size(m) -} -func (m *Error) XXX_DiscardUnknown() { - xxx_messageInfo_Error.DiscardUnknown(m) -} - -var xxx_messageInfo_Error proto.InternalMessageInfo - -func (m *Error) GetErrCode() int32 { - if m != nil { - return m.ErrCode - } - return 0 -} - -func (m *Error) GetErrMessage() string { - if m != nil { - return m.ErrMessage - } - return "" -} - -func (m *Error) GetErrDetail() *any.Any { - if m != nil { - return m.ErrDetail - } - return nil -} - -func init() { - proto.RegisterType((*Error)(nil), "err.Error") -} - -func init() { proto.RegisterFile("error.proto", fileDescriptor_error_28aad86a4e53115b) } - -var fileDescriptor_error_28aad86a4e53115b = []byte{ - // 164 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x8d, 0xc1, 0xca, 0x82, 0x40, - 0x14, 0x85, 0x99, 0x5f, 0xfc, 0xcb, 0x71, 0x37, 0xb4, 0xd0, 0x36, 0x49, 0x2b, 0x57, 0x23, 0xe4, - 0x13, 0x44, 0xb5, 0x6c, 0xe3, 0x0b, 0x88, 0xe6, 0x49, 0x02, 0xf3, 0xc6, 0xd1, 0x20, 0xdf, 0x3e, - 0x1c, 0x69, 0x79, 0xcf, 0xf7, 0x71, 0x3f, 0x1d, 0x82, 0x14, 0xda, 0x17, 0x65, 0x14, 0xe3, 0x81, - 0xdc, 0xc6, 0xad, 0x48, 0xdb, 0x21, 0x73, 0x53, 0xfd, 0xbe, 0x67, 0x55, 0x3f, 0x2d, 0x7c, 0xff, - 0xd1, 0xfe, 0x65, 0xd6, 0x4d, 0xac, 0xd7, 0x20, 0xcb, 0x9b, 0x34, 0x88, 0x54, 0xa2, 0x52, 0xbf, - 0x58, 0x81, 0x3c, 0x49, 0x03, 0xb3, 0x73, 0x2f, 0xcb, 0x27, 0x86, 0xa1, 0x6a, 0x11, 0xfd, 0x25, - 0x2a, 0x0d, 0x0a, 0x0d, 0xf2, 0xba, 0x2c, 0x26, 0xd7, 0xf3, 0x55, 0x36, 0x18, 0xab, 0x47, 0x17, - 0x79, 0x89, 0x4a, 0xc3, 0xc3, 0xc6, 0x2e, 0x51, 0xfb, 0x8b, 0xda, 0x63, 0x3f, 0x15, 0x01, 0xc8, - 0xb3, 0xd3, 0xea, 0x7f, 0x07, 0xf2, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf7, 0x41, 0x22, 0xfd, - 0xaf, 0x00, 0x00, 0x00, -} diff --git a/pkg/net/rpc/warden/internal/pb/ecode.proto b/pkg/net/rpc/warden/internal/pb/ecode.proto deleted file mode 100644 index 1a18c6957..000000000 --- a/pkg/net/rpc/warden/internal/pb/ecode.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; - -package pb; - -import "google/protobuf/any.proto"; - -option go_package = "github.com/bilibili/kratos/pkg/net/rpc/warden/internal/pb"; - -message Error { - int32 err_code = 1; - string err_message = 2; - google.protobuf.Any err_detail = 3; -} diff --git a/pkg/net/rpc/warden/internal/status/status.go b/pkg/net/rpc/warden/internal/status/status.go index a54eaa437..e5ceeb241 100644 --- a/pkg/net/rpc/warden/internal/status/status.go +++ b/pkg/net/rpc/warden/internal/status/status.go @@ -4,14 +4,12 @@ import ( "context" "strconv" + "github.com/bilibili/kratos/pkg/ecode" + "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" "github.com/pkg/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - - "github.com/bilibili/kratos/pkg/ecode" - "github.com/bilibili/kratos/pkg/net/rpc/warden/internal/pb" ) // togRPCCode convert ecode.Codo to gRPC code @@ -93,9 +91,6 @@ func FromError(svrErr error) (gst *status.Status) { func gRPCStatusFromEcode(code ecode.Codes) (*status.Status, error) { var st *ecode.Status switch v := code.(type) { - // compatible old pb.Error remove it after nobody use pb.Error. - case *pb.Error: - return status.New(codes.Unknown, v.Error()).WithDetails(v) case *ecode.Status: st = v case ecode.Code: @@ -108,40 +103,14 @@ func gRPCStatusFromEcode(code ecode.Codes) (*status.Status, error) { } } } - // gst := status.New(togRPCCode(st), st.Message()) - // NOTE: compatible with PHP swoole gRPC put code in status message as string. - // gst := status.New(togRPCCode(st), strconv.Itoa(st.Code())) gst := status.New(codes.Unknown, strconv.Itoa(st.Code())) - pbe := &pb.Error{ErrCode: int32(st.Code()), ErrMessage: gst.Message()} - // NOTE: server return ecode.Status will be covert to pb.Error details will be ignored - // and put it at details[0] for compatible old client - return gst.WithDetails(pbe, st.Proto()) + return gst.WithDetails(st.Proto()) } // ToEcode convert grpc.status to ecode.Codes func ToEcode(gst *status.Status) ecode.Codes { details := gst.Details() - // reverse range details, details may contain three case, - // if details contain pb.Error and ecode.Status use eocde.Status first. - // - // Details layout: - // pb.Error [0: pb.Error] - // both pb.Error and ecode.Status [0: pb.Error, 1: ecode.Status] - // ecode.Status [0: ecode.Status] - for i := len(details) - 1; i >= 0; i-- { - detail := details[i] - // compatible with old pb.Error. - if pe, ok := detail.(*pb.Error); ok { - st := ecode.Error(ecode.Code(pe.ErrCode), pe.ErrMessage) - if pe.ErrDetail != nil { - dynMsg := new(ptypes.DynamicAny) - // TODO deal with unmarshalAny error. - if err := ptypes.UnmarshalAny(pe.ErrDetail, dynMsg); err == nil { - st, _ = st.WithDetails(dynMsg.Message) - } - } - return st - } + for _, detail := range details { // convert detail to status only use first detail if pb, ok := detail.(proto.Message); ok { return ecode.FromProto(pb) diff --git a/pkg/net/rpc/warden/internal/status/status_test.go b/pkg/net/rpc/warden/internal/status/status_test.go index 43013e9b4..918673630 100644 --- a/pkg/net/rpc/warden/internal/status/status_test.go +++ b/pkg/net/rpc/warden/internal/status/status_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/timestamp" pkgerr "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -15,7 +14,6 @@ import ( "google.golang.org/grpc/status" "github.com/bilibili/kratos/pkg/ecode" - "github.com/bilibili/kratos/pkg/net/rpc/warden/internal/pb" ) func TestCodeConvert(t *testing.T) { @@ -89,16 +87,6 @@ func TestFromError(t *testing.T) { assert.Equal(t, codes.Unknown, gst.Code()) assert.Equal(t, "-504", gst.Message()) }) - t.Run("input pb.Error", func(t *testing.T) { - m := ×tamp.Timestamp{Seconds: time.Now().Unix()} - detail, _ := ptypes.MarshalAny(m) - err := &pb.Error{ErrCode: 2233, ErrMessage: "message", ErrDetail: detail} - gst := FromError(err) - - assert.Equal(t, codes.Unknown, gst.Code()) - assert.Len(t, gst.Details(), 1) - assert.Equal(t, "2233", gst.Message()) - }) t.Run("input ecode.Status", func(t *testing.T) { m := ×tamp.Timestamp{Seconds: time.Now().Unix()} err, _ := ecode.Error(ecode.Unauthorized, "unauthorized").WithDetails(m) @@ -107,10 +95,9 @@ func TestFromError(t *testing.T) { //assert.Equal(t, codes.Unauthenticated, gst.Code()) // NOTE: set all grpc.status as Unkown when error is ecode.Codes for compatible assert.Equal(t, codes.Unknown, gst.Code()) - assert.Len(t, gst.Details(), 2) + assert.Len(t, gst.Details(), 1) details := gst.Details() - assert.IsType(t, &pb.Error{}, details[0]) - assert.IsType(t, err.Proto(), details[1]) + assert.IsType(t, err.Proto(), details[0]) }) } @@ -123,32 +110,6 @@ func TestToEcode(t *testing.T) { assert.Equal(t, "-500", ec.Message()) assert.Len(t, ec.Details(), 0) }) - - t.Run("input pb.Error", func(t *testing.T) { - m := ×tamp.Timestamp{Seconds: time.Now().Unix()} - detail, _ := ptypes.MarshalAny(m) - gst := status.New(codes.InvalidArgument, "requesterr") - gst, _ = gst.WithDetails(&pb.Error{ErrCode: 1122, ErrMessage: "message", ErrDetail: detail}) - ec := ToEcode(gst) - - assert.Equal(t, 1122, ec.Code()) - assert.Equal(t, "message", ec.Message()) - assert.Len(t, ec.Details(), 1) - assert.IsType(t, m, ec.Details()[0]) - }) - - t.Run("input pb.Error and ecode.Status", func(t *testing.T) { - gst := status.New(codes.InvalidArgument, "requesterr") - gst, _ = gst.WithDetails( - &pb.Error{ErrCode: 401, ErrMessage: "message"}, - ecode.Errorf(ecode.Unauthorized, "Unauthorized").Proto(), - ) - ec := ToEcode(gst) - - assert.Equal(t, int(ecode.Unauthorized), ec.Code()) - assert.Equal(t, "Unauthorized", ec.Message()) - }) - t.Run("input encode.Status", func(t *testing.T) { m := ×tamp.Timestamp{Seconds: time.Now().Unix()} st, _ := ecode.Errorf(ecode.Unauthorized, "Unauthorized").WithDetails(m) diff --git a/pkg/net/rpc/warden/resolver/direct/test/direct_test.go b/pkg/net/rpc/warden/resolver/direct/test/direct_test.go index a3de922bf..b898fd643 100644 --- a/pkg/net/rpc/warden/resolver/direct/test/direct_test.go +++ b/pkg/net/rpc/warden/resolver/direct/test/direct_test.go @@ -55,10 +55,9 @@ func createTestClient(t *testing.T, connStr string) pb.GreeterClient { Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), Bucket: 10, - Ratio: 0.3, Request: 20, + K: 1.5, }, }) conn, err := client.Dial(context.TODO(), connStr) diff --git a/pkg/net/rpc/warden/resolver/test/resovler_test.go b/pkg/net/rpc/warden/resolver/test/resovler_test.go index 6e69fcb3b..7af8873a0 100644 --- a/pkg/net/rpc/warden/resolver/test/resovler_test.go +++ b/pkg/net/rpc/warden/resolver/test/resovler_test.go @@ -74,10 +74,9 @@ func createTestClient(t *testing.T) pb.GreeterClient { Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), Bucket: 10, - Ratio: 0.3, Request: 20, + K: 1.5, }, }) conn, err := client.Dial(context.TODO(), "mockdiscovery://authority/main.test") diff --git a/pkg/net/rpc/warden/server.go b/pkg/net/rpc/warden/server.go index 60656cd22..ade2c2b8a 100644 --- a/pkg/net/rpc/warden/server.go +++ b/pkg/net/rpc/warden/server.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" "google.golang.org/grpc" + _ "google.golang.org/grpc/encoding/gzip" // NOTE: use grpc gzip by header grpc-accept-encoding "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -110,6 +111,7 @@ func (s *Server) handle() grpc.UnaryServerInterceptor { var t trace.Trace cmd := nmd.MD{} if gmd, ok := metadata.FromIncomingContext(ctx); ok { + t, _ = trace.Extract(trace.GRPCFormat, gmd) for key, vals := range gmd { if nmd.IsIncomingKey(key) { cmd[key] = vals[0] @@ -300,6 +302,26 @@ func (s *Server) RunUnix(file string) error { // will panic if any error happend // return server itself func (s *Server) Start() (*Server, error) { + _, err := s.startWithAddr() + if err != nil { + return nil, err + } + return s, nil +} + +// StartWithAddr create a new goroutine run server with configured listen addr +// will panic if any error happend +// return server itself and the actually listened address (if configured listen +// port is zero, the os will allocate an unused port) +func (s *Server) StartWithAddr() (*Server, net.Addr, error) { + addr, err := s.startWithAddr() + if err != nil { + return nil, nil, err + } + return s, addr, nil +} + +func (s *Server) startWithAddr() (net.Addr, error) { lis, err := net.Listen(s.conf.Network, s.conf.Addr) if err != nil { return nil, err @@ -311,7 +333,7 @@ func (s *Server) Start() (*Server, error) { panic(err) } }() - return s, nil + return lis.Addr(), nil } // Serve accepts incoming connections on the listener lis, creating a new diff --git a/pkg/net/rpc/warden/server_test.go b/pkg/net/rpc/warden/server_test.go index 248caff4e..5d0ea2afa 100644 --- a/pkg/net/rpc/warden/server_test.go +++ b/pkg/net/rpc/warden/server_test.go @@ -41,11 +41,9 @@ var ( Dial: xtime.Duration(time.Second * 10), Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ - Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), - Bucket: 10, - Ratio: 0.3, - Request: 20, + Window: xtime.Duration(3 * time.Second), + Bucket: 10, + K: 1.5, }, } clientConfig2 = ClientConfig{ @@ -53,10 +51,9 @@ var ( Timeout: xtime.Duration(time.Second * 10), Breaker: &breaker.Config{ Window: xtime.Duration(3 * time.Second), - Sleep: xtime.Duration(3 * time.Second), Bucket: 10, - Ratio: 0.3, Request: 20, + K: 1.5, }, Method: map[string]*ClientConfig{`/testproto.Greeter/SayHello`: {Timeout: xtime.Duration(time.Millisecond * 200)}}, } @@ -293,7 +290,7 @@ func testBreaker(t *testing.T) { } defer conn.Close() c := pb.NewGreeterClient(conn) - for i := 0; i < 50; i++ { + for i := 0; i < 1000; i++ { _, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: "breaker_test"}) if err != nil { if ecode.EqualError(ecode.ServiceUnavailable, err) { @@ -594,3 +591,15 @@ func TestMetadata(t *testing.T) { _, err := cli.SayHello(ctx, &pb.HelloRequest{Name: "test"}) assert.Nil(t, err) } + +func TestStartWithAddr(t *testing.T) { + configuredAddr := "127.0.0.1:0" + server = NewServer(&ServerConfig{Addr: configuredAddr, Timeout: xtime.Duration(time.Second)}) + if _, realAddr, err := server.StartWithAddr(); err == nil && realAddr != nil { + assert.NotEqual(t, realAddr.String(), configuredAddr) + } else { + assert.NotNil(t, realAddr) + assert.Nil(t, err) + } +} + diff --git a/pkg/net/trace/sample_test.go b/pkg/net/trace/sample_test.go index 7ccd01b36..329ec5f16 100644 --- a/pkg/net/trace/sample_test.go +++ b/pkg/net/trace/sample_test.go @@ -21,8 +21,8 @@ func TestProbabilitySampling(t *testing.T) { count++ } } - if count < 60 || count > 120 { - t.Errorf("expect count between 60~120 get %d", count) + if count < 60 || count > 150 { + t.Errorf("expect count between 60~150 get %d", count) } }) } diff --git a/pkg/net/trace/zipkin/zipkin_test.go b/pkg/net/trace/zipkin/zipkin_test.go index a68884a91..c65dd5888 100644 --- a/pkg/net/trace/zipkin/zipkin_test.go +++ b/pkg/net/trace/zipkin/zipkin_test.go @@ -37,7 +37,7 @@ func TestZipkin(t *testing.T) { t2 := trace.NewTracer("service2", report, true) sp1 := t1.New("option_1") sp2 := sp1.Fork("service3", "opt_client") - sp2.SetLog(trace.Log("log_k","log_v")) + sp2.SetLog(trace.Log("log_k", "log_v")) // inject header := make(http.Header) t1.Inject(sp2, trace.HTTPFormat, header) diff --git a/pkg/ratelimit/bbr/bbr.go b/pkg/ratelimit/bbr/bbr.go index 304ab648d..98a9d648e 100644 --- a/pkg/ratelimit/bbr/bbr.go +++ b/pkg/ratelimit/bbr/bbr.go @@ -32,7 +32,7 @@ func init() { go cpuproc() } -// cpu = cpuᵗ⁻¹ * decay + cpuᵗ * (1 - decay) +// cpu = cpuᵗ⁻¹ * decay + cpuᵗ * (1 - decay) func cpuproc() { ticker := time.NewTicker(time.Millisecond * 250) defer func() { diff --git a/pkg/stat/sys/cpu/cgroup.go b/pkg/stat/sys/cpu/cgroup.go index a3edd52f4..b297f611f 100644 --- a/pkg/stat/sys/cpu/cgroup.go +++ b/pkg/stat/sys/cpu/cgroup.go @@ -56,7 +56,10 @@ func (c *cgroup) CPUAcctUsagePerCPU() ([]uint64, error) { if u, err = parseUint(v); err != nil { return nil, err } - usage = append(usage, u) + // fix possible_cpu:https://www.ibm.com/support/knowledgecenter/en/linuxonibm/com.ibm.linux.z.lgdd/lgdd_r_posscpusparm.html + if u != 0 { + usage = append(usage, u) + } } return usage, nil } diff --git a/pkg/stat/sys/cpu/cgroupCPU.go b/pkg/stat/sys/cpu/cgroupCPU.go index dda467ee7..db1372ae9 100644 --- a/pkg/stat/sys/cpu/cgroupCPU.go +++ b/pkg/stat/sys/cpu/cgroupCPU.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/pkg/errors" + pscpu "github.com/shirou/gopsutil/cpu" ) type cgroupCPU struct { @@ -21,12 +22,17 @@ type cgroupCPU struct { } func newCgroupCPU() (cpu *cgroupCPU, err error) { - cpus, err := perCPUUsage() - if err != nil { - err = errors.Errorf("perCPUUsage() failed!err:=%v", err) - return + var cores int + cores, err = pscpu.Counts(true) + if err != nil || cores == 0 { + var cpus []uint64 + cpus, err = perCPUUsage() + if err != nil { + err = errors.Errorf("perCPUUsage() failed!err:=%v", err) + return + } + cores = len(cpus) } - cores := uint64(len(cpus)) sets, err := cpuSets() if err != nil { @@ -61,7 +67,7 @@ func newCgroupCPU() (cpu *cgroupCPU, err error) { cpu = &cgroupCPU{ frequency: maxFreq, quota: quota, - cores: cores, + cores: uint64(cores), preSystem: preSystem, preTotal: preTotal, } diff --git a/pkg/sync/pipeline/CHANGELOG.md b/pkg/sync/pipeline/CHANGELOG.md new file mode 100755 index 000000000..2120a2356 --- /dev/null +++ b/pkg/sync/pipeline/CHANGELOG.md @@ -0,0 +1,9 @@ +### pipeline + +#### Version 1.2.0 +> 1. 默认为平滑触发事件 +> 2. 增加metric上报 +#### Version 1.1.0 +> 1. 增加平滑时间的支持 +#### Version 1.0.0 +> 1. 提供聚合方法 内部区分压测流量 diff --git a/pkg/sync/pipeline/fanout/CHANGELOG.md b/pkg/sync/pipeline/fanout/CHANGELOG.md new file mode 100755 index 000000000..c289a6425 --- /dev/null +++ b/pkg/sync/pipeline/fanout/CHANGELOG.md @@ -0,0 +1,6 @@ +### pipeline/fanout + +#### Version 1.1.0 +> 1. 增加处理速度metric上报 +#### Version 1.0.0 +> 1. library/cache包改为fanout diff --git a/pkg/sync/pipeline/fanout/fanout.go b/pkg/sync/pipeline/fanout/fanout.go index 79ac0da8c..8636f4a7e 100644 --- a/pkg/sync/pipeline/fanout/fanout.go +++ b/pkg/sync/pipeline/fanout/fanout.go @@ -15,8 +15,8 @@ var ( // ErrFull chan full. ErrFull = errors.New("fanout: chan full") traceTags = []trace.Tag{ - trace.Tag{Key: trace.TagSpanKind, Value: "background"}, - trace.Tag{Key: trace.TagComponent, Value: "sync/pipeline/fanout"}, + {Key: trace.TagSpanKind, Value: "background"}, + {Key: trace.TagComponent, Value: "sync/pipeline/fanout"}, } ) @@ -67,7 +67,7 @@ type Fanout struct { // New new a fanout struct. func New(name string, opts ...Option) *Fanout { if name == "" { - name = "fanout" + name = "anonymous" } o := &options{ worker: 1, @@ -96,6 +96,7 @@ func (c *Fanout) proc() { case t := <-c.ch: wrapFunc(t.f)(t.ctx) _metricChanSize.Set(float64(len(c.ch)), c.name) + _metricCount.Inc(c.name) case <-c.ctx.Done(): return } diff --git a/pkg/sync/pipeline/fanout/metrics.go b/pkg/sync/pipeline/fanout/metrics.go index 0014b7f35..0f0d9fef5 100644 --- a/pkg/sync/pipeline/fanout/metrics.go +++ b/pkg/sync/pipeline/fanout/metrics.go @@ -1,15 +1,27 @@ package fanout -import "github.com/bilibili/kratos/pkg/stat/metric" +import ( + "github.com/bilibili/kratos/pkg/stat/metric" +) -const namespace = "sync" +const ( + _metricNamespace = "sync" + _metricSubSystem = "pipeline_fanout" +) var ( _metricChanSize = metric.NewGaugeVec(&metric.GaugeVecOpts{ - Namespace: namespace, - Subsystem: "pipeline_fanout", - Name: "current", + Namespace: _metricNamespace, + Subsystem: _metricSubSystem, + Name: "chan_len", Help: "sync pipeline fanout current channel size.", Labels: []string{"name"}, }) + _metricCount = metric.NewCounterVec(&metric.CounterVecOpts{ + Namespace: _metricNamespace, + Subsystem: _metricSubSystem, + Name: "process_count", + Help: "process count", + Labels: []string{"name"}, + }) ) diff --git a/pkg/sync/pipeline/pipeline.go b/pkg/sync/pipeline/pipeline.go index ef9263cb9..f1f923dde 100644 --- a/pkg/sync/pipeline/pipeline.go +++ b/pkg/sync/pipeline/pipeline.go @@ -3,16 +3,38 @@ package pipeline import ( "context" "errors" + "strconv" "sync" "time" "github.com/bilibili/kratos/pkg/net/metadata" + "github.com/bilibili/kratos/pkg/stat/metric" xtime "github.com/bilibili/kratos/pkg/time" ) // ErrFull channel full error var ErrFull = errors.New("channel full") +const _metricNamespace = "sync" +const _metricSubSystem = "pipeline" + +var ( + _metricCount = metric.NewCounterVec(&metric.CounterVecOpts{ + Namespace: _metricNamespace, + Subsystem: _metricSubSystem, + Name: "process_count", + Help: "process count", + Labels: []string{"name", "chan"}, + }) + _metricChanLen = metric.NewGaugeVec(&metric.GaugeVecOpts{ + Namespace: _metricNamespace, + Subsystem: _metricSubSystem, + Name: "chan_len", + Help: "channel length", + Labels: []string{"name", "chan"}, + }) +) + type message struct { key string value interface{} @@ -26,6 +48,7 @@ type Pipeline struct { mirrorChans []chan *message config *Config wait sync.WaitGroup + name string } // Config Pipeline config @@ -38,8 +61,8 @@ type Config struct { Buffer int // Worker channel number Worker int - // Smooth smoothing interval - Smooth bool + // Name use for metrics + Name string } func (c *Config) fix() { @@ -55,6 +78,9 @@ func (c *Config) fix() { if c.Worker <= 0 { c.Worker = 10 } + if c.Name == "" { + c.Name = "anonymous" + } } // NewPipeline new pipline @@ -67,6 +93,7 @@ func NewPipeline(config *Config) (res *Pipeline) { chans: make([]chan *message, config.Worker), mirrorChans: make([]chan *message, config.Worker), config: config, + name: config.Name, } for i := 0; i < config.Worker; i++ { res.chans[i] = make(chan *message, config.Buffer) @@ -144,7 +171,7 @@ func (p *Pipeline) mergeproc(mirror bool, index int, ch <-chan *message) { inteval = p.config.Interval oldTicker = true ) - if p.config.Smooth && index > 0 { + if index > 0 { inteval = xtime.Duration(int64(index) * (int64(p.config.Interval) / int64(p.config.Worker))) } ticker := time.NewTicker(time.Duration(inteval)) @@ -162,21 +189,26 @@ func (p *Pipeline) mergeproc(mirror bool, index int, ch <-chan *message) { } continue case <-ticker.C: - if p.config.Smooth && oldTicker { + if oldTicker { ticker.Stop() ticker = time.NewTicker(time.Duration(p.config.Interval)) oldTicker = false } } + name := p.name + process := count if len(vals) > 0 { ctx := context.Background() if mirror { ctx = metadata.NewContext(ctx, metadata.MD{metadata.Mirror: "1"}) + name = "mirror_" + name } p.Do(ctx, index, vals) vals = make(map[string][]interface{}, p.config.MaxSize) count = 0 } + _metricChanLen.Set(float64(len(ch)), name, strconv.Itoa(index)) + _metricCount.Add(float64(process), name, strconv.Itoa(index)) if closed { ticker.Stop() return diff --git a/pkg/sync/pipeline/pipeline_test.go b/pkg/sync/pipeline/pipeline_test.go index dcfdd8902..79aa27762 100644 --- a/pkg/sync/pipeline/pipeline_test.go +++ b/pkg/sync/pipeline/pipeline_test.go @@ -92,7 +92,6 @@ func TestPipelineSmooth(t *testing.T) { Interval: xtime.Duration(time.Second), Buffer: 100, Worker: 10, - Smooth: true, } type result struct { index int diff --git a/pkg/testing/lich/README.md b/pkg/testing/lich/README.md new file mode 100644 index 000000000..180224fb5 --- /dev/null +++ b/pkg/testing/lich/README.md @@ -0,0 +1,4 @@ +## testing/lich 运行环境构建 +基于 docker-compose 实现跨平台跨语言环境的容器依赖管理方案，以解决运行ut场景下的 (mysql, redis, mc)容器依赖问题。 + +使用说明参见：https://github.com/bilibili/kratos/tree/master/tool/testcli/README.md \ No newline at end of file diff --git a/pkg/testing/lich/composer.go b/pkg/testing/lich/composer.go new file mode 100644 index 000000000..5a7078d5f --- /dev/null +++ b/pkg/testing/lich/composer.go @@ -0,0 +1,125 @@ +package lich + +import ( + "bytes" + "crypto/md5" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/bilibili/kratos/pkg/log" +) + +var ( + retry int + noDown bool + yamlPath string + pathHash string + services map[string]*Container +) + +func init() { + flag.StringVar(&yamlPath, "f", "docker-compose.yaml", "composer yaml path.") + flag.BoolVar(&noDown, "nodown", false, "containers are not recycled.") +} + +func runCompose(args ...string) (output []byte, err error) { + if _, err = os.Stat(yamlPath); os.IsNotExist(err) { + log.Error("os.Stat(%s) composer yaml is not exist!", yamlPath) + return + } + if yamlPath, err = filepath.Abs(yamlPath); err != nil { + log.Error("filepath.Abs(%s) error(%v)", yamlPath, err) + return + } + pathHash = fmt.Sprintf("%x", md5.Sum([]byte(yamlPath)))[:9] + args = append([]string{"-f", yamlPath, "-p", pathHash}, args...) + if output, err = exec.Command("docker-compose", args...).CombinedOutput(); err != nil { + log.Error("exec.Command(docker-compose) args(%v) stdout(%s) error(%v)", args, string(output), err) + return + } + return +} + +// Setup setup UT related environment dependence for everything. +func Setup() (err error) { + if _, err = runCompose("up", "-d"); err != nil { + return + } + defer func() { + if err != nil { + Teardown() + } + }() + if _, err = getServices(); err != nil { + return + } + _, err = checkServices() + return +} + +// Teardown unsetup all environment dependence. +func Teardown() (err error) { + if !noDown { + _, err = runCompose("down") + } + return +} + +func getServices() (output []byte, err error) { + if output, err = runCompose("config", "--services"); err != nil { + return + } + services = make(map[string]*Container) + output = bytes.TrimSpace(output) + for _, svr := range bytes.Split(output, []byte("\n")) { + if output, err = runCompose("ps", "-a", "-q", string(svr)); err != nil { + return + } + var ( + id = string(bytes.TrimSpace(output)) + args = []string{"inspect", id, "--format", "'{{json .}}'"} + ) + if output, err = exec.Command("docker", args...).CombinedOutput(); err != nil { + log.Error("exec.Command(docker) args(%v) stdout(%s) error(%v)", args, string(output), err) + return + } + if output = bytes.TrimSpace(output); bytes.Equal(output, []byte("")) { + err = fmt.Errorf("service: %s | container: %s fails to launch", svr, id) + log.Error("exec.Command(docker) args(%v) error(%v)", args, err) + return + } + var c = &Container{} + if err = json.Unmarshal(bytes.Trim(output, "'"), c); err != nil { + log.Error("json.Unmarshal(%s) error(%v)", string(output), err) + return + } + services[string(svr)] = c + } + return +} + +func checkServices() (output []byte, err error) { + defer func() { + if err != nil && retry < 4 { + retry++ + getServices() + time.Sleep(time.Second * 5) + output, err = checkServices() + return + } + retry = 0 + }() + for svr, c := range services { + if err = c.Healthcheck(); err != nil { + log.Error("healthcheck(%s) error(%v) retrying %d times...", svr, err, 5-retry) + return + } + // TODO About container check and more... + } + return +} diff --git a/pkg/testing/lich/healthcheck.go b/pkg/testing/lich/healthcheck.go new file mode 100644 index 000000000..9e5fa1958 --- /dev/null +++ b/pkg/testing/lich/healthcheck.go @@ -0,0 +1,85 @@ +package lich + +import ( + "database/sql" + "fmt" + "net" + "strconv" + "strings" + + "github.com/bilibili/kratos/pkg/log" + // Register go-sql-driver stuff + _ "github.com/go-sql-driver/mysql" +) + +var healthchecks = map[string]func(*Container) error{"mysql": checkMysql, "mariadb": checkMysql} + +// Healthcheck check container health. +func (c *Container) Healthcheck() (err error) { + if status, health := c.State.Status, c.State.Health.Status; !c.State.Running || (health != "" && health != "healthy") { + err = fmt.Errorf("service: %s | container: %s not running", c.GetImage(), c.GetID()) + log.Error("docker status(%s) health(%s) error(%v)", status, health, err) + return + } + if check, ok := healthchecks[c.GetImage()]; ok { + err = check(c) + return + } + for proto, ports := range c.NetworkSettings.Ports { + if id := c.GetID(); !strings.Contains(proto, "tcp") { + log.Error("container: %s proto(%s) unsupported.", id, proto) + continue + } + for _, publish := range ports { + var ( + ip = net.ParseIP(publish.HostIP) + port, _ = strconv.Atoi(publish.HostPort) + tcpAddr = &net.TCPAddr{IP: ip, Port: port} + tcpConn *net.TCPConn + ) + if tcpConn, err = net.DialTCP("tcp", nil, tcpAddr); err != nil { + log.Error("net.DialTCP(%s:%s) error(%v)", publish.HostIP, publish.HostPort, err) + return + } + tcpConn.Close() + } + } + return +} + +func checkMysql(c *Container) (err error) { + var ip, port, user, passwd string + for _, env := range c.Config.Env { + splits := strings.Split(env, "=") + if strings.Contains(splits[0], "MYSQL_ROOT_PASSWORD") { + user, passwd = "root", splits[1] + continue + } + if strings.Contains(splits[0], "MYSQL_ALLOW_EMPTY_PASSWORD") { + user, passwd = "root", "" + continue + } + if strings.Contains(splits[0], "MYSQL_USER") { + user = splits[1] + continue + } + if strings.Contains(splits[0], "MYSQL_PASSWORD") { + passwd = splits[1] + continue + } + } + var db *sql.DB + if ports, ok := c.NetworkSettings.Ports["3306/tcp"]; ok { + ip, port = ports[0].HostIP, ports[0].HostPort + } + var dsn = fmt.Sprintf("%s:%s@tcp(%s:%s)/", user, passwd, ip, port) + if db, err = sql.Open("mysql", dsn); err != nil { + log.Error("sql.Open(mysql) dsn(%s) error(%v)", dsn, err) + return + } + if err = db.Ping(); err != nil { + log.Error("ping(db) dsn(%s) error(%v)", dsn, err) + } + defer db.Close() + return +} diff --git a/pkg/testing/lich/model.go b/pkg/testing/lich/model.go new file mode 100644 index 000000000..64653a681 --- /dev/null +++ b/pkg/testing/lich/model.go @@ -0,0 +1,88 @@ +package lich + +import ( + "strings" + "time" +) + +// Container docker inspect resp. +type Container struct { + ID string `json:"Id"` + Created time.Time `json:"Created"` + Path string `json:"Path"` + Args []string `json:"Args"` + State struct { + Status string `json:"Status"` + Running bool `json:"Running"` + Paused bool `json:"Paused"` + Restarting bool `json:"Restarting"` + OOMKilled bool `json:"OOMKilled"` + Dead bool `json:"Dead"` + Pid int `json:"Pid"` + ExitCode int `json:"ExitCode"` + Error string `json:"Error"` + StartedAt time.Time `json:"StartedAt"` + FinishedAt time.Time `json:"FinishedAt"` + Health struct { + Status string `json:"Status"` + FailingStreak int `json:"FailingStreak"` + Log []struct { + Start time.Time `json:"Start"` + End time.Time `json:"End"` + ExitCode int `json:"ExitCode"` + Output string `json:"Output"` + } `json:"Log"` + } `json:"Health"` + } `json:"State"` + Config struct { + Hostname string `json:"Hostname"` + Domainname string `json:"Domainname"` + User string `json:"User"` + Tty bool `json:"Tty"` + OpenStdin bool `json:"OpenStdin"` + StdinOnce bool `json:"StdinOnce"` + Env []string `json:"Env"` + Cmd []string `json:"Cmd"` + Image string `json:"Image"` + WorkingDir string `json:"WorkingDir"` + Entrypoint []string `json:"Entrypoint"` + } `json:"Config"` + Image string `json:"Image"` + ResolvConfPath string `json:"ResolvConfPath"` + HostnamePath string `json:"HostnamePath"` + HostsPath string `json:"HostsPath"` + LogPath string `json:"LogPath"` + Name string `json:"Name"` + RestartCount int `json:"RestartCount"` + Driver string `json:"Driver"` + Platform string `json:"Platform"` + MountLabel string `json:"MountLabel"` + ProcessLabel string `json:"ProcessLabel"` + AppArmorProfile string `json:"AppArmorProfile"` + NetworkSettings struct { + Bridge string `json:"Bridge"` + SandboxID string `json:"SandboxID"` + HairpinMode bool `json:"HairpinMode"` + Ports map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + } `json:"Ports"` + } `json:"NetworkSettings"` +} + +// GetImage get image name at container +func (c *Container) GetImage() (image string) { + image = c.Config.Image + if images := strings.Split(image, ":"); len(images) > 0 { + image = images[0] + } + return +} + +// GetID get id at container +func (c *Container) GetID() (id string) { + if id = c.ID; len(id) > 9 { + id = id[0:9] + } + return +} diff --git a/tool/kratos-protoc/bm.go b/tool/kratos-protoc/bm.go index 2c6b4e13f..5f88e00dd 100644 --- a/tool/kratos-protoc/bm.go +++ b/tool/kratos-protoc/bm.go @@ -2,8 +2,6 @@ package main import ( "os/exec" - - "github.com/urfave/cli" ) const ( @@ -20,6 +18,6 @@ func installBMGen() error { return nil } -func genBM(ctx *cli.Context) error { - return generate(ctx, _bmProtoc) -} +func genBM(files []string) error { + return generate(_bmProtoc, files) +} \ No newline at end of file diff --git a/tool/kratos-protoc/ecode.go b/tool/kratos-protoc/ecode.go index 22966e121..48da9ca9f 100644 --- a/tool/kratos-protoc/ecode.go +++ b/tool/kratos-protoc/ecode.go @@ -2,8 +2,6 @@ package main import ( "os/exec" - - "github.com/urfave/cli" ) const ( @@ -20,6 +18,6 @@ func installEcodeGen() error { return nil } -func genEcode(ctx *cli.Context) error { - return generate(ctx, _ecodeProtoc) +func genEcode(files []string) error { + return generate(_ecodeProtoc, files) } diff --git a/tool/kratos-protoc/grpc.go b/tool/kratos-protoc/grpc.go index b21da8dad..760f29574 100644 --- a/tool/kratos-protoc/grpc.go +++ b/tool/kratos-protoc/grpc.go @@ -2,8 +2,6 @@ package main import ( "os/exec" - - "github.com/urfave/cli" ) const ( @@ -20,6 +18,6 @@ func installGRPCGen() error { return nil } -func genGRPC(ctx *cli.Context) error { - return generate(ctx, _grpcProtoc) +func genGRPC(files []string) error { + return generate(_grpcProtoc, files) } diff --git a/tool/kratos-protoc/protoc.go b/tool/kratos-protoc/protoc.go index af96ad91b..ed2ba5f5c 100644 --- a/tool/kratos-protoc/protoc.go +++ b/tool/kratos-protoc/protoc.go @@ -27,6 +27,10 @@ func protocAction(ctx *cli.Context) (err error) { if err = checkProtoc(); err != nil { return err } + files := []string(ctx.Args()) + if len(files) == 0 { + files, _ = filepath.Glob("*.proto") + } if !withGRPC && !withBM && !withSwagger && !withEcode { withBM = true withGRPC = true @@ -37,7 +41,7 @@ func protocAction(ctx *cli.Context) (err error) { if err = installBMGen(); err != nil { return } - if err = genBM(ctx); err != nil { + if err = genBM(files); err != nil { return } } @@ -45,7 +49,7 @@ func protocAction(ctx *cli.Context) (err error) { if err = installGRPCGen(); err != nil { return err } - if err = genGRPC(ctx); err != nil { + if err = genGRPC(files); err != nil { return } } @@ -53,7 +57,7 @@ func protocAction(ctx *cli.Context) (err error) { if err = installSwaggerGen(); err != nil { return } - if err = genSwagger(ctx); err != nil { + if err = genSwagger(files); err != nil { return } } @@ -61,11 +65,11 @@ func protocAction(ctx *cli.Context) (err error) { if err = installEcodeGen(); err != nil { return } - if err = genEcode(ctx); err != nil { + if err = genEcode(files); err != nil { return } } - log.Printf("generate %v success.\n", ctx.Args()) + log.Printf("generate %s success.\n", strings.Join(files, " ")) return nil } @@ -95,7 +99,7 @@ func checkProtoc() error { return nil } -func generate(ctx *cli.Context, protoc string) error { +func generate(protoc string, files []string) error { pwd, _ := os.Getwd() gosrc := path.Join(gopath(), "src") ext, err := latestKratos() @@ -103,9 +107,9 @@ func generate(ctx *cli.Context, protoc string) error { return err } line := fmt.Sprintf(protoc, gosrc, ext, pwd) - log.Println(line, strings.Join(ctx.Args(), " ")) + log.Println(line, strings.Join(files, " ")) args := strings.Split(line, " ") - args = append(args, ctx.Args()...) + args = append(args, files...) cmd := exec.Command(args[0], args[1:]...) cmd.Dir = pwd cmd.Env = os.Environ() @@ -144,8 +148,9 @@ func latestKratos() (string, error) { } func gopath() (gp string) { - gopaths := strings.Split(os.Getenv("GOPATH"), ":") - if len(gopaths) == 1 { + gopaths := strings.Split(os.Getenv("GOPATH"), string(filepath.ListSeparator)) + + if len(gopaths) == 1 && gopaths[0] != "" { return gopaths[0] } pwd, err := os.Getwd() @@ -157,6 +162,9 @@ func gopath() (gp string) { return } for _, gopath := range gopaths { + if gopath == "" { + continue + } absgp, err := filepath.Abs(gopath) if err != nil { return diff --git a/tool/kratos-protoc/swagger.go b/tool/kratos-protoc/swagger.go index 7d1b61aaf..be89670a7 100644 --- a/tool/kratos-protoc/swagger.go +++ b/tool/kratos-protoc/swagger.go @@ -2,8 +2,6 @@ package main import ( "os/exec" - - "github.com/urfave/cli" ) const ( @@ -20,6 +18,6 @@ func installSwaggerGen() error { return nil } -func genSwagger(ctx *cli.Context) error { - return generate(ctx, _swaggerProtoc) +func genSwagger(files []string) error { + return generate(_swaggerProtoc, files) } diff --git a/tool/kratos/new.go b/tool/kratos/new.go index 8f9325f5d..2b2b08ad7 100644 --- a/tool/kratos/new.go +++ b/tool/kratos/new.go @@ -5,13 +5,14 @@ import ( "fmt" "os" "path" + "path/filepath" "github.com/urfave/cli" ) func runNew(ctx *cli.Context) error { if len(ctx.Args()) == 0 { - return errors.New("required project name") + return errors.New("project name required, please enter a project name you want to create") } p.Name = ctx.Args()[0] @@ -25,7 +26,8 @@ func runNew(ctx *cli.Context) error { pwd, _ := os.Getwd() p.Path = path.Join(pwd, p.Name) } - // creata a project + p.Path = filepath.FromSlash(p.Path) + // Create a project if err := create(); err != nil { return err } diff --git a/tool/kratos/template.go b/tool/kratos/template.go index 06a2c8b83..683677a2a 100644 --- a/tool/kratos/template.go +++ b/tool/kratos/template.go @@ -76,6 +76,8 @@ import ( "{{.ModuleName}}/internal/service" "github.com/bilibili/kratos/pkg/conf/paladin" "github.com/bilibili/kratos/pkg/log" + + _ "go.uber.org/automaxprocs" ) func main() { @@ -127,6 +129,8 @@ import ( "{{.ModuleName}}/internal/service" "github.com/bilibili/kratos/pkg/conf/paladin" "github.com/bilibili/kratos/pkg/log" + + _ "go.uber.org/automaxprocs" ) func main() { @@ -546,7 +550,7 @@ type Kratos struct { go 1.12 require ( - github.com/bilibili/kratos v0.2.2 + github.com/bilibili/kratos master github.com/gogo/protobuf v1.2.1 github.com/golang/protobuf v1.3.2 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 diff --git a/tool/kratos/tool.go b/tool/kratos/tool.go index 45769a265..cc7f0c258 100644 --- a/tool/kratos/tool.go +++ b/tool/kratos/tool.go @@ -186,8 +186,9 @@ func (t Tool) installed() bool { } func gopath() (gp string) { - gopaths := strings.Split(os.Getenv("GOPATH"), ":") - if len(gopaths) == 1 { + gopaths := strings.Split(os.Getenv("GOPATH"), string(filepath.ListSeparator)) + + if len(gopaths) == 1 && gopaths[0] != "" { return gopaths[0] } pwd, err := os.Getwd() @@ -199,6 +200,9 @@ func gopath() (gp string) { return } for _, gopath := range gopaths { + if gopath == "" { + continue + } absgp, err := filepath.Abs(gopath) if err != nil { return diff --git a/tool/kratos/version.go b/tool/kratos/version.go index 1aac093c3..57664d625 100644 --- a/tool/kratos/version.go +++ b/tool/kratos/version.go @@ -8,9 +8,9 @@ import ( var ( // Version is version - Version = "0.2.2" + Version = "0.2.3" // BuildTime is BuildTime - BuildTime = "2019/07/24" + BuildTime = "2019/10/08" ) // VersionOptions include version diff --git a/tool/protobuf/pkg/extensions/gogoproto/Makefile b/tool/protobuf/pkg/extensions/gogoproto/Makefile deleted file mode 100644 index 2d6cd0d07..000000000 --- a/tool/protobuf/pkg/extensions/gogoproto/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -regenerate: - go install go-common/vendor/github.com/golang/protobuf/protoc-gen-go - protoc --go_out=paths=source_relative:. gogo.proto - -restore: - cp gogo.pb.golden gogo.pb.go - -preserve: - cp gogo.pb.go gogo.pb.golden diff --git a/tool/protobuf/pkg/gen/main.go b/tool/protobuf/pkg/gen/main.go index e68f6a212..071ba2ae2 100644 --- a/tool/protobuf/pkg/gen/main.go +++ b/tool/protobuf/pkg/gen/main.go @@ -3,9 +3,9 @@ package gen import ( "io" "io/ioutil" + "log" "os" "strings" - "log" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/protoc-gen-go/descriptor" @@ -70,7 +70,6 @@ func writeResponse(w io.Writer, resp *plugin.CodeGeneratorResponse) { } } - // Fail log and exit func Fail(msgs ...string) { s := strings.Join(msgs, " ") @@ -85,10 +84,9 @@ func Info(msgs ...string) { os.Exit(1) } - // Error log and exit func Error(err error, msgs ...string) { s := strings.Join(msgs, " ") + ":" + err.Error() log.Print("error:", s) os.Exit(1) -} \ No newline at end of file +} diff --git a/tool/protobuf/pkg/utils/stringutils.go b/tool/protobuf/pkg/utils/stringutils.go index d36704c6b..8da21c8fe 100644 --- a/tool/protobuf/pkg/utils/stringutils.go +++ b/tool/protobuf/pkg/utils/stringutils.go @@ -1,4 +1,3 @@ - package utils import ( @@ -95,4 +94,4 @@ func AlphaDigitize(r rune) rune { // letters, numbers, and underscore. func CleanIdentifier(s string) string { return strings.Map(AlphaDigitize, s) -} \ No newline at end of file +} diff --git a/tool/protobuf/protoc-gen-bm/generator/generator.go b/tool/protobuf/protoc-gen-bm/generator/generator.go index ca9a06b50..19c325bfe 100644 --- a/tool/protobuf/protoc-gen-bm/generator/generator.go +++ b/tool/protobuf/protoc-gen-bm/generator/generator.go @@ -125,18 +125,7 @@ func (t *bm) generateImports(file *descriptor.FileDescriptorProto) { deps := make(map[string]string) // Map of package name to quoted import path. deps = t.DeduceDeps(file) for pkg, importPath := range deps { - for _, service := range file.Service { - for _, method := range service.Method { - inputType := t.GoTypeName(method.GetInputType()) - outputType := t.GoTypeName(method.GetOutputType()) - if strings.HasPrefix(pkg, outputType) || strings.HasPrefix(pkg, inputType) { - t.P(`import `, pkg, ` `, importPath) - } - } - } - } - if len(deps) > 0 { - t.P() + t.P(`import `, pkg, ` `, importPath) } t.P() t.P(`// to suppressed 'imported but not used warning'`) diff --git a/tool/testcli/README.MD b/tool/testcli/README.MD new file mode 100644 index 000000000..f17e3d281 --- /dev/null +++ b/tool/testcli/README.MD @@ -0,0 +1,154 @@ +## testcli UT运行环境构建工具 +基于 docker-compose 实现跨平台跨语言环境的容器依赖管理方案，以解决运行ut场景下的 (mysql, redis, mc)容器依赖问题。 + +*这个是testing/lich的二进制工具版本（Go请直接使用库版本：github.com/bilibili/kratos/pkg/testing/lich)* + +### 功能和特性 +- 自动读取 test 目录下的 yaml 并启动依赖 +- 自动导入 test 目录下的 DB 初始化 SQL +- 提供特定容器内的 healthcheck (mysql, mc, redis) +- 提供一站式解决 UT 服务依赖的工具版本 (testcli) + +### 编译安装 +*使用本工具/库需要前置安装好 docker & docker-compose@v1.24.1^* + +#### Method 1. With go get +```shell +go get -u github.com/bilibili/kratos/tool/testcli +$GOPATH/bin/testcli -h +``` +#### Method 2. Build with Go +```shell +cd github.com/bilibili/kratos/tool/testcli +go build -o $GOPATH/bin/testcli +$GOPATH/bin/testcli -h +``` +#### Method 3. Import with Kratos pkg +```Go +import "github.com/bilibili/kratos/pkg/testing/lich" +``` + +### 构建数据 +#### Step 1. create docker-compose.yml +创建依赖服务的 docker-compose.yml，并把它放在项目路径下的 test 文件夹下面。例如： +```shell +mkdir -p $YOUR_PROJECT/test +``` +```yaml +version: "3.7" + +services: + db: + image: mysql:5.6 + ports: + - 3306:3306 + environment: + - MYSQL_ROOT_PASSWORD=root + volumes: + - .:/docker-entrypoint-initdb.d + command: [ + '--character-set-server=utf8', + '--collation-server=utf8_unicode_ci' + ] + + redis: + image: redis + ports: + - 6379:6379 +``` +一般来讲，我们推荐在项目根目录创建 test 目录，里面存放描述服务的yml，以及需要初始化的数据（database.sql等）。 + +同时也需要注意，正确的对容器内服务进行健康检测，testcli会在容器的health状态执行UT，其实我们也内置了针对几个较为通用镜像（mysql mariadb mc redis）的健康检测，也就是不写也没事(^^;; + +#### Step 2. export database.sql +构造初始化的数据（database.sql等），当然也把它也在 test 文件夹里。 +```sql +CREATE DATABASE IF NOT EXISTS `YOUR_DATABASE_NAME`; + +SET NAMES 'utf8'; +USE `YOUR_DATABASE_NAME`; + +CREATE TABLE IF NOT EXISTS `YOUR_TABLE_NAME` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键'， + PRIMARY KEY (`id`), +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='YOUR_TABLE_NAME'; +``` +这里需要注意，在创建库/表的时候尽量加上 IF NOT EXISTS，以给予一定程度的容错，以及 SET NAMES 'utf8'; 用于解决客户端连接乱码问题。 + +#### Step 3. change your project mysql config +```toml +[mysql] + addr = "127.0.0.1:3306" + dsn = "root:root@tcp(127.0.0.1:3306)/YOUR_DATABASE?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8" + active = 20 + idle = 10 + idleTimeout ="1s" + queryTimeout = "1s" + execTimeout = "1s" + tranTimeout = "1s" +``` +在 *Step 1* 我们已经指定了服务对外暴露的端口为3306（这当然也可以是你指定的任何值），那理所应当的我们也要修改项目连接数据库的配置～ + +Great! 至此你已经完成了运行所需要用到的数据配置，接下来就来运行它。 + +### 运行 +开头也说过本工具支持两种运行方式：testcli 二进制工具版本和 go package 源码包，业务方可以根据需求场景进行选择。 +#### Method 1. With testcli tool +*已支持的 flag： -f，--nodown，down，run* +- -f，指定 docker-compose.yaml 文件路径，默认为当前目录下。 +- --nodown，指定是否在UT执行完成后保留容器，以供下次复用。 +- down，teardown 销毁当前项目下这个 compose 文件产生的容器。 +- run，运行你当前语言的单测执行命令（如：golang为 go test -v ./） + +example: +```shell +testcli -f ../../test/docker-compose.yaml run go test -v ./ +``` +#### Method 2. Import with Kratos pkg +- Step1. 在 Dao|Service 层中的 TestMain 单测主入口中，import "github.com/bilibili/kratos/pkg/testing/lich" 引入testcli工具的go库版本。 +- Step2. 使用 flag.Set("f", "../../test/docker-compose.yaml") 指定 docker-compose.yaml 文件的路径。 +- Step3. 在 flag.Parse() 后即可使用 lich.Setup() 安装依赖&初始化数据（注意测试用例执行结束后 lich.Teardown() 回收下～） +- Step4. 运行 `go test -v ./ `看看效果吧～ + +example: +```Go +package dao + + +import ( + "flag" + "os" + "strings" + "testing" + + "github.com/bilibili/kratos/pkg/conf/paladin" + "github.com/bilibili/kratos/pkg/testing/lich" + ) + +var ( + d *Dao +) + +func TestMain(m *testing.M) { + flag.Set("conf", "../../configs") + flag.Set("f", "../../test/docker-compose.yaml") + flag.Parse() + if err := paladin.Init(); err != nil { + panic(err) + } + if err := lich.Setup(); err != nil { + panic(err) + } + defer lich.Teardown() + d = New() + if code := m.Run(); code != 0 { + panic(code) + } +} + ``` +## 注意 +因为启动mysql容器较为缓慢，健康检测的机制会重试3次，每次暂留5秒钟，基本在10s内mysql就能从creating到服务正常启动！ + +当然你也可以在使用 testcli 时加上 --nodown，使其不用每次跑都新建容器，只在第一次跑的时候会初始化容器，后面都进行复用，这样速度会快很多。 + +成功启动后就欢乐奔放的玩耍吧～ Good Lucky! \ No newline at end of file diff --git a/tool/testcli/docker-compose.yaml b/tool/testcli/docker-compose.yaml new file mode 100644 index 000000000..57f78dac8 --- /dev/null +++ b/tool/testcli/docker-compose.yaml @@ -0,0 +1,26 @@ +version: "3.7" + +services: + db: + image: mysql:5.6 + ports: + - 3306:3306 + environment: + - MYSQL_ROOT_PASSWORD=root + - TZ=Asia/Shanghai + volumes: + - .:/docker-entrypoint-initdb.d + command: [ + '--character-set-server=utf8', + '--collation-server=utf8_unicode_ci' + ] + + redis: + image: redis + ports: + - 6379:6379 + + memcached: + image: memcached + ports: + - 11211:11211 \ No newline at end of file diff --git a/tool/testcli/main.go b/tool/testcli/main.go new file mode 100644 index 000000000..563a9d132 --- /dev/null +++ b/tool/testcli/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "flag" + "os" + "os/exec" + "strings" + + "github.com/bilibili/kratos/pkg/testing/lich" +) + +func parseArgs() (flags map[string]string) { + flags = make(map[string]string) + for idx, arg := range os.Args { + if idx == 0 { + continue + } + if arg == "down" { + flags["down"] = "" + return + } + if cmds := os.Args[idx+1:]; arg == "run" { + flags["run"] = strings.Join(cmds, " ") + return + } + } + return +} + +func main() { + flag.Parse() + flags := parseArgs() + if _, ok := flags["down"]; ok { + lich.Teardown() + return + } + if cmd, ok := flags["run"]; !ok || cmd == "" { + panic("Your need 'run' flag assign to be run commands.") + } + if err := lich.Setup(); err != nil { + panic(err) + } + defer lich.Teardown() + cmds := strings.Split(flags["run"], " ") + cmd := exec.Command(cmds[0], cmds[1:]...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + panic(err) + } +} diff --git a/tool/testgen/README.md b/tool/testgen/README.md new file mode 100644 index 000000000..3ec686e7f --- /dev/null +++ b/tool/testgen/README.md @@ -0,0 +1,52 @@ +## testgen UT代码自动生成器 +解放你的双手，让你的UT一步到位！ + +### 功能和特性 +- 支持生成 Dao|Service 层UT代码功能（每个方法包含一个正向用例） +- 支持生成 Dao|Service 层测试入口文件dao_test.go, service_test.go（用于控制初始化，控制测试流程等） +- 支持生成Mock代码（使用GoMock框架） +- 支持选择不同模式生成不同代码（使用"–m mode"指定） +- 生成单元测试代码时，同时支持传入目录或文件 +- 支持指定方法追加生成测试用例（使用"–func funcName"指定） + +### 编译安装 +#### Method 1. With go get +```shell +go get -u github.com/bilibili/kratos/tool/testgen +$GOPATH/bin/testgen -h +``` +#### Method 2. Build with Go +```shell +cd github.com/bilibili/kratos/tool/testgen +go build -o $GOPATH/bin/testgen +$GOPATH/bin/testgen -h +``` +### 运行 +#### 生成Dao/Service层单元UT +```shell +$GOPATH/bin/testgen YOUR_PROJECT/dao # default mode +$GOPATH/bin/testgen --m test path/to/your/pkg +$GOPATH/bin/testgen --func functionName path/to/your/pkg +``` + +#### 生成接口类型 +```shell +$GOPATH/bin/testgen --m interface YOUR_PROJECT/dao #当前仅支持传目录，如目录包含子目录也会做处理 +``` + +#### 生成Mock代码 + ```shell +$GOPATH/bin/testgen --m mock YOUR_PROJECT/dao #仅传入包路径即可 +``` + +#### 生成Monkey代码 +```shell +$GOPATH/bin/testgen --m monkey yourCodeDirPath #仅传入包路径即可 +``` +### 赋诗一首 +``` +莫生气 莫生气 +代码辣鸡非我意 +自己动手分田地 +谈笑风生活长命 +``` \ No newline at end of file diff --git a/tool/testgen/gen.go b/tool/testgen/gen.go new file mode 100644 index 000000000..b515fda29 --- /dev/null +++ b/tool/testgen/gen.go @@ -0,0 +1,419 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/otokaze/mock/mockgen" + "github.com/otokaze/mock/mockgen/model" +) + +func genTest(parses []*parse) (err error) { + for _, p := range parses { + switch { + case strings.HasSuffix(p.Path, "_mock.go") || + strings.HasSuffix(p.Path, ".intf.go"): + continue + case strings.HasSuffix(p.Path, "dao.go") || + strings.HasSuffix(p.Path, "service.go"): + err = p.genTestMain() + default: + err = p.genUTTest() + } + if err != nil { + break + } + } + return +} + +func (p *parse) genUTTest() (err error) { + var ( + buffer bytes.Buffer + impts = strings.Join([]string{ + `"context"`, + `"testing"`, + `. "github.com/smartystreets/goconvey/convey"`, + }, "\n\t") + content []byte + ) + filename := strings.Replace(p.Path, ".go", "_test.go", -1) + if _, err = os.Stat(filename); (_func == "" && err == nil) || + (err != nil && os.IsExist(err)) { + err = nil + return + } + for _, impt := range p.Imports { + impts += "\n\t\"" + impt.V + "\"" + } + if _func == "" { + buffer.WriteString(fmt.Sprintf(tpPackage, p.Package)) + buffer.WriteString(fmt.Sprintf(tpImport, impts)) + } + for _, parseFunc := range p.Funcs { + if _func != "" && _func != parseFunc.Name { + continue + } + var ( + methodK string + tpVars string + vars []string + val []string + notice = "Then " + reset string + ) + if method := ConvertMethod(p.Path); method != "" { + methodK = method + "." + } + tpTestFuncs := fmt.Sprintf(tpTestFunc, strings.Title(p.Package), parseFunc.Name, "", parseFunc.Name, "%s", "%s", "%s") + tpTestFuncBeCall := methodK + parseFunc.Name + "(%s)\n\t\t\tConvey(\"%s\", func() {" + if parseFunc.Result == nil { + tpTestFuncBeCall = fmt.Sprintf(tpTestFuncBeCall, "%s", "No return values") + tpTestFuncs = fmt.Sprintf(tpTestFuncs, "%s", tpTestFuncBeCall, "%s") + } + for k, res := range parseFunc.Result { + if res.K == "" { + res.K = fmt.Sprintf("p%d", k+1) + } + var so string + if res.V == "error" { + res.K = "err" + so = fmt.Sprintf("\tSo(%s, ShouldBeNil)", res.K) + notice += "err should be nil." + } else { + so = fmt.Sprintf("\tSo(%s, ShouldNotBeNil)", res.K) + val = append(val, res.K) + } + if len(parseFunc.Result) <= k+1 { + if len(val) != 0 { + notice += strings.Join(val, ",") + " should not be nil." + } + tpTestFuncBeCall = fmt.Sprintf(tpTestFuncBeCall, "%s", notice) + res.K += " := " + tpTestFuncBeCall + } else { + res.K += ", %s" + } + tpTestFuncs = fmt.Sprintf(tpTestFuncs, "%s", res.K+"\n\t\t\t%s", "%s") + tpTestFuncs = fmt.Sprintf(tpTestFuncs, "%s", "%s", so, "%s") + } + if parseFunc.Params == nil { + tpTestFuncs = fmt.Sprintf(tpTestFuncs, "%s", "", "%s") + } + for k, pType := range parseFunc.Params { + if pType.K == "" { + pType.K = fmt.Sprintf("a%d", k+1) + } + var ( + init string + params = pType.K + ) + switch { + case strings.HasPrefix(pType.V, "context"): + init = params + " = context.Background()" + case strings.HasPrefix(pType.V, "[]byte"): + init = params + " = " + pType.V + "(\"\")" + case strings.HasPrefix(pType.V, "[]"): + init = params + " = " + pType.V + "{}" + case strings.HasPrefix(pType.V, "int") || + strings.HasPrefix(pType.V, "uint") || + strings.HasPrefix(pType.V, "float") || + strings.HasPrefix(pType.V, "double"): + init = params + " = " + pType.V + "(0)" + case strings.HasPrefix(pType.V, "string"): + init = params + " = \"\"" + case strings.Contains(pType.V, "*xsql.Tx"): + init = params + ",_ = " + methodK + "BeginTran(c)" + reset += "\n\t" + params + ".Commit()" + case strings.HasPrefix(pType.V, "*"): + init = params + " = " + strings.Replace(pType.V, "*", "&", -1) + "{}" + case strings.Contains(pType.V, "chan"): + init = params + " = " + pType.V + case pType.V == "time.Time": + init = params + " = time.Now()" + case strings.Contains(pType.V, "chan"): + init = params + " = " + pType.V + default: + init = params + " " + pType.V + } + vars = append(vars, "\t\t"+init) + if len(parseFunc.Params) > k+1 { + params += ", %s" + } + tpTestFuncs = fmt.Sprintf(tpTestFuncs, "%s", params, "%s") + } + if len(vars) > 0 { + tpVars = fmt.Sprintf(tpVar, strings.Join(vars, "\n\t")) + } + tpTestFuncs = fmt.Sprintf(tpTestFuncs, tpVars, "%s") + if reset != "" { + tpTestResets := fmt.Sprintf(tpTestReset, reset) + tpTestFuncs = fmt.Sprintf(tpTestFuncs, tpTestResets) + } else { + tpTestFuncs = fmt.Sprintf(tpTestFuncs, "") + } + buffer.WriteString(tpTestFuncs) + } + var ( + file *os.File + flag = os.O_RDWR | os.O_CREATE | os.O_APPEND + ) + if file, err = os.OpenFile(filename, flag, 0644); err != nil { + return + } + if _func == "" { + content, _ = GoImport(filename, buffer.Bytes()) + } else { + content = buffer.Bytes() + } + if _, err = file.Write(content); err != nil { + return + } + if err = file.Close(); err != nil { + return + } + return +} + +func (p *parse) genTestMain() (err error) { + var ( + new bool + buffer bytes.Buffer + impts string + vars, mainFunc string + content []byte + instance, confFunc string + tomlPath = "**PUT PATH TO YOUR CONFIG FILES HERE**" + filename = strings.Replace(p.Path, ".go", "_test.go", -1) + ) + if p.Imports["paladin"] != nil { + new = true + } + // if _intfMode { + // imptsList = append(imptsList, `"github.com/golang/mock/gomock"`) + // for _, field := range p.Structs { + // var hit bool + // pkgName := strings.Split(field.V, ".")[0] + // interfaceName := strings.Split(field.V, ".")[1] + // if p.Imports[pkgName] != nil { + // if hit, err = checkInterfaceMock(strings.Split(field.V, ".")[1], p.Imports[pkgName].V); err != nil { + // return + // } + // } + // if hit { + // imptsList = append(imptsList, "mock"+p.Imports[pkgName].K+" \""+p.Imports[pkgName].V+"/mock\"") + // pkgName = "mock" + strings.Title(pkgName) + // interfaceName = "Mock" + interfaceName + // varsList = append(varsList, "mock"+strings.Title(field.K)+" *"+pkgName+"."+interfaceName) + // mockStmt += "\tmock" + strings.Title(field.K) + " = " + pkgName + ".New" + interfaceName + "(mockCtrl)\n" + // newStmt += "\t\t" + field.K + ":\tmock" + strings.Title(field.K) + ",\n" + // } else { + // pkgName = subString(field.V, "*", ".") + // if p.Imports[pkgName] != nil && pkgName != "conf" { + // imptsList = append(imptsList, p.Imports[pkgName].K+" \""+p.Imports[pkgName].V+"\"") + // } + // switch { + // case strings.HasPrefix(field.V, "*conf."): + // newStmt += "\t\t" + field.K + ":\tconf.Conf,\n" + // case strings.HasPrefix(field.V, "*"): + // newStmt += "\t\t" + field.K + ":\t" + strings.Replace(field.V, "*", "&", -1) + "{},\n" + // default: + // newStmt += "\t\t" + field.K + ":\t" + field.V + ",\n" + // } + // } + // } + // mockStmt = fmt.Sprintf(_tpTestServiceMainMockStmt, mockStmt) + // newStmt = fmt.Sprintf(_tpTestServiceMainNewStmt, newStmt) + // } + if instance = ConvertMethod(p.Path); instance == "s" { + vars = strings.Join([]string{"s *Service"}, "\n\t") + mainFunc = tpTestServiceMain + } else { + vars = strings.Join([]string{"d *Dao"}, "\n\t") + mainFunc = tpTestDaoMain + } + if new { + impts = strings.Join([]string{`"os"`, `"flag"`, `"testing"`, p.Imports["paladin"].V}, "\n\t") + confFunc = fmt.Sprintf(tpTestMainNew, instance+" = New()") + } else { + impts = strings.Join(append([]string{`"os"`, `"flag"`, `"testing"`}), "\n\t") + confFunc = fmt.Sprintf(tpTestMainOld, instance+" = New(conf.Conf)") + } + if _, err := os.Stat(filename); os.IsNotExist(err) { + buffer.WriteString(fmt.Sprintf(tpPackage, p.Package)) + buffer.WriteString(fmt.Sprintf(tpImport, impts)) + buffer.WriteString(fmt.Sprintf(tpVar, vars)) + buffer.WriteString(fmt.Sprintf(mainFunc, tomlPath, confFunc)) + content, _ = GoImport(filename, buffer.Bytes()) + ioutil.WriteFile(filename, content, 0644) + } + return +} + +func genInterface(parses []*parse) (err error) { + var ( + parse *parse + pkg = make(map[string]string) + ) + for _, parse = range parses { + if strings.Contains(parse.Path, ".intf.go") { + continue + } + dirPath := filepath.Dir(parse.Path) + for _, parseFunc := range parse.Funcs { + if (parseFunc.Method == nil) || + !(parseFunc.Name[0] >= 'A' && parseFunc.Name[0] <= 'Z') { + continue + } + var ( + params string + results string + ) + for k, param := range parseFunc.Params { + params += param.K + " " + param.P + param.V + if len(parseFunc.Params) > k+1 { + params += ", " + } + } + for k, res := range parseFunc.Result { + results += res.K + " " + res.P + res.V + if len(parseFunc.Result) > k+1 { + results += ", " + } + } + if len(results) != 0 { + results = "(" + results + ")" + } + pkg[dirPath] += "\t" + fmt.Sprintf(tpIntfcFunc, parseFunc.Name, params, results) + } + } + for k, v := range pkg { + var buffer bytes.Buffer + pathSplit := strings.Split(k, "/") + filename := k + "/" + pathSplit[len(pathSplit)-1] + ".intf.go" + if _, exist := os.Stat(filename); os.IsExist(exist) { + continue + } + buffer.WriteString(fmt.Sprintf(tpPackage, pathSplit[len(pathSplit)-1])) + buffer.WriteString(fmt.Sprintf(tpInterface, strings.Title(pathSplit[len(pathSplit)-1]), v)) + content, _ := GoImport(filename, buffer.Bytes()) + err = ioutil.WriteFile(filename, content, 0644) + } + return +} + +func genMock(files ...string) (err error) { + for _, file := range files { + var pkg *model.Package + if pkg, err = mockgen.ParseFile(file); err != nil { + return + } + if len(pkg.Interfaces) == 0 { + continue + } + var mockDir = pkg.SrcDir + "/mock" + if _, err = os.Stat(mockDir); os.IsNotExist(err) { + err = nil + os.Mkdir(mockDir, 0744) + } + var mockPath = mockDir + "/" + pkg.Name + "_mock.go" + if _, exist := os.Stat(mockPath); os.IsExist(exist) { + continue + } + var g = &mockgen.Generator{Filename: file} + if err = g.Generate(pkg, "mock", mockPath); err != nil { + return + } + if err = ioutil.WriteFile(mockPath, g.Output(), 0644); err != nil { + return + } + } + return +} + +func genMonkey(parses []*parse) (err error) { + var ( + pkg = make(map[string]string) + ) + for _, parse := range parses { + if strings.Contains(parse.Path, "monkey.go") || + strings.Contains(parse.Path, "/mock/") { + continue + } + var ( + path = strings.Split(filepath.Dir(parse.Path), "/") + pack = ConvertHump(path[len(path)-1]) + refer = path[len(path)-1] + mockVar, mockType, srcDir string + ) + for i := len(path) - 1; i > len(path)-4; i-- { + if path[i] == "dao" || path[i] == "service" { + srcDir = strings.Join(path[:i+1], "/") + break + } + pack = ConvertHump(path[i-1]) + pack + } + if mockVar = ConvertMethod(parse.Path); mockType == "d" { + mockType = "*" + refer + ".Dao" + } else { + mockType = "*" + refer + ".Service" + } + for _, parseFunc := range parse.Funcs { + if (parseFunc.Method == nil) || (parseFunc.Result == nil) || + !(parseFunc.Name[0] >= 'A' && parseFunc.Name[0] <= 'Z') { + continue + } + var ( + funcParams, funcResults, mockKey, mockValue, funcName string + ) + funcName = pack + parseFunc.Name + for k, param := range parseFunc.Params { + funcParams += "_ " + param.V + if len(parseFunc.Params) > k+1 { + funcParams += ", " + } + } + for k, res := range parseFunc.Result { + if res.K == "" { + if res.V == "error" { + res.K = "err" + } else { + res.K = fmt.Sprintf("p%d", k+1) + } + } + mockKey += res.K + mockValue += res.V + funcResults += res.K + " " + res.P + res.V + if len(parseFunc.Result) > k+1 { + mockKey += ", " + mockValue += ", " + funcResults += ", " + } + } + pkg[srcDir+"."+refer] += fmt.Sprintf(tpMonkeyFunc, funcName, funcName, mockVar, mockType, funcResults, mockVar, parseFunc.Name, mockType, funcParams, mockValue, mockKey) + } + } + for path, content := range pkg { + var ( + buffer bytes.Buffer + dir = strings.Split(path, ".") + mockDir = dir[0] + "/mock" + filename = mockDir + "/monkey_" + dir[1] + ".go" + ) + if _, err = os.Stat(mockDir); os.IsNotExist(err) { + err = nil + os.Mkdir(mockDir, 0744) + } + if _, err := os.Stat(filename); os.IsExist(err) { + continue + } + buffer.WriteString(fmt.Sprintf(tpPackage, "mock")) + buffer.WriteString(content) + content, _ := GoImport(filename, buffer.Bytes()) + ioutil.WriteFile(filename, content, 0644) + } + return +} diff --git a/tool/testgen/main.go b/tool/testgen/main.go new file mode 100644 index 000000000..4d94e6aec --- /dev/null +++ b/tool/testgen/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "flag" + "fmt" + "os" +) + +var ( + err error + _mode, _func string + files []string + parses []*parse +) + +func main() { + flag.StringVar(&_mode, "m", "test", "Generating code by Working mode. [test|interface|mock...]") + flag.StringVar(&_func, "func", "", "Generating code by function.") + flag.Parse() + if len(os.Args) == 1 { + println("Creater is a tool for generating code.\n\nUsage: creater [-m]") + flag.PrintDefaults() + return + } + if err = parseArgs(os.Args[1:], &files, 0); err != nil { + panic(err) + } + switch _mode { + case "monkey": + if parses, err = parseFile(files...); err != nil { + panic(err) + } + if err = genMonkey(parses); err != nil { + panic(err) + } + case "test": + if parses, err = parseFile(files...); err != nil { + panic(err) + } + if err = genTest(parses); err != nil { + panic(err) + } + case "interface": + if parses, err = parseFile(files...); err != nil { + panic(err) + } + if err = genInterface(parses); err != nil { + panic(err) + } + case "mock": + if err = genMock(files...); err != nil { + panic(err) + } + default: + } + fmt.Println(print) +} diff --git a/tool/testgen/parser.go b/tool/testgen/parser.go new file mode 100644 index 000000000..ec363fd04 --- /dev/null +++ b/tool/testgen/parser.go @@ -0,0 +1,193 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type param struct{ K, V, P string } + +type parse struct { + Path string + Package string + // Imports []string + Imports map[string]*param + // Structs []*param + // Interfaces []string + Funcs []*struct { + Name string + Method, Params, Result []*param + } +} + +func parseArgs(args []string, res *[]string, index int) (err error) { + if len(args) <= index { + return + } + if strings.HasPrefix(args[index], "-") { + index += 2 + parseArgs(args, res, index) + return + } + var f os.FileInfo + if f, err = os.Stat(args[index]); err != nil { + return + } + if f.IsDir() { + if !strings.HasSuffix(args[index], "/") { + args[index] += "/" + } + var fs []os.FileInfo + if fs, err = ioutil.ReadDir(args[index]); err != nil { + return + } + for _, f = range fs { + path, _ := filepath.Abs(args[index] + f.Name()) + args = append(args, path) + } + } else { + if strings.HasSuffix(args[index], ".go") && + !strings.HasSuffix(args[index], "_test.go") { + *res = append(*res, args[index]) + } + } + index++ + return parseArgs(args, res, index) +} + +func parseFile(files ...string) (parses []*parse, err error) { + for _, file := range files { + var ( + astFile *ast.File + fSet = token.NewFileSet() + parse = &parse{ + Imports: make(map[string]*param), + } + ) + if astFile, err = parser.ParseFile(fSet, file, nil, 0); err != nil { + return + } + if astFile.Name != nil { + parse.Path = file + parse.Package = astFile.Name.Name + } + for _, decl := range astFile.Decls { + switch decl.(type) { + case *ast.GenDecl: + if specs := decl.(*ast.GenDecl).Specs; len(specs) > 0 { + parse.Imports = parseImports(specs) + } + case *ast.FuncDecl: + var ( + dec = decl.(*ast.FuncDecl) + parseFunc = &struct { + Name string + Method, Params, Result []*param + }{Name: dec.Name.Name} + ) + if dec.Recv != nil { + parseFunc.Method = parserParams(dec.Recv.List) + } + if dec.Type.Params != nil { + parseFunc.Params = parserParams(dec.Type.Params.List) + } + if dec.Type.Results != nil { + parseFunc.Result = parserParams(dec.Type.Results.List) + } + parse.Funcs = append(parse.Funcs, parseFunc) + } + } + parses = append(parses, parse) + } + return +} + +func parserParams(fields []*ast.Field) (params []*param) { + for _, field := range fields { + p := ¶m{} + p.V = parseType(field.Type) + if field.Names == nil { + params = append(params, p) + } + for _, name := range field.Names { + sp := ¶m{} + sp.K = name.Name + sp.V = p.V + sp.P = p.P + params = append(params, sp) + } + } + return +} + +func parseType(expr ast.Expr) string { + switch expr.(type) { + case *ast.Ident: + return expr.(*ast.Ident).Name + case *ast.StarExpr: + return "*" + parseType(expr.(*ast.StarExpr).X) + case *ast.ArrayType: + return "[" + parseType(expr.(*ast.ArrayType).Len) + "]" + parseType(expr.(*ast.ArrayType).Elt) + case *ast.SelectorExpr: + return parseType(expr.(*ast.SelectorExpr).X) + "." + expr.(*ast.SelectorExpr).Sel.Name + case *ast.MapType: + return "map[" + parseType(expr.(*ast.MapType).Key) + "]" + parseType(expr.(*ast.MapType).Value) + case *ast.StructType: + return "struct{}" + case *ast.InterfaceType: + return "interface{}" + case *ast.FuncType: + var ( + pTemp string + rTemp string + ) + pTemp = parseFuncType(pTemp, expr.(*ast.FuncType).Params) + if expr.(*ast.FuncType).Results != nil { + rTemp = parseFuncType(rTemp, expr.(*ast.FuncType).Results) + return fmt.Sprintf("func(%s) (%s)", pTemp, rTemp) + } + return fmt.Sprintf("func(%s)", pTemp) + case *ast.ChanType: + return fmt.Sprintf("make(chan %s)", parseType(expr.(*ast.ChanType).Value)) + case *ast.Ellipsis: + return parseType(expr.(*ast.Ellipsis).Elt) + } + return "" +} + +func parseFuncType(temp string, data *ast.FieldList) string { + var params = parserParams(data.List) + for i, param := range params { + if i == 0 { + temp = param.K + " " + param.V + continue + } + t := param.K + " " + param.V + temp = fmt.Sprintf("%s, %s", temp, t) + } + return temp +} + +func parseImports(specs []ast.Spec) (params map[string]*param) { + params = make(map[string]*param) + for _, spec := range specs { + switch spec.(type) { + case *ast.ImportSpec: + p := ¶m{V: strings.Replace(spec.(*ast.ImportSpec).Path.Value, "\"", "", -1)} + if spec.(*ast.ImportSpec).Name != nil { + p.K = spec.(*ast.ImportSpec).Name.Name + params[p.K] = p + } else { + vs := strings.Split(p.V, "/") + params[vs[len(vs)-1]] = p + } + } + } + return +} diff --git a/tool/testgen/templete.go b/tool/testgen/templete.go new file mode 100644 index 000000000..bb42d5831 --- /dev/null +++ b/tool/testgen/templete.go @@ -0,0 +1,41 @@ +package main + +var ( + tpPackage = "package %s\n\n" + tpImport = "import (\n\t%s\n)\n\n" + tpVar = "var (\n\t%s\n)\n" + tpInterface = "type %sInterface interface {\n%s}\n" + tpIntfcFunc = "%s(%s) %s\n" + tpMonkeyFunc = "// Mock%s .\nfunc Mock%s(%s %s,%s) (guard *monkey.PatchGuard) {\n\treturn monkey.PatchInstanceMethod(reflect.TypeOf(%s), \"%s\", func(_ %s, %s) (%s) {\n\t\treturn %s\n\t})\n}\n\n" + tpTestReset = "\n\t\tReset(func() {%s\n\t\t})" + tpTestFunc = "func Test%s%s(t *testing.T){%s\n\tConvey(\"%s\", t, func(){\n\t\t%s\tConvey(\"When everything goes positive\", func(){\n\t\t\t%s\n\t\t\t})\n\t\t})%s\n\t})\n}\n\n" + tpTestDaoMain = `func TestMain(m *testing.M) { + flag.Set("conf", "%s") + flag.Parse() + %s + os.Exit(m.Run()) +} +` + tpTestServiceMain = `func TestMain(m *testing.M){ + flag.Set("conf", "%s") + flag.Parse() + %s + os.Exit(m.Run()) +} +` + tpTestMainNew = `if err := paladin.Init(); err != nil { + panic(err) + } + %s` + tpTestMainOld = `if err := conf.Init(); err != nil { + panic(err) + } + %s` + print = `Generation success! + 莫生气 + 代码辣鸡非我意, + 自己动手分田地; + 你若气死谁如意? + 谈笑风生活长命. +// Release 1.2.3. Powered by Kratos` +) diff --git a/tool/testgen/utils.go b/tool/testgen/utils.go new file mode 100644 index 000000000..025fb91a7 --- /dev/null +++ b/tool/testgen/utils.go @@ -0,0 +1,42 @@ +package main + +import ( + "fmt" + "strings" + + "golang.org/x/tools/imports" +) + +// GoImport Use golang.org/x/tools/imports auto import pkg +func GoImport(file string, bytes []byte) (res []byte, err error) { + options := &imports.Options{ + TabWidth: 8, + TabIndent: true, + Comments: true, + Fragment: true, + } + if res, err = imports.Process(file, bytes, options); err != nil { + fmt.Printf("GoImport(%s) error(%v)", file, err) + res = bytes + return + } + return +} + +// ConvertMethod checkout the file belongs to dao or not +func ConvertMethod(path string) (method string) { + switch { + case strings.Contains(path, "/dao"): + method = "d" + case strings.Contains(path, "/service"): + method = "s" + default: + method = "" + } + return +} + +//ConvertHump convert words to hump style +func ConvertHump(words string) string { + return strings.ToUpper(words[0:1]) + words[1:] +}