godsoul
5 years ago
committed by
GitHub
commit
60c1b68039
@ -1,4 +1,31 @@ |
||||
go.sum |
||||
BUILD |
||||
# idea ignore |
||||
.idea/ |
||||
*.ipr |
||||
*.iml |
||||
*.iws |
||||
.vscode/ |
||||
|
||||
# temp ignore |
||||
*.log |
||||
*.cache |
||||
*.diff |
||||
*.exe |
||||
*.exe~ |
||||
*.patch |
||||
*.swp |
||||
*.tmp |
||||
|
||||
# system ignore |
||||
.DS_Store |
||||
Thumbs.db |
||||
|
||||
# project |
||||
*.cert |
||||
*.key |
||||
tool/kratos/kratos |
||||
tool/kratos-protoc/kratos-protoc |
||||
tool/kratos-gen-bts/kratos-gen-bts |
||||
tool/kratos-gen-mc/kratos-gen-mc |
||||
tool/kratos/kratos-protoc/kratos-protoc |
||||
tool/kratos/protobuf/protoc-gen-bm/protoc-gen-bm |
||||
tool/kratos/protobuf/protoc-gen-bswagger/protoc-gen-bswagger |
||||
|
||||
@ -0,0 +1,21 @@ |
||||
MIT License |
||||
|
||||
Copyright (c) 2018 bilibili |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all |
||||
copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
SOFTWARE. |
||||
|
After Width: | Height: | Size: 34 KiB |
|
After Width: | Height: | Size: 7.3 KiB |
|
After Width: | Height: | Size: 7.8 KiB |
|
After Width: | Height: | Size: 5.8 KiB |
|
Before Width: | Height: | Size: 2.1 MiB |
|
After Width: | Height: | Size: 661 KiB |
|
After Width: | Height: | Size: 19 KiB |
@ -0,0 +1,40 @@ |
||||
# Kratos |
||||
|
||||
Kratos是bilibili开源的一套Go微服务框架，包含大量微服务相关框架及工具。 |
||||
|
||||
### Goals |
||||
|
||||
我们致力于提供完整的微服务研发体验，整合相关框架及工具后，微服务治理相关部分可对整体业务开发周期无感，从而更加聚焦于业务交付。对每位开发者而言，整套Kratos框架也是不错的学习仓库，可以了解和参考到bilibili在微服务方面的技术积累和经验。 |
||||
|
||||
### Principles |
||||
|
||||
* 简单：不过度设计，代码平实简单 |
||||
* 通用：通用业务开发所需要的基础库的功能 |
||||
* 高效：提高业务迭代的效率 |
||||
* 稳定：基础库可测试性高，覆盖率高，有线上实践安全可靠 |
||||
* 健壮：通过良好的基础库设计，减少错用 |
||||
* 高性能：性能高，但不特定为了性能做hack优化，引入unsafe |
||||
* 扩展性：良好的接口设计，来扩展实现，或者通过新增基础库目录来扩展功能 |
||||
* 容错性：为失败设计，大量引入对SRE的理解，鲁棒性高 |
||||
* 工具链：包含大量工具链，比如cache代码生成，lint工具等等 |
||||
|
||||
### Features |
||||
* HTTP Blademaster：核心基于[gin](https://github.com/gin-gonic/gin)进行模块化设计，简单易用、核心足够轻量； |
||||
* GRPC Warden：基于官方gRPC开发，集成[discovery](https://github.com/bilibili/discovery)服务发现，并融合P2C负载均衡； |
||||
* Cache：优雅的接口化设计，非常方便的缓存序列化，推荐结合代理模式[overlord](https://github.com/bilibili/overlord)； |
||||
* Database：集成MySQL/HBase/TiDB，添加熔断保护和统计支持，可快速发现数据层压力； |
||||
* Config：方便易用的[paladin sdk](config-paladin.md)，可配合远程配置中心，实现配置版本管理和更新； |
||||
* Log：类似[zap](https://github.com/uber-go/zap)的field实现高性能日志库，并结合log-agent实现远程日志管理； |
||||
* Trace：基于opentracing，集成了全链路trace支持（gRPC/HTTP/MySQL/Redis/Memcached）； |
||||
* Kratos Tool：工具链，可快速生成标准项目，或者通过Protobuf生成代码，非常便捷使用gRPC、HTTP、swagger文档； |
||||
|
||||
|
||||
<br> |
||||
|
||||
------------- |
||||
|
||||
> 名字来源于:《战神》游戏以希腊神话为背景，讲述由凡人成为战神的奎托斯（Kratos）成为战神并展开弑神屠杀的冒险历程。 |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,86 @@ |
||||
# 介绍 |
||||
|
||||
基于proto文件可以快速生成`bm`框架对应的代码，提前需要准备以下工作： |
||||
|
||||
* 安装`kratos tool protoc`工具，请看[kratos工具](kratos-tool.md) |
||||
* 编写`proto`文件，示例可参考[kratos-demo内proto文件](https://github.com/bilibili/kratos-demo/blob/master/api/api.proto) |
||||
|
||||
### kratos工具说明 |
||||
|
||||
`kratos tool protoc`工具可以生成`warden` `bm` `swagger`对应的代码和文档，想要单独生成`bm`代码只需加上`--bm`如： |
||||
|
||||
```shell |
||||
# generate BM HTTP |
||||
kratos tool protoc --bm api.proto |
||||
``` |
||||
|
||||
### proto文件说明 |
||||
|
||||
请注意想要生成`bm`代码，需要特别在`proto`的`service`内指定`google.api.http`配置，如下： |
||||
|
||||
```go |
||||
service Demo { |
||||
rpc SayHello (HelloReq) returns (.google.protobuf.Empty); |
||||
rpc SayHelloURL(HelloReq) returns (HelloResp) { |
||||
option (google.api.http) = { // 该配置指定SayHelloURL方法对应的url |
||||
get:"/kratos-demo/say_hello" // 指定url和请求方式为GET |
||||
}; |
||||
}; |
||||
} |
||||
``` |
||||
|
||||
# 使用 |
||||
|
||||
建议在项目`api`目录下编写`proto`文件及生成对应的代码，可参考[kratos-demo内的api目录](https://github.com/bilibili/kratos-demo/tree/master/api)。 |
||||
|
||||
执行命令后生成的`api.bm.go`代码，注意其中的`type DemoBMServer interface`和`RegisterDemoBMServer`，其中： |
||||
|
||||
* `DemoBMServer`接口，包含`proto`文件内配置了`google.api.http`选项的所有方法 |
||||
* `RegisterDemoBMServer`方法提供注册`DemoBMServer`接口的实现对象，和`bm`的`Engine`用于注册路由 |
||||
* `DemoBMServer`接口的实现，一般为`internal/service`内的业务逻辑代码，需要实现`DemoBMServer`接口 |
||||
|
||||
使用`RegisterDemoBMServer`示例代码请参考[kratos-demo内的http](https://github.com/bilibili/kratos-demo/blob/master/internal/server/http/server.go)内的如下代码： |
||||
|
||||
```go |
||||
engine = bm.DefaultServer(hc.Server) |
||||
pb.RegisterDemoBMServer(engine, svc) |
||||
initRouter(engine) |
||||
``` |
||||
|
||||
`internal/service`内的`Service`结构实现了`DemoBMServer`接口可参考[kratos-demo内的service](https://github.com/bilibili/kratos-demo/blob/master/internal/service/service.go)内的如下代码： |
||||
|
||||
```go |
||||
// SayHelloURL bm demo func. |
||||
func (s *Service) SayHelloURL(ctx context.Context, req *pb.HelloReq) (reply *pb.HelloResp, err error) { |
||||
reply = &pb.HelloResp{ |
||||
Content: "hello " + req.Name, |
||||
} |
||||
fmt.Printf("hello url %s", req.Name) |
||||
return |
||||
} |
||||
``` |
||||
|
||||
# 文档 |
||||
|
||||
基于同一份`proto`文件还可以生成对应的`swagger`文档，运行命令如下： |
||||
|
||||
```shell |
||||
# generate swagger |
||||
kratos tool protoc --swagger api.proto |
||||
``` |
||||
|
||||
该命令将生成对应的`swagger.json`文件，可用于`swagger`工具通过WEBUI的方式打开使用，可运行命令如下： |
||||
|
||||
```shell |
||||
kratos tool swagger serve api/api.swagger.json |
||||
``` |
||||
|
||||
# 扩展阅读 |
||||
|
||||
[bm快速开始](blademaster-quickstart.md) |
||||
[bm模块说明](blademaster-mod.md) |
||||
[bm中间件](blademaster-mid.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,23 @@ |
||||
# 背景 |
||||
|
||||
我们需要统一的cache包，用于进行各类缓存操作。 |
||||
|
||||
# 概览 |
||||
|
||||
* 缓存操作均使用连接池，保证较快的数据读写速度且提高系统的安全可靠性。 |
||||
|
||||
# Memcache |
||||
|
||||
提供protobuf，gob，json序列化方式，gzip的memcache接口 |
||||
|
||||
[memcache模块说明](cache-mc.md) |
||||
|
||||
# Redis |
||||
|
||||
提供redis操作的各类接口以及各类将redis server返回值转换为golang类型的快捷方法。 |
||||
|
||||
[redis模块说明](cache-redis.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,111 @@ |
||||
# Paladin SDK |
||||
|
||||
## 配置模块化 |
||||
进行配置的模块化是为了更好地管理配置，尽可能避免由修改配置带来的失误。 |
||||
在配置种类里，可以看到其实 环境配置 和 应用配置 已经由平台进行管理化。 |
||||
我们通常业务里只用配置 业务配置 和 在线配置 就可以了，之前我们大部分都是单个文件配置，而为了更好管理我们需要按类型进行拆分配置文件。 |
||||
|
||||
例如： |
||||
|
||||
| 名称 | 说明 | |
||||
|:------|:------| |
||||
| application.toml | 在线配置 | |
||||
| mysql.toml | 业务db配置 | |
||||
| hbase.toml | 业务hbase配置 | |
||||
| memcache.toml | 业务mc配置 | |
||||
| redis.toml | 业务redis配置 | |
||||
| http.toml | 业务http client/server/auth配置 | |
||||
| grpc.toml | 业务grpc client/server配置 | |
||||
|
||||
## 使用方式 |
||||
|
||||
paladin 是一个config SDK客户端，包括了remote、file、mock几个抽象功能，方便使用本地文件或者远程配置中心，并且集成了对象自动reload功能。 |
||||
|
||||
### 远程配置中心 |
||||
可以通过环境变量注入，例如：APP_ID/DEPLOY_ENV/ZONE/HOSTNAME，然后通过paladin实现远程配置中心SDK进行配合使用。 |
||||
|
||||
### 指定本地文件： |
||||
```shell |
||||
./cmd -conf=/data/conf/app/demo.toml |
||||
# or multi file |
||||
./cmd -conf=/data/conf/app/ |
||||
``` |
||||
|
||||
### mock配置文件 |
||||
```go |
||||
func TestMain(t *testing.M) { |
||||
mock := make(map[string]string]) |
||||
mock["application.toml"] = ` |
||||
demoSwitch = false |
||||
demoNum = 100 |
||||
demoAPI = "xxx" |
||||
` |
||||
paladin.DefaultClient = paladin.NewMock(mock) |
||||
} |
||||
``` |
||||
|
||||
### example main |
||||
```go |
||||
# main.go |
||||
func main() { |
||||
# 初始化paladin |
||||
if err := paladin.Init(); err != nil { |
||||
panic(err) |
||||
} |
||||
log.Init(nil) // debug flag: log.dir={path} |
||||
defer log.Close() |
||||
} |
||||
``` |
||||
|
||||
### example HTTP/gRPC |
||||
```go |
||||
# http.toml |
||||
[server] |
||||
addr = "0.0.0.0:9000" |
||||
timeout = "1s" |
||||
|
||||
# server.go |
||||
func NewServer() { |
||||
# 默认配置用nil，这时读取HTTP/gRPC构架中的flag或者环境变量（可能是docker注入的环境变量，默认端口：8000/9000） |
||||
engine := bm.DefaultServer(nil) |
||||
|
||||
# 除非自己要替换了配置，用http.toml |
||||
var bc struct { |
||||
Server *bm.ServerConfig |
||||
} |
||||
if err = paladin.Get("http.toml").UnmarshalTOML("server", &bc); err != nil { |
||||
// 不存在时，将会为nil使用默认配置 |
||||
if err ！= paladin.ErrNotExist { |
||||
panic(err) |
||||
} |
||||
} |
||||
engine := bm.DefaultServer(conf) |
||||
} |
||||
``` |
||||
|
||||
### example Service（在线配置热加载配置） |
||||
```go |
||||
# service.go |
||||
type Service struct { |
||||
ac *paladin.Map |
||||
} |
||||
func New() *Service { |
||||
# paladin.Map 通过atomic.Value支持自动热加载 |
||||
var ac = new(paladin.TOML) |
||||
if err := paladin.Watch("application.toml", ac); err != nil { |
||||
panic(err) |
||||
} |
||||
s := &Service{ |
||||
ac : ac; |
||||
} |
||||
return s |
||||
} |
||||
func (s *Service) Test() { |
||||
switch, err := s.ac.Bool("switch") |
||||
if err != nil { |
||||
// TODO |
||||
} |
||||
# or use default value |
||||
switch := paladin.Bool(s.ac.Value("switch"), false) |
||||
} |
||||
``` |
||||
@ -0,0 +1,51 @@ |
||||
# config |
||||
|
||||
## 介绍 |
||||
初看起来，配置管理可能很简单，但是这其实是不稳定的一个重要来源。 |
||||
即变更管理导致的故障，我们目前基于配置中心（config-service）的部署方式，二级制文件的发布与配置文件的修改是异步进行的，每次变更配置，需要重新构建发版。 |
||||
由此，我们整体对配置文件进行梳理，对配置进行模块化，以及方便易用的paladin config sdk。 |
||||
|
||||
## 环境配置 |
||||
|
||||
| flag | env | remark | |
||||
|:----------|:----------|:------| |
||||
| region | REGION | 部署地区，sh-上海、gz-广州、bj-北京 | |
||||
| zone | ZONE | 分布区域，sh001-上海核心、sh004-上海嘉定 | |
||||
| deploy.env | DEPLOY_ENV | dev-开发、fat1-功能、uat-集成、pre-预发、prod-生产 | |
||||
| deploy.color | DEPLOY_COLOR | 服务颜色，blue（测试feature染色请求） | |
||||
| - | HOSTNAME | 主机名，xxx-hostname | |
||||
|
||||
全局公用环境变量，通常为部署环境配置，由系统、发布系统或supervisor进行环境变量注入，并不用进行例外配置，如果开发过程中可以通过flag注入进行运行测试。 |
||||
|
||||
## 应用配置 |
||||
|
||||
| flag | env | default | remark | |
||||
|:----------|:----------|:-------------|:------| |
||||
| appid | APP_ID | - | 应用ID | |
||||
| http | HTTP | tcp://0.0.0.0:8000/?timeout=1s | http 监听端口 | |
||||
| http.perf | HTTP_PERF | tcp://0.0.0.0:2233/?timeout=1s | http perf 监听端口 | |
||||
| grpc | GRPC | tcp://0.0.0.0:9000/?timeout=1s&idle_timeout=60s | grpc 监听端口 | |
||||
| grpc.target | - | - | 指定服务运行：<br>-grpc.target=demo.service=127.0.0.1:9000 <br>-grpc.target=demo.service=127.0.0.2:9000 | |
||||
| discovery.nodes | DISCOVERY_NODES | - | 服务发现节点：127.0.0.1:7171,127.0.0.2:7171 | |
||||
| log.v | LOG_V | 0 | 日志级别：<br>DEBUG:0 INFO:1 WARN:2 ERROR:3 FATAL:4 | |
||||
| log.stdout | LOG_STDOUT | false | 是否标准输出：true、false| |
||||
| log.dir | LOG_DIR | - | 日志文件目录，如果配置会输出日志到文件，否则不输出日志文件 | |
||||
| log.agent | LOG_AGENT | - | 日志采集agent：<br>unixpacket:///var/run/lancer/collector_tcp.sock?timeout=100ms&chan=1024 | |
||||
| log.module | LOG_MODULE | - | 指定field信息 format: file=1,file2=2. | |
||||
| log.filter | LOG_FILTER | - | 过虑敏感信息 format: field1,field2. | |
||||
|
||||
基本为一些应用相关的配置信息，通常发布系统和supervisor都有对应的部署环境进行配置注入，并不用进行例外配置，如果开发过程中可以通过flag进行注入运行测试。 |
||||
|
||||
## 业务配置 |
||||
Redis、MySQL等业务组件，可以使用静态的配置文件来初始化，根据应用业务集群进行配置。 |
||||
|
||||
## 在线配置 |
||||
需要在线读取、变更的配置信息，比如某个业务开关，可以实现配置reload实时更新。 |
||||
|
||||
## 扩展阅读 |
||||
|
||||
[paladin配置sdk](config-paladin.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,54 @@ |
||||
# database/hbase |
||||
|
||||
## 说明 |
||||
Hbase Client，进行封装加入了链路追踪和统计。 |
||||
|
||||
## 配置 |
||||
需要指定hbase集群的zookeeper地址。 |
||||
``` |
||||
config := &hbase.Config{Zookeeper: &hbase.ZKConfig{Addrs: []string{"localhost"}}} |
||||
client := hbase.NewClient(config) |
||||
``` |
||||
|
||||
## 使用方式 |
||||
``` |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
|
||||
"github.com/bilibili/kratos/pkg/database/hbase" |
||||
) |
||||
|
||||
func main() { |
||||
config := &hbase.Config{Zookeeper: &hbase.ZKConfig{Addrs: []string{"localhost"}}} |
||||
client := hbase.NewClient(config) |
||||
|
||||
// |
||||
values := map[string]map[string][]byte{"name": {"firstname": []byte("hello"), "lastname": []byte("world")}} |
||||
ctx := context.Background() |
||||
|
||||
// 写入信息 |
||||
// table: user |
||||
// rowkey: user1 |
||||
// values["family"] = columns |
||||
_, err := client.PutStr(ctx, "user", "user1", values) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
|
||||
// 读取信息 |
||||
// table: user |
||||
// rowkey: user1 |
||||
result, err := client.GetStr(ctx, "user", "user1") |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
fmt.Printf("%v", result) |
||||
} |
||||
``` |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,21 @@ |
||||
# database/sql |
||||
|
||||
## 背景 |
||||
数据库驱动，进行封装加入了熔断、链路追踪和统计，以及链路超时。 |
||||
通常数据模块都写在`internal/dao`目录中，并提供对应的数据访问接口。 |
||||
|
||||
## MySQL |
||||
MySQL数据库驱动，支持读写分离、context、timeout、trace和统计功能，以及错误熔断防止数据库雪崩。 |
||||
[mysql client](database-mysql.md) |
||||
|
||||
## HBase |
||||
HBase客户端，支持trace、slowlog和统计功能。 |
||||
[hbase client](database-hbase.md) |
||||
|
||||
## TiDB |
||||
TiDB客户端，支持服务发现和熔断功能。 |
||||
[tidb client](database-tidb.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,30 @@ |
||||
### kratos tool genbts |
||||
|
||||
> 缓存回源代码生成 |
||||
|
||||
在internal/dao/dao.go中添加mc缓存interface定义，可以指定对应的[注解参数](../../tool/kratos-gen-mc/README.md)； |
||||
并且在接口前面添加`go:generate kratos tool genbts`； |
||||
然后在当前目录执行`go generate`，可以看到自动生成的dao.bts.go代码。 |
||||
|
||||
### 回源模板 |
||||
```go |
||||
//go:generate kratos tool genbts |
||||
type _bts interface { |
||||
// bts: -batch=2 -max_group=20 -batch_err=break -nullcache=&Demo{ID:-1} -check_null_code=$.ID==-1 |
||||
Demos(c context.Context, keys []int64) (map[int64]*Demo, error) |
||||
// bts: -sync=true -nullcache=&Demo{ID:-1} -check_null_code=$.ID==-1 |
||||
Demo(c context.Context, key int64) (*Demo, error) |
||||
// bts: -paging=true |
||||
Demo1(c context.Context, key int64, pn int, ps int) (*Demo, error) |
||||
// bts: -nullcache=&Demo{ID:-1} -check_null_code=$.ID==-1 |
||||
None(c context.Context) (*Demo, error) |
||||
} |
||||
``` |
||||
|
||||
### 参考 |
||||
|
||||
也可以参考完整的testdata例子：kratos/tool/kratos-gen-bts/testdata |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,71 @@ |
||||
### kratos tool genmc |
||||
|
||||
> 缓存代码生成 |
||||
|
||||
在internal/dao/dao.go中添加mc缓存interface定义，可以指定对应的[注解参数](../../tool/kratos-gen-mc/README.md)； |
||||
并且在接口前面添加`go:generate kratos tool genmc`； |
||||
然后在当前目录执行`go generate`，可以看到自动生成的mc.cache.go代码。 |
||||
|
||||
### 缓存模板 |
||||
```go |
||||
//go:generate kratos tool genmc |
||||
type _mc interface { |
||||
// mc: -key=demoKey |
||||
CacheDemos(c context.Context, keys []int64) (map[int64]*Demo, error) |
||||
// mc: -key=demoKey |
||||
CacheDemo(c context.Context, key int64) (*Demo, error) |
||||
// mc: -key=keyMid |
||||
CacheDemo1(c context.Context, key int64, mid int64) (*Demo, error) |
||||
// mc: -key=noneKey |
||||
CacheNone(c context.Context) (*Demo, error) |
||||
// mc: -key=demoKey |
||||
CacheString(c context.Context, key int64) (string, error) |
||||
|
||||
// mc: -key=demoKey -expire=d.demoExpire -encode=json |
||||
AddCacheDemos(c context.Context, values map[int64]*Demo) error |
||||
// mc: -key=demo2Key -expire=d.demoExpire -encode=json |
||||
AddCacheDemos2(c context.Context, values map[int64]*Demo, tp int64) error |
||||
// 这里也支持自定义注释 会替换默认的注释 |
||||
// mc: -key=demoKey -expire=d.demoExpire -encode=json|gzip |
||||
AddCacheDemo(c context.Context, key int64, value *Demo) error |
||||
// mc: -key=keyMid -expire=d.demoExpire -encode=gob |
||||
AddCacheDemo1(c context.Context, key int64, value *Demo, mid int64) error |
||||
// mc: -key=noneKey |
||||
AddCacheNone(c context.Context, value *Demo) error |
||||
// mc: -key=demoKey -expire=d.demoExpire |
||||
AddCacheString(c context.Context, key int64, value string) error |
||||
|
||||
// mc: -key=demoKey |
||||
DelCacheDemos(c context.Context, keys []int64) error |
||||
// mc: -key=demoKey |
||||
DelCacheDemo(c context.Context, key int64) error |
||||
// mc: -key=keyMid |
||||
DelCacheDemo1(c context.Context, key int64, mid int64) error |
||||
// mc: -key=noneKey |
||||
DelCacheNone(c context.Context) error |
||||
} |
||||
|
||||
func demoKey(id int64) string { |
||||
return fmt.Sprintf("art_%d", id) |
||||
} |
||||
|
||||
func demo2Key(id, tp int64) string { |
||||
return fmt.Sprintf("art_%d_%d", id, tp) |
||||
} |
||||
|
||||
func keyMid(id, mid int64) string { |
||||
return fmt.Sprintf("art_%d_%d", id, mid) |
||||
} |
||||
|
||||
func noneKey() string { |
||||
return "none" |
||||
} |
||||
``` |
||||
|
||||
### 参考 |
||||
|
||||
也可以参考完整的testdata例子：kratos/tool/kratos-gen-mc/testdata |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,38 @@ |
||||
### kratos tool protoc |
||||
|
||||
``` |
||||
// generate all |
||||
kratos tool protoc api.proto |
||||
// generate gRPC |
||||
kratos tool protoc --grpc api.proto |
||||
// generate BM HTTP |
||||
kratos tool protoc --bm api.proto |
||||
// generate swagger |
||||
kratos tool protoc --swagger api.proto |
||||
``` |
||||
执行对应生成 `api.pb.go/api.bm.go/api.swagger.json` 源文档。 |
||||
|
||||
> 该工具在Windows/Linux下运行，需提前安装好 protobuf 工具 |
||||
|
||||
该工具实际是一段`shell`脚本，其中自动将`protoc`命令进行了拼接，识别了需要的`*.proto`文件和当前目录下的`proto`文件，最终会拼接为如下命令进行执行： |
||||
|
||||
```shell |
||||
export $KRATOS_HOME = kratos路径 |
||||
export $KRATOS_DEMO = 项目路径 |
||||
|
||||
// 生成：api.pb.go |
||||
protoc -I$GOPATH/src:$KRATOS_HOME/third_party:$KRATOS_DEMO/api --gofast_out=plugins=grpc:$KRATOS_DEMO/api $KRATOS_DEMO/api/api.proto |
||||
|
||||
// 生成：api.bm.go |
||||
protoc -I$GOPATH/src:$KRATOS_HOME/third_party:$KRATOS_DEMO/api --bm_out=$KRATOS_DEMO/api $KRATOS_DEMO/api/api.proto |
||||
|
||||
// 生成：api.swagger.json |
||||
protoc -I$GOPATH/src:$KRATOS_HOME/third_party:$KRATOS_DEMO/api --bswagger_out=$KRATOS_DEMO/api $KRATOS_DEMO/api/api.proto |
||||
``` |
||||
|
||||
大家也可以参考该命令进行`proto`生成，也可以参考 [protobuf](https://github.com/google/protobuf) 官方参数。 |
||||
|
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,11 @@ |
||||
### kratos tool swagger |
||||
```shell |
||||
kratos tool swagger serve api/api.swagger.json |
||||
``` |
||||
执行命令后，浏览器会自动打开swagger文档地址。 |
||||
同时也可以查看更多的 [go-swagger](https://github.com/go-swagger/go-swagger) 官方参数进行使用。 |
||||
|
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,102 @@ |
||||
# 介绍 |
||||
|
||||
kratos包含了一批好用的工具集，比如项目一键生成、基于proto生成http&grpc代码，生成缓存回源代码，生成memcache执行代码，生成swagger文档等。 |
||||
|
||||
# 获取工具 |
||||
|
||||
执行以下命令，即可快速安装好`kratos`工具 |
||||
```shell |
||||
go get -u github.com/bilibili/kratos/tool/kratos |
||||
``` |
||||
|
||||
那么接下来让我们快速开始熟悉工具的用法~ |
||||
|
||||
# kratos本体 |
||||
|
||||
`kratos`是所有工具集的本体，就像`go`一样，拥有执行各种子工具的能力，如`go build`和`go tool`。先让我们看看`-h`的输出： |
||||
|
||||
``` |
||||
NAME: |
||||
kratos - kratos tool |
||||
|
||||
USAGE: |
||||
kratos [global options] command [command options] [arguments...] |
||||
|
||||
VERSION: |
||||
0.0.1 |
||||
|
||||
COMMANDS: |
||||
new, n create new project |
||||
build, b kratos build |
||||
run, r kratos run |
||||
tool, t kratos tool |
||||
version, v kratos version |
||||
self-upgrade kratos self-upgrade |
||||
help, h Shows a list of commands or help for one command |
||||
|
||||
GLOBAL OPTIONS: |
||||
--help, -h show help |
||||
--version, -v print the version |
||||
``` |
||||
|
||||
可以看到`kratos`有如：`new` `build` `run` `tool`等在内的COMMANDS，那么接下来一一演示如何使用。 |
||||
|
||||
# kratos new |
||||
|
||||
`kratos new`是快速创建一个项目的命令，执行如下： |
||||
|
||||
```shell |
||||
kratos new kratos-demo |
||||
``` |
||||
|
||||
即可快速在当前目录生成一个叫`kratos-demo`的项目。此外还支持指定owner和path，如下： |
||||
|
||||
```shell |
||||
kratos new kratos-demo -o YourName -d YourPath |
||||
``` |
||||
|
||||
注意，`kratos new`默认是不会生成通过 protobuf 定义的`grpc`和`bm`示例代码的，如需生成请加`--proto`，如下： |
||||
|
||||
```shell |
||||
kratos new kratos-demo -o YourName -d YourPath --proto |
||||
``` |
||||
|
||||
> 特别注意，如果不是MacOS系统，需要自己进行手动安装protoc，用于生成的示例项目`api`目录下的`proto`文件并不会自动生成对应的`.pb.go`和`.bm.go`文件。 |
||||
|
||||
> 也可以参考以下说明进行生成：[protoc说明](protoc.md) |
||||
|
||||
# kratos build & run |
||||
|
||||
`kratos build`和`kratos run`是`go build`和`go run`的封装，可以在当前项目任意目录进行快速运行进行调试，并无特别用途。 |
||||
|
||||
# kratos tool |
||||
|
||||
`kratos tool`是基于proto生成http&grpc代码，生成缓存回源代码，生成memcache执行代码，生成swagger文档等工具集，先看下的执行效果： |
||||
|
||||
``` |
||||
kratos tool |
||||
|
||||
swagger(已安装): swagger api文档 Author(goswagger.io) [2019/05/05] |
||||
protoc(已安装): 快速方便生成pb.go和bm.go的protoc封装，windows、Linux请先安装protoc工具 Author(kratos) [2019/05/04] |
||||
kratos(已安装): Kratos工具集本体 Author(kratos) [2019/04/02] |
||||
|
||||
安装工具: kratos tool install demo |
||||
执行工具: kratos tool demo |
||||
安装全部工具: kratos tool install all |
||||
|
||||
详细文档： https://github.com/bilibili/kratos/blob/master/doc/wiki-cn/kratos-tool.md |
||||
``` |
||||
|
||||
> 小小说明：如未安装工具，第一次运行也可自动安装，不需要特别执行install |
||||
|
||||
目前已经集成的工具有： |
||||
|
||||
* [kratos](kratos-tool.md) 为本体工具，只用于安装更新使用； |
||||
* [protoc](kratos-protoc.md) 用于快速生成gRPC、HTTP、Swagger文件，该命令Windows，Linux用户需要手动安装 protobuf 工具； |
||||
* [swagger](kratos-swagger.md) 用于显示自动生成的HTTP API接口文档，通过 `kratos tool swagger serve api/api.swagger.json` 可以查看文档； |
||||
* [genmc](kratos-genmc.md) 用于自动生成memcached缓存代码； |
||||
* [genbts](kratos-genbts.md) 用于生成缓存回源代码生成，如果miss则调用回源函数从数据源获取，然后塞入缓存； |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,37 @@ |
||||
# 日志基础库 |
||||
|
||||
## 概览 |
||||
基于[zap](https://github.com/uber-go/zap)的field方式实现的高性能log库，提供Info、Warn、Error日志级别； |
||||
并提供了context支持，方便打印环境信息以及日志的链路追踪，在框架中都通过field方式实现，避免format日志带来的性能消耗。 |
||||
|
||||
## 配置选项 |
||||
|
||||
| flag | env | type | remark | |
||||
|:----------|:----------|:-------------:|:------| |
||||
| log.v | LOG_V | int | 日志级别：DEBUG:0 INFO:1 WARN:2 ERROR:3 FATAL:4 | |
||||
| log.stdout | LOG_STDOUT | bool | 是否标准输出：true、false| |
||||
| log.dir | LOG_DIR | string | 日志文件目录，如果配置会输出日志到文件，否则不输出日志文件 | |
||||
| log.agent | LOG_AGENT | string | 日志采集agent：unixpacket:///var/run/lancer/collector_tcp.sock?timeout=100ms&chan=1024 | |
||||
| log.module | LOG_MODULE | string | 指定field信息 format: file=1,file2=2. | |
||||
| log.filter | LOG_FILTER | string | 过虑敏感信息 format: field1,field2. | |
||||
|
||||
## 使用方式 |
||||
```go |
||||
func main() { |
||||
// 解析flag |
||||
flag.Parse() |
||||
// 初始化日志模块 |
||||
log.Init(nil) |
||||
// 打印日志 |
||||
log.Info("hi:%s", "kratos") |
||||
log.Infoc(Context.TODO(), "hi:%s", "kratos") |
||||
log.Infov(Context.TODO(), log.KVInt("key1", 100), log.KVString("key2", "test value") |
||||
} |
||||
``` |
||||
|
||||
## 扩展阅读 |
||||
* [log-agent](log-agent.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,29 @@ |
||||
# protoc |
||||
|
||||
`protobuf`是Google官方出品的一种轻便高效的结构化数据存储格式，可以用于结构化数据串行化，或者说序列化。它很适合做数据存储或 RPC 数据交换格式。可用于通讯协议、数据存储等领域的语言无关、平台无关、可扩展的序列化结构数据格式。 |
||||
|
||||
使用`protobuf`，需要先书写`.proto`文件，然后编译该文件。编译`proto`文件则需要使用到官方的`protoc`工具，安装文档请参看：[google官方protoc工具](https://github.com/protocolbuffers/protobuf#protocol-compiler-installation)。 |
||||
|
||||
注意：`protoc`是用于编辑`proto`文件的工具，它并不具备生成对应语言代码的能力，所以正常都是`protoc`配合对应语言的代码生成工具来使用，如Go语言的[gogo protobuf](https://github.com/gogo/protobuf)，请先点击按文档说明安装。 |
||||
|
||||
安装好对应工具后，我们可以进入`api`目录，执行如下命令： |
||||
|
||||
```shell |
||||
export $KRATOS_HOME = kratos路径 |
||||
export $KRATOS_DEMO = 项目路径 |
||||
|
||||
// 生成：api.pb.go |
||||
protoc -I$GOPATH/src:$KRATOS_HOME/tool/protobuf/pkg/extensions:$KRATOS_DEMO/api --gogofast_out=plugins=grpc:$KRATOS_DEMO/api $KRATOS_DEMO/api/api.proto |
||||
|
||||
// 生成：api.bm.go |
||||
protoc -I$GOPATH/src:$KRATOS_HOME/tool/protobuf/pkg/extensions:$KRATOS_DEMO/api --bm_out=$KRATOS_DEMO/api $KRATOS_DEMO/api/api.proto |
||||
|
||||
// 生成：api.swagger.json |
||||
protoc -I$GOPATH/src:$KRATOS_HOME/tool/protobuf/pkg/extensions:$KRATOS_DEMO/api --bswagger_out=$KRATOS_DEMO/api $KRATOS_DEMO/api/api.proto |
||||
``` |
||||
|
||||
请注意替换`/Users/felix/work/go/src`目录为你本地开发环境对应GOPATH目录，其中`--gogofast_out`意味着告诉`protoc`工具需要使用`gogo protobuf`的工具生成代码。 |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,60 @@ |
||||
# 自适应限流保护 |
||||
|
||||
kratos 借鉴了 Sentinel 项目的自适应限流系统，通过综合分析服务的 cpu 使用率、请求成功的 qps 和请求成功的 rt 来做自适应限流保护。 |
||||
|
||||
|
||||
## 核心目标 |
||||
|
||||
* 自动嗅探负载和 qps，减少人工配置 |
||||
* 削顶，保证超载时系统不被拖垮，并能以高水位 qps 继续运行 |
||||
|
||||
|
||||
## 限流规则 |
||||
|
||||
1，指标介绍 |
||||
|
||||
|指标名称|指标含义| |
||||
|---|---| |
||||
|cpu|最近 1s 的 CPU 使用率均值，使用滑动平均计算，采样周期是 250ms| |
||||
|inflight|当前处理中正在处理的请求数量| |
||||
|pass|请求处理成功的量| |
||||
|rt|请求成功的响应耗时| |
||||
|
||||
|
||||
2，滑动窗口 |
||||
|
||||
在自适应限流保护中，采集到的指标的时效性非常强，系统只需要采集最近一小段时间内的 qps、rt 即可，对于较老的数据，会自动丢弃。为了实现这个效果，kratos 使用了滑动窗口来保存采样数据。 |
||||
|
||||
 |
||||
|
||||
如上图，展示了一个具有两个桶（bucket）的滑动窗口（rolling window）。整个滑动窗口用来保存最近 1s 的采样数据，每个小的桶用来保存 500ms 的采样数据。 |
||||
当时间流动之后，过期的桶会自动被新桶的数据覆盖掉，在图中，在 1000-1500ms 时，bucket 1 的数据因为过期而被丢弃，之后 bucket 3 的数据填到了窗口的头部。 |
||||
|
||||
|
||||
3，限流公式 |
||||
|
||||
判断是否丢弃当前请求的算法如下： |
||||
|
||||
`cpu > 800 AND (Now - PrevDrop) < 1s AND (MaxPass * MinRt * windows / 1000) < InFlight` |
||||
|
||||
MaxPass 表示最近 5s 内，单个采样窗口中最大的请求数。 |
||||
MinRt 表示最近 5s 内，单个采样窗口中最小的响应时间。 |
||||
windows 表示一秒内采样窗口的数量，默认配置中是 5s 50 个采样，那么 windows 的值为 10。 |
||||
|
||||
## 压测报告 |
||||
|
||||
场景1，请求以每秒增加1个的速度不停上升，压测效果如下： |
||||
|
||||
 |
||||
|
||||
左测是没有限流的压测效果，右侧是带限流的压测效果。 |
||||
可以看到，没有限流的场景里，系统在 700qps 时开始抖动，在 1k qps 时被拖垮，几乎没有新的请求能被放行，然而在使用限流之后，系统请求能够稳定在 600 qps 左右，rt 没有暴增，服务也没有被打垮，可见，限流有效的保护了服务。 |
||||
|
||||
|
||||
参考资料： |
||||
|
||||
[Sentinel 系统自适应限流](https://github.com/alibaba/Sentinel/wiki/%E7%B3%BB%E7%BB%9F%E8%87%AA%E9%80%82%E5%BA%94%E9%99%90%E6%B5%81) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,43 @@ |
||||
# Warden Balancer |
||||
|
||||
## 介绍 |
||||
grpc-go内置了round-robin轮询，但由于自带的轮询算法不支持权重，也不支持color筛选等需求，故需要重新实现一个负载均衡算法。 |
||||
|
||||
## WRR (Weighted Round Robin) |
||||
该算法在加权轮询法基础上增加了动态调节权重值，用户可以在为每一个节点先配置一个初始的权重分，之后算法会根据节点cpu、延迟、服务端错误率、客户端错误率动态打分，在将打分乘用户自定义的初始权重分得到最后的权重值。 |
||||
|
||||
## P2C (Pick of two choices) |
||||
本算法通过随机选择两个node选择优胜者来避免羊群效应，并通过ewma尽量获取服务端的实时状态。 |
||||
|
||||
服务端： |
||||
服务端获取最近500ms内的CPU使用率（需要将cgroup设置的限制考虑进去，并除于CPU核心数），并将CPUC使用率乘与1000后塞入每次grpc请求中的的Trailer中夹带返回： |
||||
cpu_usage |
||||
uint64 encoded with string |
||||
cpu_usage : 1000 |
||||
|
||||
客户端： |
||||
主要参数： |
||||
* server_cpu：通过每次请求中服务端塞在trailer中的cpu_usage拿到服务端最近500ms内的cpu使用率 |
||||
* inflight：当前客户端正在发送并等待response的请求数（pending request） |
||||
* latency: 加权移动平均算法计算出的接口延迟 |
||||
* client_success:加权移动平均算法计算出的请求成功率（只记录grpc内部错误，比如context deadline） |
||||
|
||||
目前客户端，已经默认使用p2c负载均衡算法`grpc.WithBalancerName(p2c.Name)`： |
||||
```go |
||||
// NewClient returns a new blank Client instance with a default client interceptor. |
||||
// opt can be used to add grpc dial options. |
||||
func NewClient(conf *ClientConfig, opt ...grpc.DialOption) *Client { |
||||
c := new(Client) |
||||
if err := c.SetConfig(conf); err != nil { |
||||
panic(err) |
||||
} |
||||
c.UseOpt(grpc.WithBalancerName(p2c.Name)) |
||||
c.UseOpt(opt...) |
||||
c.Use(c.recovery(), clientLogging(), c.handle()) |
||||
return c |
||||
} |
||||
``` |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,376 @@ |
||||
# 说明 |
||||
|
||||
gRPC暴露了两个拦截器接口，分别是： |
||||
|
||||
* `grpc.UnaryServerInterceptor`服务端拦截器 |
||||
* `grpc.UnaryClientInterceptor`客户端拦截器 |
||||
|
||||
基于两个拦截器可以针对性的定制公共模块的封装代码，比如`warden/logging.go`是通用日志逻辑。 |
||||
|
||||
# 分析 |
||||
|
||||
## 服务端拦截器 |
||||
|
||||
让我们先看一下`grpc.UnaryServerInterceptor`的声明，[官方代码位置](https://github.com/grpc/grpc-go/blob/master/interceptor.go)： |
||||
|
||||
```go |
||||
// UnaryServerInfo consists of various information about a unary RPC on |
||||
// server side. All per-rpc information may be mutated by the interceptor. |
||||
type UnaryServerInfo struct { |
||||
// Server is the service implementation the user provides. This is read-only. |
||||
Server interface{} |
||||
// FullMethod is the full RPC method string, i.e., /package.service/method. |
||||
FullMethod string |
||||
} |
||||
|
||||
// UnaryHandler defines the handler invoked by UnaryServerInterceptor to complete the normal |
||||
// execution of a unary RPC. If a UnaryHandler returns an error, it should be produced by the |
||||
// status package, or else gRPC will use codes.Unknown as the status code and err.Error() as |
||||
// the status message of the RPC. |
||||
type UnaryHandler func(ctx context.Context, req interface{}) (interface{}, error) |
||||
|
||||
// UnaryServerInterceptor provides a hook to intercept the execution of a unary RPC on the server. info |
||||
// contains all the information of this RPC the interceptor can operate on. And handler is the wrapper |
||||
// of the service method implementation. It is the responsibility of the interceptor to invoke handler |
||||
// to complete the RPC. |
||||
type UnaryServerInterceptor func(ctx context.Context, req interface{}, info *UnaryServerInfo, handler UnaryHandler) (resp interface{}, err error) |
||||
``` |
||||
|
||||
看起来很简单包括： |
||||
|
||||
* 一个`UnaryServerInfo`结构体用于`Server`和`FullMethod`字段传递，`Server`为`gRPC server`的对象实例，`FullMethod`为请求方法的全名 |
||||
* 一个`UnaryHandler`方法用于传递`Handler`，就是基于`proto`文件`service`内声明而生成的方法 |
||||
* 一个`UnaryServerInterceptor`用于拦截`Handler`方法，可在`Handler`执行前后插入拦截代码 |
||||
|
||||
为了更形象的说明拦截器的执行过程，请看基于`proto`生成的以下代码[代码位置](https://github.com/bilibili/kratos-demo/blob/master/api/api.pb.go)： |
||||
|
||||
```go |
||||
func _Demo_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
||||
in := new(HelloReq) |
||||
if err := dec(in); err != nil { |
||||
return nil, err |
||||
} |
||||
if interceptor == nil { |
||||
return srv.(DemoServer).SayHello(ctx, in) |
||||
} |
||||
info := &grpc.UnaryServerInfo{ |
||||
Server: srv, |
||||
FullMethod: "/demo.service.v1.Demo/SayHello", |
||||
} |
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
||||
return srv.(DemoServer).SayHello(ctx, req.(*HelloReq)) |
||||
} |
||||
return interceptor(ctx, in, info, handler) |
||||
} |
||||
``` |
||||
|
||||
这个`_Demo_SayHello_Handler`方法是关键，该方法会被包装为`grpc.ServiceDesc`结构，被注册到gRPC内部，具体可在生成的`pb.go`代码内查找`s.RegisterService(&_Demo_serviceDesc, srv)`。 |
||||
|
||||
* 当`gRPC server`收到一次请求时，首先根据请求方法从注册到`server`内的`grpc.ServiceDesc`找到该方法对应的`Handler`如：`_Demo_SayHello_Handler`并执行 |
||||
* `_Demo_SayHello_Handler`执行过程请看上面具体代码，当`interceptor`不为`nil`时，会将`SayHello`包装为`grpc.UnaryHandler`结构传递给`interceptor` |
||||
|
||||
这样就完成了`UnaryServerInterceptor`的执行过程。那么`_Demo_SayHello_Handler`内的`interceptor`是如何注入到`gRPC server`内，则看下面这段代码[官方代码位置](https://github.com/grpc/grpc-go/blob/master/server.go)： |
||||
|
||||
```go |
||||
// UnaryInterceptor returns a ServerOption that sets the UnaryServerInterceptor for the |
||||
// server. Only one unary interceptor can be installed. The construction of multiple |
||||
// interceptors (e.g., chaining) can be implemented at the caller. |
||||
func UnaryInterceptor(i UnaryServerInterceptor) ServerOption { |
||||
return func(o *options) { |
||||
if o.unaryInt != nil { |
||||
panic("The unary server interceptor was already set and may not be reset.") |
||||
} |
||||
o.unaryInt = i |
||||
} |
||||
} |
||||
``` |
||||
|
||||
请一定注意这方法的注释！！！ |
||||
|
||||
> Only one unary interceptor can be installed. The construction of multiple interceptors (e.g., chaining) can be implemented at the caller. |
||||
|
||||
`gRPC`本身只支持一个`interceptor`，想要多`interceptors`需要自己实现~~所以`warden`基于`grpc.UnaryClientInterceptor`实现了`interceptor chain`，请看下面代码[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/net/rpc/warden/server.go)： |
||||
|
||||
```go |
||||
// Use attachs a global inteceptor to the server. |
||||
// For example, this is the right place for a rate limiter or error management inteceptor. |
||||
func (s *Server) Use(handlers ...grpc.UnaryServerInterceptor) *Server { |
||||
finalSize := len(s.handlers) + len(handlers) |
||||
if finalSize >= int(_abortIndex) { |
||||
panic("warden: server use too many handlers") |
||||
} |
||||
mergedHandlers := make([]grpc.UnaryServerInterceptor, finalSize) |
||||
copy(mergedHandlers, s.handlers) |
||||
copy(mergedHandlers[len(s.handlers):], handlers) |
||||
s.handlers = mergedHandlers |
||||
return s |
||||
} |
||||
|
||||
// interceptor is a single interceptor out of a chain of many interceptors. |
||||
// Execution is done in left-to-right order, including passing of context. |
||||
// For example ChainUnaryServer(one, two, three) will execute one before two before three, and three |
||||
// will see context changes of one and two. |
||||
func (s *Server) interceptor(ctx context.Context, req interface{}, args *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { |
||||
var ( |
||||
i int |
||||
chain grpc.UnaryHandler |
||||
) |
||||
|
||||
n := len(s.handlers) |
||||
if n == 0 { |
||||
return handler(ctx, req) |
||||
} |
||||
|
||||
chain = func(ic context.Context, ir interface{}) (interface{}, error) { |
||||
if i == n-1 { |
||||
return handler(ic, ir) |
||||
} |
||||
i++ |
||||
return s.handlers[i](ic, ir, args, chain) |
||||
} |
||||
|
||||
return s.handlers[0](ctx, req, args, chain) |
||||
} |
||||
``` |
||||
|
||||
很简单的逻辑： |
||||
|
||||
* `warden server`使用`Use`方法进行`grpc.UnaryServerInterceptor`的注入，而`func (s *Server) interceptor`本身就实现了`grpc.UnaryServerInterceptor` |
||||
* `func (s *Server) interceptor`可以根据注册的`grpc.UnaryServerInterceptor`顺序从前到后依次执行 |
||||
|
||||
而`warden`在初始化的时候将该方法本身注册到了`gRPC server`，在`NewServer`方法内可以看到下面代码： |
||||
|
||||
```go |
||||
opt = append(opt, keepParam, grpc.UnaryInterceptor(s.interceptor)) |
||||
s.server = grpc.NewServer(opt...) |
||||
``` |
||||
|
||||
如此完整的服务端拦截器逻辑就串联完成。 |
||||
|
||||
## 客户端拦截器 |
||||
|
||||
|
||||
让我们先看一下`grpc.UnaryClientInterceptor`的声明，[官方代码位置](https://github.com/grpc/grpc-go/blob/master/interceptor.go)： |
||||
|
||||
```go |
||||
// UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. |
||||
type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error |
||||
|
||||
// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. invoker is the handler to complete the RPC |
||||
// and it is the responsibility of the interceptor to call it. |
||||
// This is an EXPERIMENTAL API. |
||||
type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error |
||||
``` |
||||
|
||||
看起来和服务端拦截器并没有什么太大的区别，比较简单包括： |
||||
|
||||
* 一个`UnaryInvoker`表示客户端具体要发出的执行方法 |
||||
* 一个`UnaryClientInterceptor`用于拦截`Invoker`方法，可在`Invoker`执行前后插入拦截代码 |
||||
|
||||
具体执行过程，请看基于`proto`生成的下面代码[代码位置](https://github.com/bilibili/kratos-demo/blob/master/api/api.pb.go)： |
||||
|
||||
```go |
||||
func (c *demoClient) SayHello(ctx context.Context, in *HelloReq, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) { |
||||
out := new(google_protobuf1.Empty) |
||||
err := grpc.Invoke(ctx, "/demo.service.v1.Demo/SayHello", in, out, c.cc, opts...) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return out, nil |
||||
} |
||||
``` |
||||
|
||||
当客户端调用`SayHello`时可以看到执行了`grpc.Invoke`方法，并且将`fullMethod`和其他参数传入，最终会执行下面代码[官方代码位置](https://github.com/grpc/grpc-go/blob/master/call.go)： |
||||
|
||||
```go |
||||
// Invoke sends the RPC request on the wire and returns after response is |
||||
// received. This is typically called by generated code. |
||||
// |
||||
// All errors returned by Invoke are compatible with the status package. |
||||
func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply interface{}, opts ...CallOption) error { |
||||
// allow interceptor to see all applicable call options, which means those |
||||
// configured as defaults from dial option as well as per-call options |
||||
opts = combine(cc.dopts.callOptions, opts) |
||||
|
||||
if cc.dopts.unaryInt != nil { |
||||
return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...) |
||||
} |
||||
return invoke(ctx, method, args, reply, cc, opts...) |
||||
} |
||||
``` |
||||
|
||||
其中的`unaryInt`即为客户端连接创建时注册的拦截器，使用下面代码注册[官方代码位置](https://github.com/grpc/grpc-go/blob/master/dialoptions.go)： |
||||
|
||||
```go |
||||
// WithUnaryInterceptor returns a DialOption that specifies the interceptor for |
||||
// unary RPCs. |
||||
func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption { |
||||
return newFuncDialOption(func(o *dialOptions) { |
||||
o.unaryInt = f |
||||
}) |
||||
} |
||||
``` |
||||
|
||||
需要注意的是客户端的拦截器在官方`gRPC`内也只能支持注册一个，与服务端拦截器`interceptor chain`逻辑类似`warden`在客户端拦截器也做了相同处理，并且在客户端连接时进行注册，请看下面代码[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/net/rpc/warden/client.go)： |
||||
|
||||
```go |
||||
// Use attachs a global inteceptor to the Client. |
||||
// For example, this is the right place for a circuit breaker or error management inteceptor. |
||||
func (c *Client) Use(handlers ...grpc.UnaryClientInterceptor) *Client { |
||||
finalSize := len(c.handlers) + len(handlers) |
||||
if finalSize >= int(_abortIndex) { |
||||
panic("warden: client use too many handlers") |
||||
} |
||||
mergedHandlers := make([]grpc.UnaryClientInterceptor, finalSize) |
||||
copy(mergedHandlers, c.handlers) |
||||
copy(mergedHandlers[len(c.handlers):], handlers) |
||||
c.handlers = mergedHandlers |
||||
return c |
||||
} |
||||
|
||||
// chainUnaryClient creates a single interceptor out of a chain of many interceptors. |
||||
// |
||||
// Execution is done in left-to-right order, including passing of context. |
||||
// For example ChainUnaryClient(one, two, three) will execute one before two before three. |
||||
func (c *Client) chainUnaryClient() grpc.UnaryClientInterceptor { |
||||
n := len(c.handlers) |
||||
if n == 0 { |
||||
return func(ctx context.Context, method string, req, reply interface{}, |
||||
cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { |
||||
return invoker(ctx, method, req, reply, cc, opts...) |
||||
} |
||||
} |
||||
|
||||
return func(ctx context.Context, method string, req, reply interface{}, |
||||
cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { |
||||
var ( |
||||
i int |
||||
chainHandler grpc.UnaryInvoker |
||||
) |
||||
chainHandler = func(ictx context.Context, imethod string, ireq, ireply interface{}, ic *grpc.ClientConn, iopts ...grpc.CallOption) error { |
||||
if i == n-1 { |
||||
return invoker(ictx, imethod, ireq, ireply, ic, iopts...) |
||||
} |
||||
i++ |
||||
return c.handlers[i](ictx, imethod, ireq, ireply, ic, chainHandler, iopts...) |
||||
} |
||||
|
||||
return c.handlers[0](ctx, method, req, reply, cc, chainHandler, opts...) |
||||
} |
||||
} |
||||
``` |
||||
|
||||
如此完整的客户端拦截器逻辑就串联完成。 |
||||
|
||||
# 实现自己的拦截器 |
||||
|
||||
以服务端拦截器`logging`为例： |
||||
|
||||
```go |
||||
// serverLogging warden grpc logging |
||||
func serverLogging() grpc.UnaryServerInterceptor { |
||||
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { |
||||
// NOTE: handler执行之前的拦截代码：主要获取一些关键参数，如耗时计时、ip等 |
||||
// 如果自定义的拦截器只需要在handler执行后，那么可以直接执行handler |
||||
|
||||
startTime := time.Now() |
||||
caller := metadata.String(ctx, metadata.Caller) |
||||
if caller == "" { |
||||
caller = "no_user" |
||||
} |
||||
var remoteIP string |
||||
if peerInfo, ok := peer.FromContext(ctx); ok { |
||||
remoteIP = peerInfo.Addr.String() |
||||
} |
||||
var quota float64 |
||||
if deadline, ok := ctx.Deadline(); ok { |
||||
quota = time.Until(deadline).Seconds() |
||||
} |
||||
|
||||
// call server handler |
||||
resp, err := handler(ctx, req) // NOTE: 以具体执行的handler为分界线！！！ |
||||
|
||||
// NOTE: handler执行之后的拦截代码：主要进行耗时计算、日志记录 |
||||
// 如果自定义的拦截器在handler执行后不需要逻辑，这可直接返回 |
||||
|
||||
// after server response |
||||
code := ecode.Cause(err).Code() |
||||
duration := time.Since(startTime) |
||||
|
||||
// monitor |
||||
statsServer.Timing(caller, int64(duration/time.Millisecond), info.FullMethod) |
||||
statsServer.Incr(caller, info.FullMethod, strconv.Itoa(code)) |
||||
logFields := []log.D{ |
||||
log.KVString("user", caller), |
||||
log.KVString("ip", remoteIP), |
||||
log.KVString("path", info.FullMethod), |
||||
log.KVInt("ret", code), |
||||
// TODO: it will panic if someone remove String method from protobuf message struct that auto generate from protoc. |
||||
log.KVString("args", req.(fmt.Stringer).String()), |
||||
log.KVFloat64("ts", duration.Seconds()), |
||||
log.KVFloat64("timeout_quota", quota), |
||||
log.KVString("source", "grpc-access-log"), |
||||
} |
||||
if err != nil { |
||||
logFields = append(logFields, log.KV("error", err.Error()), log.KV("stack", fmt.Sprintf("%+v", err))) |
||||
} |
||||
logFn(code, duration)(ctx, logFields...) |
||||
return resp, err |
||||
} |
||||
} |
||||
``` |
||||
|
||||
# 内置拦截器 |
||||
|
||||
## 自适应限流拦截器 |
||||
|
||||
更多关于自适应限流的信息，请参考：[kratos 自适应限流](/doc/wiki-cn/ratelimit.md) |
||||
|
||||
```go |
||||
package grpc |
||||
|
||||
import ( |
||||
pb "kratos-demo/api" |
||||
"kratos-demo/internal/service" |
||||
"github.com/bilibili/kratos/pkg/conf/paladin" |
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden" |
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden/ratelimiter" |
||||
) |
||||
|
||||
// New new a grpc server. |
||||
func New(svc *service.Service) *warden.Server { |
||||
var rc struct { |
||||
Server *warden.ServerConfig |
||||
} |
||||
if err := paladin.Get("grpc.toml").UnmarshalTOML(&rc); err != nil { |
||||
if err != paladin.ErrNotExist { |
||||
panic(err) |
||||
} |
||||
} |
||||
ws := warden.NewServer(rc.Server) |
||||
|
||||
// 挂载自适应限流拦截器到 warden server，使用默认配置 |
||||
limiter := ratelimiter.New(nil) |
||||
ws.Use(limiter.Limit()) |
||||
|
||||
// 注意替换这里： |
||||
// RegisterDemoServer方法是在"api"目录下代码生成的 |
||||
// 对应proto文件内自定义的service名字，请使用正确方法名替换 |
||||
pb.RegisterDemoServer(ws.Server(), svc) |
||||
|
||||
ws, err := ws.Start() |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
return ws |
||||
} |
||||
``` |
||||
|
||||
# 扩展阅读 |
||||
|
||||
[warden快速开始](warden-quickstart.md) [warden基于pb生成](warden-pb.md) [warden负载均衡](warden-balancer.md) [warden服务发现](warden-resolver.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
|
||||
@ -0,0 +1,47 @@ |
||||
# 介绍 |
||||
|
||||
基于proto文件可以快速生成`warden`框架对应的代码，提前需要准备以下工作： |
||||
|
||||
* 安装`kratos tool protoc`工具，请看[kratos工具](kratos-tool.md) |
||||
* 编写`proto`文件，示例可参考[kratos-demo内proto文件](https://github.com/bilibili/kratos-demo/blob/master/api/api.proto) |
||||
|
||||
### kratos工具说明 |
||||
|
||||
`kratos tool protoc`工具可以生成`warden` `bm` `swagger`对应的代码和文档，想要单独生成`warden`代码只需加上`--grpc`如： |
||||
|
||||
```shell |
||||
# generate gRPC |
||||
kratos tool protoc --grpc api.proto |
||||
``` |
||||
|
||||
# 使用 |
||||
|
||||
建议在项目`api`目录下编写`proto`文件及生成对应的代码，可参考[kratos-demo内的api目录](https://github.com/bilibili/kratos-demo/tree/master/api)。 |
||||
|
||||
执行命令后生成的`api.pb.go`代码，注意其中的`DemoClient`和`DemoServer`，其中： |
||||
|
||||
* `DemoClient`接口为客户端调用接口，相对应的有`demoClient`结构体为其实现 |
||||
* `DemoServer`接口为服务端接口声明，需要业务自己实现该接口的所有方法，`kratos`建议在`internal/service`目录下使用`Service`结构体实现 |
||||
|
||||
`internal/service`内的`Service`结构实现了`DemoServer`接口可参考[kratos-demo内的service](https://github.com/bilibili/kratos-demo/blob/master/internal/service/service.go)内的如下代码： |
||||
|
||||
```go |
||||
// SayHelloURL bm demo func. |
||||
func (s *Service) SayHelloURL(ctx context.Context, req *pb.HelloReq) (reply *pb.HelloResp, err error) { |
||||
reply = &pb.HelloResp{ |
||||
Content: "hello " + req.Name, |
||||
} |
||||
fmt.Printf("hello url %s", req.Name) |
||||
return |
||||
} |
||||
``` |
||||
|
||||
更详细的客户端和服务端使用请看[warden快速开始](warden-quickstart.md) |
||||
|
||||
# 扩展阅读 |
||||
|
||||
[warden快速开始](warden-quickstart.md) [warden拦截器](warden-mid.md) [warden负载均衡](warden-balancer.md) [warden服务发现](warden-resolver.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,171 @@ |
||||
# 准备工作 |
||||
|
||||
推荐使用[kratos工具](kratos-tool.md)快速生成带`--grpc`的项目，如我们生成一个叫`kratos-demo`的项目。 |
||||
|
||||
# pb文件 |
||||
|
||||
创建项目成功后，进入`api`目录下可以看到`api.proto`和`api.pb.go`和`generate.go`文件，其中： |
||||
|
||||
* `api.proto`是gRPC server的描述文件 |
||||
* `api.pb.go`是基于`api.proto`生成的代码文件 |
||||
* `generate.go`是用于`kratos tool protoc`执行`go generate`进行代码生成的临时文件 |
||||
|
||||
接下来可以将以上三个文件全部删除或者保留`generate.go`，之后编写自己的proto文件，确认proto无误后，进行代码生成： |
||||
|
||||
* 可直接执行`kratos tool protoc`，该命令会调用protoc工具生成`.pb.go`文件 |
||||
* 如`generate.go`没删除，也可以执行`go generate`命令，将调用`kratos tool protoc`工具进行代码生成 |
||||
|
||||
[kratos工具请看](kratos-tool.md) |
||||
|
||||
### 如没看kprotoc文档，请看下面这段话 |
||||
|
||||
`kratos tool protoc`用于快速生成`pb.go`文件，但目前windows和Linux需要先自己安装`protoc`工具，具体请看[protoc说明](protoc.md)。 |
||||
|
||||
# 注册server |
||||
|
||||
进入`internal/server/grpc`目录打开`server.go`文件，可以看到以下代码，只需要替换以下注释内容就可以启动一个gRPC服务。 |
||||
|
||||
```go |
||||
package grpc |
||||
|
||||
import ( |
||||
pb "kratos-demo/api" |
||||
"kratos-demo/internal/service" |
||||
"github.com/bilibili/kratos/pkg/conf/paladin" |
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden" |
||||
) |
||||
|
||||
// New new a grpc server. |
||||
func New(svc *service.Service) *warden.Server { |
||||
var rc struct { |
||||
Server *warden.ServerConfig |
||||
} |
||||
if err := paladin.Get("grpc.toml").UnmarshalTOML(&rc); err != nil { |
||||
if err != paladin.ErrNotExist { |
||||
panic(err) |
||||
} |
||||
} |
||||
ws := warden.NewServer(rc.Server) |
||||
// 注意替换这里： |
||||
// RegisterDemoServer方法是在"api"目录下代码生成的 |
||||
// 对应proto文件内自定义的service名字，请使用正确方法名替换 |
||||
pb.RegisterDemoServer(ws.Server(), svc) |
||||
ws, err := ws.Start() |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
return ws |
||||
} |
||||
``` |
||||
|
||||
### 注册注意 |
||||
|
||||
```go |
||||
// SayHello grpc demo func. |
||||
func (s *Service) SayHello(ctx context.Context, req *pb.HelloReq) (reply *empty.Empty, err error) { |
||||
reply = new(empty.Empty) |
||||
fmt.Printf("hello %s", req.Name) |
||||
return |
||||
} |
||||
``` |
||||
|
||||
请进入`internal/service`内找到`SayHello`方法，注意方法的入参和出参，都是按照gRPC的方法声明对应的： |
||||
|
||||
* 第一个参数必须是`context.Context`，第二个必须是proto内定义的`message`对应生成的结构体 |
||||
* 第一个返回值必须是proto内定义的`message`对应生成的结构体，第二个参数必须是`error` |
||||
* 在http框架bm中，如果共用proto文件生成bm代码，那么也可以直接使用该service方法 |
||||
|
||||
建议service严格按照此格式声明方法使其能够在bm和warden内共用。 |
||||
|
||||
# client调用 |
||||
|
||||
请进入`internal/dao`方法内，一般对资源的处理都会在这一层封装。 |
||||
对于`client`端，前提必须有对应`proto`文件生成的代码，那么有两种选择： |
||||
|
||||
* 拷贝proto文件到自己项目下并且执行代码生成 |
||||
* 直接import服务端的api package |
||||
|
||||
> 这也是业务代码我们加了一层`internal`的关系，服务对外暴露的只有接口 |
||||
|
||||
不管哪一种方式，以下初始化gRPC client的代码建议伴随生成的代码存放在统一目录下： |
||||
|
||||
```go |
||||
package dao |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden" |
||||
|
||||
"google.golang.org/grpc" |
||||
) |
||||
|
||||
// target server addrs. |
||||
const target = "direct://default/127.0.0.1:9000,127.0.0.1:9091" // NOTE: example |
||||
|
||||
// NewClient new member grpc client |
||||
func NewClient(cfg *warden.ClientConfig, opts ...grpc.DialOption) (DemoClient, error) { |
||||
client := warden.NewClient(cfg, opts...) |
||||
conn, err := client.Dial(context.Background(), target) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
// 注意替换这里： |
||||
// NewDemoClient方法是在"api"目录下代码生成的 |
||||
// 对应proto文件内自定义的service名字，请使用正确方法名替换 |
||||
return NewDemoClient(conn), nil |
||||
} |
||||
``` |
||||
|
||||
其中，`target`为gRPC用于服务发现的目标，使用标准url资源格式提供给resolver用于服务发现。`warden`默认使用`direct`直连方式，直接与`server`端进行连接。如果在使用其他服务发现组件请看[warden服务发现](warden-resolver.md)。 |
||||
|
||||
有了初始化`Client`的代码，我们的`Dao`对象即可进行初始化和使用，以下以直接import服务端api包为例： |
||||
|
||||
```go |
||||
package dao |
||||
|
||||
import( |
||||
demoapi "kratos-demo/api" |
||||
grpcempty "github.com/golang/protobuf/ptypes/empty" |
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden" |
||||
|
||||
"github.com/pkg/errors" |
||||
) |
||||
|
||||
type Dao struct{ |
||||
demoClient demoapi.DemoClient |
||||
} |
||||
|
||||
// New account dao. |
||||
func New() (d *Dao) { |
||||
cfg := &warden.ClientConfig{} |
||||
paladin.Get("grpc.toml").UnmarshalTOML(cfg) |
||||
d = &Dao{} |
||||
var err error |
||||
if d.demoClient, err = demoapi.NewClient(cfg); err != nil { |
||||
panic(err) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// SayHello say hello. |
||||
func (d *Dao) SayHello(c context.Context, req *demoapi.HelloReq) (resp *grpcempty.Empty, err error) { |
||||
if resp, err = d.demoClient.SayHello(c, req); err != nil { |
||||
err = errors.Wrapf(err, "%v", arg) |
||||
} |
||||
return |
||||
} |
||||
``` |
||||
|
||||
如此在`internal/service`层就可以进行资源的方法调用。 |
||||
|
||||
# 扩展阅读 |
||||
|
||||
[warden拦截器](warden-mid.md) |
||||
[warden基于pb生成](warden-pb.md) |
||||
[warden服务发现](warden-resolver.md) |
||||
[warden负载均衡](warden-balancer.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,204 @@ |
||||
# 前提 |
||||
|
||||
服务注册与发现最简单的就是`direct`固定服务端地址的直连方式。也就是服务端正常监听端口启动不进行额外操作，客户端使用如下`target`： |
||||
|
||||
```url |
||||
direct://default/127.0.0.1:9000,127.0.0.1:9091 |
||||
``` |
||||
|
||||
> `target`就是标准的`URL`资源定位符[查看WIKI](https://zh.wikipedia.org/wiki/%E7%BB%9F%E4%B8%80%E8%B5%84%E6%BA%90%E5%AE%9A%E4%BD%8D%E7%AC%A6) |
||||
|
||||
其中`direct`为协议类型，此处表示直接使用该`URL`内提供的地址`127.0.0.1:9000,127.0.0.1:9091`进行连接，而`default`在此处无意义仅当做占位符。 |
||||
|
||||
# gRPC Resolver |
||||
|
||||
gRPC暴露了服务发现的接口`resolver.Builder`和`resolver.ClientConn`和`resolver.Resolver`，[官方代码位置](https://github.com/grpc/grpc-go/blob/master/resolver/resolver.go)： |
||||
|
||||
```go |
||||
// Builder creates a resolver that will be used to watch name resolution updates. |
||||
type Builder interface { |
||||
// Build creates a new resolver for the given target. |
||||
// |
||||
// gRPC dial calls Build synchronously, and fails if the returned error is |
||||
// not nil. |
||||
Build(target Target, cc ClientConn, opts BuildOption) (Resolver, error) |
||||
// Scheme returns the scheme supported by this resolver. |
||||
// Scheme is defined at https://github.com/grpc/grpc/blob/master/doc/naming.md. |
||||
Scheme() string |
||||
} |
||||
|
||||
// ClientConn contains the callbacks for resolver to notify any updates |
||||
// to the gRPC ClientConn. |
||||
// |
||||
// This interface is to be implemented by gRPC. Users should not need a |
||||
// brand new implementation of this interface. For the situations like |
||||
// testing, the new implementation should embed this interface. This allows |
||||
// gRPC to add new methods to this interface. |
||||
type ClientConn interface { |
||||
// UpdateState updates the state of the ClientConn appropriately. |
||||
UpdateState(State) |
||||
// NewAddress is called by resolver to notify ClientConn a new list |
||||
// of resolved addresses. |
||||
// The address list should be the complete list of resolved addresses. |
||||
// |
||||
// Deprecated: Use UpdateState instead. |
||||
NewAddress(addresses []Address) |
||||
// NewServiceConfig is called by resolver to notify ClientConn a new |
||||
// service config. The service config should be provided as a json string. |
||||
// |
||||
// Deprecated: Use UpdateState instead. |
||||
NewServiceConfig(serviceConfig string) |
||||
} |
||||
|
||||
// Resolver watches for the updates on the specified target. |
||||
// Updates include address updates and service config updates. |
||||
type Resolver interface { |
||||
// ResolveNow will be called by gRPC to try to resolve the target name |
||||
// again. It's just a hint, resolver can ignore this if it's not necessary. |
||||
// |
||||
// It could be called multiple times concurrently. |
||||
ResolveNow(ResolveNowOption) |
||||
// Close closes the resolver. |
||||
Close() |
||||
} |
||||
``` |
||||
|
||||
下面依次分析这三个接口的作用： |
||||
|
||||
* `Builder`用于gRPC内部创建`Resolver`接口的实现，但注意声明的`Build`方法将接口`ClientConn`作为参数传入了 |
||||
* `ClientConn`接口有两个废弃方法不用管，看`UpdateState`方法需要传入`State`结构，看代码可以发现其中包含了`Addresses []Address // Resolved addresses for the target`，可以看出是需要将服务发现得到的`Address`对象列表告诉`ClientConn`的对象 |
||||
* `Resolver`提供了`ResolveNow`用于被gRPC尝试重新进行服务发现 |
||||
|
||||
看完这三个接口就可以明白gRPC的服务发现实现逻辑，通过`Builder`进行`Reslover`的创建，在`Build`的过程中将服务发现的地址信息丢给`ClientConn`用于内部连接创建等逻辑。主要逻辑可以按下面顺序来看源码理解： |
||||
|
||||
* 当`client`在`Dial`时会根据`target`解析的`scheme`获取对应的`Builder`，[官方代码位置](https://github.com/grpc/grpc-go/blob/master/clientconn.go#L242) |
||||
* 当`Dial`成功会创建出结构体`ClientConn`的对象[官方代码位置](https://github.com/grpc/grpc-go/blob/master/clientconn.go#L447)(注意不是上面的`ClientConn`接口)，可以看到结构体`ClientConn`内的成员`resolverWrapper`又实现了接口`ClientConn`的方法[官方代码位置](https://github.com/grpc/grpc-go/blob/master/resolver_conn_wrapper.go) |
||||
* 当`resolverWrapper`被初始化时就会调用`Build`方法[官方代码位置](https://github.com/grpc/grpc-go/blob/master/resolver_conn_wrapper.go#L89)，其中参数为接口`ClientConn`传入的是`ccResolverWrapper` |
||||
* 当用户基于`Builder`的实现进行`UpdateState`调用时，则会触发结构体`ClientConn`的`updateResolverState`方法[官方代码位置](https://github.com/grpc/grpc-go/blob/master/resolver_conn_wrapper.go#L109)，`updateResolverState`则会对传入的`Address`进行初始化等逻辑[官方代码位置](https://github.com/grpc/grpc-go/blob/master/clientconn.go#L553) |
||||
|
||||
如此整个服务发现过程就结束了。从中也可以看出gRPC官方提供的三个接口还是很灵活的，但也正因为灵活要实现稍微麻烦一些，而`Address`[官方代码位置](https://github.com/grpc/grpc-go/blob/master/resolver/resolver.go#L79)如果直接被业务拿来用于服务节点信息的描述结构则显得有些过于简单。 |
||||
|
||||
所以`warden`包装了gRPC的整个服务发现实现逻辑，代码分别位于`pkg/naming/naming.go`和`warden/resolver/resolver.go`，其中： |
||||
|
||||
* `naming.go`内定义了用于描述业务实例的`Instance`结构、用于服务注册的`Registry`接口、用于服务发现的`Resolver`接口 |
||||
* `resolver.go`内实现了gRPC官方的`resolver.Builder`和`resolver.Resolver`接口，但也暴露了`naming.go`内的`naming.Builder`和`naming.Resolver`接口 |
||||
|
||||
# warden Resolver |
||||
|
||||
接下来看`naming`内的接口如下： |
||||
|
||||
```go |
||||
// Resolver resolve naming service |
||||
type Resolver interface { |
||||
Fetch(context.Context) (*InstancesInfo, bool) |
||||
Watch() <-chan struct{} |
||||
Close() error |
||||
} |
||||
|
||||
// Builder resolver builder. |
||||
type Builder interface { |
||||
Build(id string) Resolver |
||||
Scheme() string |
||||
} |
||||
``` |
||||
|
||||
可以看到封装方式与gRPC官方的方法一样，通过`Builder`进行`Resolver`的初始化。不同的是通过封装将参数进行了简化： |
||||
|
||||
* `Build`只需要传对应的服务`id`即可：`warden/resolver/resolver.go`在gRPC进行调用后，会根据`Scheme`方法查询对应的`naming.Builder`实现并调用`Build`将`id`传入，而`naming.Resolver`的实现即可通过`id`去对应的服务发现中间件进行实例信息的查询 |
||||
* 而`Resolver`则对方法进行了扩展，除了简单进行`Fetch`操作外还多了`Watch`方法，用于监听服务发现中间件的节点变化情况，从而能够实时的进行服务实例信息的更新 |
||||
|
||||
在`naming/discovery`内实现了基于[discovery](https://github.com/bilibili/discovery)为中间件的服务注册与发现逻辑。如果要实现其他中间件如`etcd`|`zookeeper`等的逻辑，参考`naming/discovery/discovery.go`内的逻辑，将与`discovery`的交互逻辑替换掉即可（后续会默认将etcd/zk等实现，敬请期待）。 |
||||
|
||||
# 使用discovery |
||||
|
||||
因为`warden`内默认使用`direct`的方式，所以要使用[discovery](https://github.com/bilibili/discovery)需要在业务的`NewClient`前进行注册，代码如下： |
||||
|
||||
```go |
||||
package dao |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"github.com/bilibili/kratos/pkg/naming/discovery" |
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden" |
||||
"github.com/bilibili/kratos/pkg/net/rpc/warden/resolver" |
||||
|
||||
"google.golang.org/grpc" |
||||
) |
||||
|
||||
// AppID your appid, ensure unique. |
||||
const AppID = "demo.service" // NOTE: example |
||||
|
||||
func init(){ |
||||
// NOTE: 注意这段代码，表示要使用discovery进行服务发现 |
||||
// NOTE: 还需注意的是，resolver.Register是全局生效的，所以建议该代码放在进程初始化的时候执行 |
||||
// NOTE: ！！！切记不要在一个进程内进行多个不同中间件的Register！！！ |
||||
// NOTE: 在启动应用时，可以通过flag(-discovery.nodes) 或者 环境配置(DISCOVERY_NODES)指定discovery节点 |
||||
resolver.Register(discovery.Builder()) |
||||
} |
||||
|
||||
// NewClient new member grpc client |
||||
func NewClient(cfg *warden.ClientConfig, opts ...grpc.DialOption) (DemoClient, error) { |
||||
client := warden.NewClient(cfg, opts...) |
||||
conn, err := client.Dial(context.Background(), "discovery://default/"+AppID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
// 注意替换这里： |
||||
// NewDemoClient方法是在"api"目录下代码生成的 |
||||
// 对应proto文件内自定义的service名字，请使用正确方法名替换 |
||||
return NewDemoClient(conn), nil |
||||
} |
||||
``` |
||||
|
||||
> 注意：`resolver.Register`是全局行为，建议放在包加载阶段或main方法开始时执行，该方法执行后会在gRPC内注册构造方法 |
||||
|
||||
`target`是`discovery://default/${appid}`，当gRPC内进行解析后会得到`scheme`=`discovery`和`appid`，然后进行以下逻辑： |
||||
|
||||
1. `warden/resolver.Builder`会通过`scheme`获取到`naming/discovery.Builder`对象（靠`resolver.Register`注册过的） |
||||
2. 拿到`naming/discovery.Builder`后执行`Build(appid)`构造`naming/discovery.Discovery` |
||||
3. `naming/discovery.Discovery`对象基于`appid`就知道要获取哪个服务的实例信息 |
||||
|
||||
# 服务注册 |
||||
|
||||
客户端既然使用了[discovery](https://github.com/bilibili/discovery)进行服务发现，也就意味着服务端启动后必须将自己注册给[discovery](https://github.com/bilibili/discovery)知道。 |
||||
|
||||
相对服务发现来讲，服务注册则简单很多，看`naming/discovery/discovery.go`内的代码实现了`naming/naming.go`内的`Registry`接口，服务端启动时可以参考下面代码进行注册： |
||||
|
||||
```go |
||||
// 该代码可放在main.go，当warden server进行初始化之后 |
||||
// 省略... |
||||
|
||||
ip := "" // NOTE: 必须拿到您实例节点的真实IP， |
||||
port := "" // NOTE: 必须拿到您实例grpc监听的真实端口，warden默认监听9000 |
||||
hn, _ := os.Hostname() |
||||
dis := discovery.New(nil) |
||||
ins := &naming.Instance{ |
||||
Zone: env.Zone, |
||||
Env: env.DeployEnv, |
||||
AppID: "your app id", |
||||
Hostname: hn, |
||||
Addrs: []string{ |
||||
"grpc://" + ip + ":" + port, |
||||
}, |
||||
} |
||||
cancel, err := dis.Register(context.Background(), ins) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
|
||||
// 省略... |
||||
|
||||
// 特别注意！！！ |
||||
// cancel必须在进程退出时执行！！！ |
||||
cancel() |
||||
|
||||
``` |
||||
|
||||
# 扩展阅读 |
||||
|
||||
[warden快速开始](warden-quickstart.md) [warden拦截器](warden-mid.md) [warden基于pb生成](warden-pb.md) [warden负载均衡](warden-balancer.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,40 @@ |
||||
# 背景 |
||||
|
||||
我们需要统一的rpc服务，经过选型讨论决定直接使用成熟的跨语言的gRPC。 |
||||
|
||||
# 概览 |
||||
|
||||
* 不改gRPC源码，基于接口进行包装集成trace、log、prom等组件 |
||||
* 打通自有服务注册发现系统[discovery](https://github.com/bilibili/discovery) |
||||
* 实现更平滑可靠的负载均衡算法 |
||||
|
||||
# 拦截器 |
||||
|
||||
gRPC暴露了两个拦截器接口，分别是： |
||||
|
||||
* `grpc.UnaryServerInterceptor`服务端拦截器 |
||||
* `grpc.UnaryClientInterceptor`客户端拦截器 |
||||
|
||||
基于两个拦截器可以针对性的定制公共模块的封装代码，比如`warden/logging.go`是通用日志逻辑。 |
||||
|
||||
[warden拦截器](warden-mid.md) |
||||
|
||||
# 服务发现 |
||||
|
||||
`warden`默认使用`direct`方式直连，正常线上都会使用第三方服务注册与发现中间件，`warden`内包含了[discovery](https://github.com/bilibili/discovery)的逻辑实现，想使用如`etcd`、`zookeeper`等也可以，都请看下面文档。 |
||||
|
||||
[warden服务发现](warden-resolver.md) |
||||
|
||||
# 负载均衡 |
||||
|
||||
实现了`wrr`和`p2c`两种算法，默认使用`p2c`。 |
||||
|
||||
[warden负载均衡](warden-balancer.md) |
||||
|
||||
# 扩展阅读 |
||||
|
||||
[warden快速开始](warden-quickstart.md) [warden拦截器](warden-mid.md) [warden负载均衡](warden-balancer.md) [warden基于pb生成](warden-pb.md) [warden服务发现](warden-resolver.md) |
||||
|
||||
------------- |
||||
|
||||
[文档目录树](summary.md) |
||||
@ -0,0 +1,106 @@ |
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= |
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= |
||||
github.com/aristanetworks/goarista v0.0.0-20190409234242-46f4bc7b73ef h1:ajsnF5qTstiBlP+V/mgh91zZfoKP477KfSmRoCoyYGU= |
||||
github.com/aristanetworks/goarista v0.0.0-20190409234242-46f4bc7b73ef/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= |
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= |
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= |
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= |
||||
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= |
||||
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= |
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= |
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= |
||||
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM= |
||||
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= |
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= |
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||
github.com/dgryski/go-farm v0.0.0-20190323231341-8198c7b169ec h1:sElGDs3V8VdCxH5tWi0ycWJzteOPLJ3HtItSSKI95PY= |
||||
github.com/dgryski/go-farm v0.0.0-20190323231341-8198c7b169ec/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= |
||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= |
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= |
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= |
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= |
||||
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= |
||||
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= |
||||
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= |
||||
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= |
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= |
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= |
||||
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= |
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= |
||||
github.com/golang/appengine v1.1.0 h1:i5Me8ymxZ0UcvrKKqVyk7iemYgabJFc19gXizKBaTa0= |
||||
github.com/golang/appengine v1.1.0/go.mod h1:C7k13PpLjU5SHb7WgwfAwCmVFgi4cpy3kl2zlpoBck8= |
||||
github.com/golang/crypto v0.0.0-20190123085648-057139ce5d2b h1:9dkUhGlF9C+jJBMDKqq91ycBLyQMvFjTdBhOqchi7lU= |
||||
github.com/golang/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:uZvAcrsnNaCxlh1HorK5dUQHGmEKPh2H/Rl1kehswPo= |
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= |
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= |
||||
github.com/golang/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= |
||||
github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= |
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= |
||||
github.com/golang/net v0.0.0-20190311183353-d8887717615a h1:4V+LPwzBFLRg7XSXZw133Jsur1mTVMY73hIv/FTdrbg= |
||||
github.com/golang/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
||||
github.com/golang/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:ovBFgdmJqyggKzXS0i5+osE+RsPEbEsUfp2sVCgys1Q= |
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= |
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
||||
github.com/golang/sync v0.0.0-20181108010431-42b317875d0f h1:vuwODIDRvDgwjIl6VTMf0c1Z9uVMUUxiu6UPUjiGhD4= |
||||
github.com/golang/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:YCHYtYb9c8Q7XgYVYjmJBPtFPKx5QvOcPxHZWjldabE= |
||||
github.com/golang/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:GJexUf2QgFNvMR9sjJ1iqs+2TxZqJko+Muhnu04tPuU= |
||||
github.com/golang/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:5JyrLPvD/ZdaYkT7IqKhsP5xt7aLjA99KXRtk4EIYDk= |
||||
github.com/golang/text v0.3.0 h1:uI5zIUA9cg047ctlTptnVc0Ghjfurf2eZMFrod8R7v8= |
||||
github.com/golang/text v0.3.0/go.mod h1:GUiq9pdJKRKKAZXiVgWFEvocYuREvC14NhI4OPgEjeE= |
||||
github.com/golang/time v0.0.0-20190308202827-9d24e82272b4 h1:F9e5QAps6/3zc8881JhdfJBCj+KjFaahs4YNEzAPc/Q= |
||||
github.com/golang/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= |
||||
github.com/golang/tools v0.0.0-20190328211700-ab21143f2384 h1:8J6Yq2enLsHiOXruypwvT3wf8eAvi7wRmS5KCt7RbHo= |
||||
github.com/golang/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
||||
github.com/google/go-genproto v0.0.0-20180817151627-c66870c02cf8 h1:I9PuChzQA31gMw88WmVPJaAwE0nZNHpMrLDUnTyzFAI= |
||||
github.com/google/go-genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:3Rcd9jSoLVkV/osPrt5CogLvLiarfI8U9/x78NwhuDU= |
||||
github.com/googleapis/google-cloud-go v0.26.0/go.mod h1:yJoOdPPE9UpqbamBhJvp7Ur6OUPPV4rUY3RnssPGNBA= |
||||
github.com/grpc/grpc-go v1.20.1 h1:pk72GtSPpOdZDTkPneppDMGW10HYPC7RqNJT/JvUpV0= |
||||
github.com/grpc/grpc-go v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= |
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= |
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= |
||||
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= |
||||
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= |
||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= |
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= |
||||
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= |
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= |
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= |
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= |
||||
github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= |
||||
github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= |
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= |
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= |
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= |
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= |
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= |
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= |
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= |
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= |
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= |
||||
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= |
||||
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= |
||||
github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec h1:6ncX5ko6B9LntYM0YBRXkiSaZMmLYeZ/NWcmeB43mMY= |
||||
github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= |
||||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= |
||||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= |
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= |
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= |
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= |
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= |
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= |
||||
github.com/tsuna/gohbase v0.0.0-20190201102810-d3184c1526df h1:jYiwqXfoRWU6pJMzClEpLn9Jofi3U/8qS+w3iRNJ/hw= |
||||
github.com/tsuna/gohbase v0.0.0-20190201102810-d3184c1526df/go.mod h1:3HfLQly3YNLGxNv/2YOfmz30vcjG9hbuME1GpxoLlGs= |
||||
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= |
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= |
||||
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= |
||||
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= |
||||
gopkg.in/go-playground/validator.v9 v9.26.0 h1:2NPPsBpD0ZoxshmLWewQru8rWmbT5JqSzz9D1ZrAjYQ= |
||||
gopkg.in/go-playground/validator.v9 v9.26.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= |
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= |
||||
@ -1,25 +0,0 @@ |
||||
# cache/memcache |
||||
|
||||
##### 项目简介 |
||||
1. 提供protobuf，gob，json序列化方式，gzip的memcache接口 |
||||
|
||||
#### 使用方式 |
||||
```golang |
||||
// 初始化 注意这里只是示例 展示用法 不能每次都New 只需要初始化一次 |
||||
mc := memcache.New(&memcache.Config{}) |
||||
// 程序关闭的时候调用close方法 |
||||
defer mc.Close() |
||||
// 增加 key |
||||
err = mc.Set(c, &memcache.Item{}) |
||||
// 删除key |
||||
err := mc.Delete(c,key) |
||||
// 获得某个key的内容 |
||||
err := mc.Get(c,key).Scan(&v) |
||||
// 获取多个key的内容 |
||||
replies, err := mc.GetMulti(c, keys) |
||||
for _, key := range replies.Keys() { |
||||
if err = replies.Scan(key, &v); err != nil { |
||||
return |
||||
} |
||||
} |
||||
``` |
||||
@ -0,0 +1,261 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"context" |
||||
"fmt" |
||||
"io" |
||||
"net" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
pkgerr "github.com/pkg/errors" |
||||
) |
||||
|
||||
var ( |
||||
crlf = []byte("\r\n") |
||||
space = []byte(" ") |
||||
replyOK = []byte("OK\r\n") |
||||
replyStored = []byte("STORED\r\n") |
||||
replyNotStored = []byte("NOT_STORED\r\n") |
||||
replyExists = []byte("EXISTS\r\n") |
||||
replyNotFound = []byte("NOT_FOUND\r\n") |
||||
replyDeleted = []byte("DELETED\r\n") |
||||
replyEnd = []byte("END\r\n") |
||||
replyTouched = []byte("TOUCHED\r\n") |
||||
replyClientErrorPrefix = []byte("CLIENT_ERROR ") |
||||
replyServerErrorPrefix = []byte("SERVER_ERROR ") |
||||
) |
||||
|
||||
var _ protocolConn = &asiiConn{} |
||||
|
||||
// asiiConn is the low-level implementation of Conn
|
||||
type asiiConn struct { |
||||
err error |
||||
conn net.Conn |
||||
// Read & Write
|
||||
readTimeout time.Duration |
||||
writeTimeout time.Duration |
||||
rw *bufio.ReadWriter |
||||
} |
||||
|
||||
func replyToError(line []byte) error { |
||||
switch { |
||||
case bytes.Equal(line, replyStored): |
||||
return nil |
||||
case bytes.Equal(line, replyOK): |
||||
return nil |
||||
case bytes.Equal(line, replyDeleted): |
||||
return nil |
||||
case bytes.Equal(line, replyTouched): |
||||
return nil |
||||
case bytes.Equal(line, replyNotStored): |
||||
return ErrNotStored |
||||
case bytes.Equal(line, replyExists): |
||||
return ErrCASConflict |
||||
case bytes.Equal(line, replyNotFound): |
||||
return ErrNotFound |
||||
case bytes.Equal(line, replyNotStored): |
||||
return ErrNotStored |
||||
case bytes.Equal(line, replyExists): |
||||
return ErrCASConflict |
||||
} |
||||
return pkgerr.WithStack(protocolError(string(line))) |
||||
} |
||||
|
||||
func (c *asiiConn) Populate(ctx context.Context, cmd string, key string, flags uint32, expiration int32, cas uint64, data []byte) error { |
||||
c.conn.SetWriteDeadline(shrinkDeadline(ctx, c.writeTimeout)) |
||||
// <command name> <key> <flags> <exptime> <bytes> [noreply]\r\n
|
||||
var err error |
||||
if cmd == "cas" { |
||||
_, err = fmt.Fprintf(c.rw, "%s %s %d %d %d %d\r\n", cmd, key, flags, expiration, len(data), cas) |
||||
} else { |
||||
_, err = fmt.Fprintf(c.rw, "%s %s %d %d %d\r\n", cmd, key, flags, expiration, len(data)) |
||||
} |
||||
if err != nil { |
||||
return c.fatal(err) |
||||
} |
||||
c.rw.Write(data) |
||||
c.rw.Write(crlf) |
||||
if err = c.rw.Flush(); err != nil { |
||||
return c.fatal(err) |
||||
} |
||||
c.conn.SetReadDeadline(shrinkDeadline(ctx, c.readTimeout)) |
||||
line, err := c.rw.ReadSlice('\n') |
||||
if err != nil { |
||||
return c.fatal(err) |
||||
} |
||||
return replyToError(line) |
||||
} |
||||
|
||||
// newConn returns a new memcache connection for the given net connection.
|
||||
func newASCIIConn(netConn net.Conn, readTimeout, writeTimeout time.Duration) (protocolConn, error) { |
||||
if writeTimeout <= 0 || readTimeout <= 0 { |
||||
return nil, pkgerr.Errorf("readTimeout writeTimeout can't be zero") |
||||
} |
||||
c := &asiiConn{ |
||||
conn: netConn, |
||||
rw: bufio.NewReadWriter(bufio.NewReader(netConn), |
||||
bufio.NewWriter(netConn)), |
||||
readTimeout: readTimeout, |
||||
writeTimeout: writeTimeout, |
||||
} |
||||
return c, nil |
||||
} |
||||
|
||||
func (c *asiiConn) Close() error { |
||||
if c.err == nil { |
||||
c.err = pkgerr.New("memcache: closed") |
||||
} |
||||
return c.conn.Close() |
||||
} |
||||
|
||||
func (c *asiiConn) fatal(err error) error { |
||||
if c.err == nil { |
||||
c.err = pkgerr.WithStack(err) |
||||
// Close connection to force errors on subsequent calls and to unblock
|
||||
// other reader or writer.
|
||||
c.conn.Close() |
||||
} |
||||
return c.err |
||||
} |
||||
|
||||
func (c *asiiConn) Err() error { |
||||
return c.err |
||||
} |
||||
|
||||
func (c *asiiConn) Get(ctx context.Context, key string) (result *Item, err error) { |
||||
c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout)) |
||||
if _, err = fmt.Fprintf(c.rw, "gets %s\r\n", key); err != nil { |
||||
return nil, c.fatal(err) |
||||
} |
||||
if err = c.rw.Flush(); err != nil { |
||||
return nil, c.fatal(err) |
||||
} |
||||
if err = c.parseGetReply(func(it *Item) { |
||||
result = it |
||||
}); err != nil { |
||||
return |
||||
} |
||||
if result == nil { |
||||
return nil, ErrNotFound |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (c *asiiConn) GetMulti(ctx context.Context, keys ...string) (map[string]*Item, error) { |
||||
var err error |
||||
c.conn.SetWriteDeadline(shrinkDeadline(ctx, c.writeTimeout)) |
||||
if _, err = fmt.Fprintf(c.rw, "gets %s\r\n", strings.Join(keys, " ")); err != nil { |
||||
return nil, c.fatal(err) |
||||
} |
||||
if err = c.rw.Flush(); err != nil { |
||||
return nil, c.fatal(err) |
||||
} |
||||
results := make(map[string]*Item, len(keys)) |
||||
if err = c.parseGetReply(func(it *Item) { |
||||
results[it.Key] = it |
||||
}); err != nil { |
||||
return nil, err |
||||
} |
||||
return results, nil |
||||
} |
||||
|
||||
func (c *asiiConn) parseGetReply(f func(*Item)) error { |
||||
c.conn.SetReadDeadline(shrinkDeadline(context.TODO(), c.readTimeout)) |
||||
for { |
||||
line, err := c.rw.ReadSlice('\n') |
||||
if err != nil { |
||||
return c.fatal(err) |
||||
} |
||||
if bytes.Equal(line, replyEnd) { |
||||
return nil |
||||
} |
||||
if bytes.HasPrefix(line, replyServerErrorPrefix) { |
||||
errMsg := line[len(replyServerErrorPrefix):] |
||||
return c.fatal(protocolError(errMsg)) |
||||
} |
||||
it := new(Item) |
||||
size, err := scanGetReply(line, it) |
||||
if err != nil { |
||||
return c.fatal(err) |
||||
} |
||||
it.Value = make([]byte, size+2) |
||||
if _, err = io.ReadFull(c.rw, it.Value); err != nil { |
||||
return c.fatal(err) |
||||
} |
||||
if !bytes.HasSuffix(it.Value, crlf) { |
||||
return c.fatal(protocolError("corrupt get reply, no except CRLF")) |
||||
} |
||||
it.Value = it.Value[:size] |
||||
f(it) |
||||
} |
||||
} |
||||
|
||||
func scanGetReply(line []byte, item *Item) (size int, err error) { |
||||
pattern := "VALUE %s %d %d %d\r\n" |
||||
dest := []interface{}{&item.Key, &item.Flags, &size, &item.cas} |
||||
if bytes.Count(line, space) == 3 { |
||||
pattern = "VALUE %s %d %d\r\n" |
||||
dest = dest[:3] |
||||
} |
||||
n, err := fmt.Sscanf(string(line), pattern, dest...) |
||||
if err != nil || n != len(dest) { |
||||
return -1, fmt.Errorf("memcache: unexpected line in get response: %q", line) |
||||
} |
||||
return size, nil |
||||
} |
||||
|
||||
func (c *asiiConn) Touch(ctx context.Context, key string, expire int32) error { |
||||
line, err := c.writeReadLine("touch %s %d\r\n", key, expire) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return replyToError(line) |
||||
} |
||||
|
||||
func (c *asiiConn) IncrDecr(ctx context.Context, cmd, key string, delta uint64) (uint64, error) { |
||||
line, err := c.writeReadLine("%s %s %d\r\n", cmd, key, delta) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
switch { |
||||
case bytes.Equal(line, replyNotFound): |
||||
return 0, ErrNotFound |
||||
case bytes.HasPrefix(line, replyClientErrorPrefix): |
||||
errMsg := line[len(replyClientErrorPrefix):] |
||||
return 0, pkgerr.WithStack(protocolError(errMsg)) |
||||
} |
||||
val, err := strconv.ParseUint(string(line[:len(line)-2]), 10, 64) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return val, nil |
||||
} |
||||
|
||||
func (c *asiiConn) Delete(ctx context.Context, key string) error { |
||||
line, err := c.writeReadLine("delete %s\r\n", key) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return replyToError(line) |
||||
} |
||||
|
||||
func (c *asiiConn) writeReadLine(format string, args ...interface{}) ([]byte, error) { |
||||
c.conn.SetWriteDeadline(shrinkDeadline(context.TODO(), c.writeTimeout)) |
||||
_, err := fmt.Fprintf(c.rw, format, args...) |
||||
if err != nil { |
||||
return nil, c.fatal(pkgerr.WithStack(err)) |
||||
} |
||||
if err = c.rw.Flush(); err != nil { |
||||
return nil, c.fatal(pkgerr.WithStack(err)) |
||||
} |
||||
c.conn.SetReadDeadline(shrinkDeadline(context.TODO(), c.readTimeout)) |
||||
line, err := c.rw.ReadSlice('\n') |
||||
if err != nil { |
||||
return line, c.fatal(pkgerr.WithStack(err)) |
||||
} |
||||
return line, nil |
||||
} |
||||
@ -0,0 +1,569 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"bytes" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
"testing" |
||||
) |
||||
|
||||
func TestASCIIConnAdd(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
e error |
||||
}{ |
||||
{ |
||||
"Add", |
||||
&Item{ |
||||
Key: "test_add", |
||||
Value: []byte("0"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"Add_Large", |
||||
&Item{ |
||||
Key: "test_add_large", |
||||
Value: bytes.Repeat(space, _largeValue+1), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"Add_Exist", |
||||
&Item{ |
||||
Key: "test_add", |
||||
Value: []byte("0"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
ErrNotStored, |
||||
}, |
||||
} |
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if err := testConnASCII.Add(test.a); err != test.e { |
||||
t.Fatal(err) |
||||
} |
||||
if b, err := testConnASCII.Get(test.a.Key); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
compareItem(t, test.a, b) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestASCIIConnGet(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
k string |
||||
e error |
||||
}{ |
||||
{ |
||||
"Get", |
||||
&Item{ |
||||
Key: "test_get", |
||||
Value: []byte("0"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
"test_get", |
||||
nil, |
||||
}, |
||||
{ |
||||
"Get_NotExist", |
||||
&Item{ |
||||
Key: "test_get_not_exist", |
||||
Value: []byte("0"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
"test_get_not_exist!", |
||||
ErrNotFound, |
||||
}, |
||||
} |
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if err := testConnASCII.Add(test.a); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if b, err := testConnASCII.Get(test.a.Key); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
compareItem(t, test.a, b) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
//func TestGetHasErr(t *testing.T) {
|
||||
// prepareEnv(t)
|
||||
//
|
||||
// st := &TestItem{Name: "json", Age: 10}
|
||||
// itemx := &Item{Key: "test", Object: st, Flags: FlagJSON}
|
||||
// c.Set(itemx)
|
||||
//
|
||||
// expected := errors.New("some error")
|
||||
// monkey.Patch(scanGetReply, func(line []byte, item *Item) (size int, err error) {
|
||||
// return 0, expected
|
||||
// })
|
||||
//
|
||||
// if _, err := c.Get("test"); err.Error() != expected.Error() {
|
||||
// t.Errorf("conn.Get() unexpected error(%v)", err)
|
||||
// }
|
||||
// if err := c.(*asciiConn).err; err.Error() != expected.Error() {
|
||||
// t.Errorf("unexpected error(%v)", err)
|
||||
// }
|
||||
//}
|
||||
|
||||
func TestASCIIConnGetMulti(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
a []*Item |
||||
k []string |
||||
e error |
||||
}{ |
||||
{"getMulti_Add", |
||||
[]*Item{ |
||||
{ |
||||
Key: "get_multi_1", |
||||
Value: []byte("test"), |
||||
Flags: FlagRAW, |
||||
Expiration: 60, |
||||
cas: 0, |
||||
}, |
||||
{ |
||||
Key: "get_multi_2", |
||||
Value: []byte("test2"), |
||||
Flags: FlagRAW, |
||||
Expiration: 60, |
||||
cas: 0, |
||||
}, |
||||
}, |
||||
[]string{"get_multi_1", "get_multi_2"}, |
||||
nil, |
||||
}, |
||||
} |
||||
|
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
for _, i := range test.a { |
||||
if err := testConnASCII.Set(i); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
if r, err := testConnASCII.GetMulti(test.k); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
reply := r["get_multi_1"] |
||||
compareItem(t, reply, test.a[0]) |
||||
reply = r["get_multi_2"] |
||||
compareItem(t, reply, test.a[1]) |
||||
} |
||||
|
||||
}) |
||||
} |
||||
|
||||
} |
||||
|
||||
func TestASCIIConnSet(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
e error |
||||
}{ |
||||
{ |
||||
"SetLowerBound", |
||||
&Item{ |
||||
Key: strings.Repeat("a", 1), |
||||
Value: []byte("4"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"SetUpperBound", |
||||
&Item{ |
||||
Key: strings.Repeat("a", 250), |
||||
Value: []byte("3"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"SetIllegalKeyZeroLength", |
||||
&Item{ |
||||
Key: "", |
||||
Value: []byte("2"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
ErrMalformedKey, |
||||
}, |
||||
{ |
||||
"SetIllegalKeyLengthExceededLimit", |
||||
&Item{ |
||||
Key: " ", |
||||
Value: []byte("1"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
ErrMalformedKey, |
||||
}, |
||||
{ |
||||
"SeJsonItem", |
||||
&Item{ |
||||
Key: "set_obj", |
||||
Object: &struct { |
||||
Name string |
||||
Age int |
||||
}{"json", 10}, |
||||
Expiration: 60, |
||||
Flags: FlagJSON, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"SeErrItemJSONGzip", |
||||
&Item{ |
||||
Key: "set_err_item", |
||||
Expiration: 60, |
||||
Flags: FlagJSON | FlagGzip, |
||||
}, |
||||
ErrItem, |
||||
}, |
||||
{ |
||||
"SeErrItemBytesValueWrongFlag", |
||||
&Item{ |
||||
Key: "set_err_item", |
||||
Value: []byte("2"), |
||||
Expiration: 60, |
||||
Flags: FlagJSON, |
||||
}, |
||||
ErrItem, |
||||
}, |
||||
} |
||||
|
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if err := testConnASCII.Set(test.a); err != test.e { |
||||
t.Fatal(err) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestASCIIConnCompareAndSwap(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
b *Item |
||||
c *Item |
||||
k string |
||||
e error |
||||
}{ |
||||
{ |
||||
"CompareAndSwap", |
||||
&Item{ |
||||
Key: "test_cas", |
||||
Value: []byte("2"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
&Item{ |
||||
Key: "test_cas", |
||||
Value: []byte("3"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
"test_cas", |
||||
nil, |
||||
}, |
||||
{ |
||||
"CompareAndSwapErrCASConflict", |
||||
&Item{ |
||||
Key: "test_cas_conflict", |
||||
Value: []byte("2"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
&Item{ |
||||
Key: "test_cas_conflict", |
||||
Value: []byte("1"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
&Item{ |
||||
Key: "test_cas_conflict", |
||||
Value: []byte("3"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
"test_cas_conflict", |
||||
ErrCASConflict, |
||||
}, |
||||
} |
||||
|
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if err := testConnASCII.Set(test.a); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
r, err := testConnASCII.Get(test.k) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if test.b != nil { |
||||
if err := testConnASCII.Set(test.b); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
r.Value = test.c.Value |
||||
if err := testConnASCII.CompareAndSwap(r); err != nil { |
||||
if err != test.e { |
||||
t.Fatal(err) |
||||
} |
||||
} else { |
||||
if fr, err := testConnASCII.Get(test.k); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
compareItem(t, fr, test.c) |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
|
||||
t.Run("TestCompareAndSwapErrNotFound", func(t *testing.T) { |
||||
ti := &Item{ |
||||
Key: "test_cas_notfound", |
||||
Value: []byte("2"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
} |
||||
if err := testConnASCII.Set(ti); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
r, err := testConnASCII.Get(ti.Key) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
r.Key = "test_cas_notfound_boom" |
||||
r.Value = []byte("3") |
||||
if err := testConnASCII.CompareAndSwap(r); err != nil { |
||||
if err != ErrNotFound { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
|
||||
func TestASCIIConnReplace(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
b *Item |
||||
e error |
||||
}{ |
||||
{ |
||||
"TestReplace", |
||||
&Item{ |
||||
Key: "test_replace", |
||||
Value: []byte("2"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
&Item{ |
||||
Key: "test_replace", |
||||
Value: []byte("3"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"TestReplaceErrNotStored", |
||||
&Item{ |
||||
Key: "test_replace_not_stored", |
||||
Value: []byte("2"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
&Item{ |
||||
Key: "test_replace_not_stored_boom", |
||||
Value: []byte("3"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
}, |
||||
ErrNotStored, |
||||
}, |
||||
} |
||||
|
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if err := testConnASCII.Set(test.a); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := testConnASCII.Replace(test.b); err != nil { |
||||
if err == test.e { |
||||
return |
||||
} |
||||
t.Fatal(err) |
||||
} |
||||
if r, err := testConnASCII.Get(test.b.Key); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
compareItem(t, r, test.b) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestASCIIConnIncrDecr(t *testing.T) { |
||||
tests := []struct { |
||||
fn func(key string, delta uint64) (uint64, error) |
||||
name string |
||||
k string |
||||
v uint64 |
||||
w uint64 |
||||
}{ |
||||
{ |
||||
testConnASCII.Increment, |
||||
"Incr_10", |
||||
"test_incr", |
||||
10, |
||||
10, |
||||
}, |
||||
{ |
||||
testConnASCII.Increment, |
||||
"Incr_10(2)", |
||||
"test_incr", |
||||
10, |
||||
20, |
||||
}, |
||||
{ |
||||
testConnASCII.Decrement, |
||||
"Decr_10", |
||||
"test_incr", |
||||
10, |
||||
10, |
||||
}, |
||||
} |
||||
if err := testConnASCII.Add(&Item{ |
||||
Key: "test_incr", |
||||
Value: []byte("0"), |
||||
}); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if a, err := test.fn(test.k, test.v); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
if a != test.w { |
||||
t.Fatalf("want %d, got %d", test.w, a) |
||||
} |
||||
} |
||||
if b, err := testConnASCII.Get(test.k); err != nil { |
||||
t.Fatal(err) |
||||
} else { |
||||
if string(b.Value) != strconv.FormatUint(test.w, 10) { |
||||
t.Fatalf("want %s, got %d", b.Value, test.w) |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestASCIIConnTouch(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
k string |
||||
a *Item |
||||
e error |
||||
}{ |
||||
{ |
||||
"Touch", |
||||
"test_touch", |
||||
&Item{ |
||||
Key: "test_touch", |
||||
Value: []byte("0"), |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"Touch_NotExist", |
||||
"test_touch_not_exist", |
||||
nil, |
||||
ErrNotFound, |
||||
}, |
||||
} |
||||
for _, test := range tests { |
||||
if test.a != nil { |
||||
if err := testConnASCII.Add(test.a); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := testConnASCII.Touch(test.k, 1); err != test.e { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestASCIIConnDelete(t *testing.T) { |
||||
tests := []struct { |
||||
name string |
||||
k string |
||||
a *Item |
||||
e error |
||||
}{ |
||||
{ |
||||
"Delete", |
||||
"test_delete", |
||||
&Item{ |
||||
Key: "test_delete", |
||||
Value: []byte("0"), |
||||
Expiration: 60, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"Delete_NotExist", |
||||
"test_delete_not_exist", |
||||
nil, |
||||
ErrNotFound, |
||||
}, |
||||
} |
||||
for _, test := range tests { |
||||
if test.a != nil { |
||||
if err := testConnASCII.Add(test.a); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := testConnASCII.Delete(test.k); err != test.e { |
||||
t.Fatal(err) |
||||
} |
||||
if _, err := testConnASCII.Get(test.k); err != ErrNotFound { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func compareItem(t *testing.T, a, b *Item) { |
||||
if a.Key != b.Key || !bytes.Equal(a.Value, b.Value) || a.Flags != b.Flags { |
||||
t.Fatalf("compareItem: a(%s, %d, %d) : b(%s, %d, %d)", a.Key, len(a.Value), a.Flags, b.Key, len(b.Value), b.Flags) |
||||
} |
||||
} |
||||
@ -1,187 +0,0 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
) |
||||
|
||||
// Memcache memcache client
|
||||
type Memcache struct { |
||||
pool *Pool |
||||
} |
||||
|
||||
// Reply is the result of Get
|
||||
type Reply struct { |
||||
err error |
||||
item *Item |
||||
conn Conn |
||||
closed bool |
||||
} |
||||
|
||||
// Replies is the result of GetMulti
|
||||
type Replies struct { |
||||
err error |
||||
items map[string]*Item |
||||
usedItems map[string]struct{} |
||||
conn Conn |
||||
closed bool |
||||
} |
||||
|
||||
// New get a memcache client
|
||||
func New(c *Config) *Memcache { |
||||
return &Memcache{pool: NewPool(c)} |
||||
} |
||||
|
||||
// Close close connection pool
|
||||
func (mc *Memcache) Close() error { |
||||
return mc.pool.Close() |
||||
} |
||||
|
||||
// Conn direct get a connection
|
||||
func (mc *Memcache) Conn(c context.Context) Conn { |
||||
return mc.pool.Get(c) |
||||
} |
||||
|
||||
// Set writes the given item, unconditionally.
|
||||
func (mc *Memcache) Set(c context.Context, item *Item) (err error) { |
||||
conn := mc.pool.Get(c) |
||||
err = conn.Set(item) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// Add writes the given item, if no value already exists for its key.
|
||||
// ErrNotStored is returned if that condition is not met.
|
||||
func (mc *Memcache) Add(c context.Context, item *Item) (err error) { |
||||
conn := mc.pool.Get(c) |
||||
err = conn.Add(item) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// Replace writes the given item, but only if the server *does* already hold data for this key.
|
||||
func (mc *Memcache) Replace(c context.Context, item *Item) (err error) { |
||||
conn := mc.pool.Get(c) |
||||
err = conn.Replace(item) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// CompareAndSwap writes the given item that was previously returned by Get
|
||||
func (mc *Memcache) CompareAndSwap(c context.Context, item *Item) (err error) { |
||||
conn := mc.pool.Get(c) |
||||
err = conn.CompareAndSwap(item) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// Get sends a command to the server for gets data.
|
||||
func (mc *Memcache) Get(c context.Context, key string) *Reply { |
||||
conn := mc.pool.Get(c) |
||||
item, err := conn.Get(key) |
||||
if err != nil { |
||||
conn.Close() |
||||
} |
||||
return &Reply{err: err, item: item, conn: conn} |
||||
} |
||||
|
||||
// Item get raw Item
|
||||
func (r *Reply) Item() *Item { |
||||
return r.item |
||||
} |
||||
|
||||
// Scan converts value, read from the memcache
|
||||
func (r *Reply) Scan(v interface{}) (err error) { |
||||
if r.err != nil { |
||||
return r.err |
||||
} |
||||
err = r.conn.Scan(r.item, v) |
||||
if !r.closed { |
||||
r.conn.Close() |
||||
r.closed = true |
||||
} |
||||
return |
||||
} |
||||
|
||||
// GetMulti is a batch version of Get
|
||||
func (mc *Memcache) GetMulti(c context.Context, keys []string) (*Replies, error) { |
||||
conn := mc.pool.Get(c) |
||||
items, err := conn.GetMulti(keys) |
||||
rs := &Replies{err: err, items: items, conn: conn, usedItems: make(map[string]struct{}, len(keys))} |
||||
if (err != nil) || (len(items) == 0) { |
||||
rs.Close() |
||||
} |
||||
return rs, err |
||||
} |
||||
|
||||
// Close close rows.
|
||||
func (rs *Replies) Close() (err error) { |
||||
if !rs.closed { |
||||
err = rs.conn.Close() |
||||
rs.closed = true |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Item get Item from rows
|
||||
func (rs *Replies) Item(key string) *Item { |
||||
return rs.items[key] |
||||
} |
||||
|
||||
// Scan converts value, read from key in rows
|
||||
func (rs *Replies) Scan(key string, v interface{}) (err error) { |
||||
if rs.err != nil { |
||||
return rs.err |
||||
} |
||||
item, ok := rs.items[key] |
||||
if !ok { |
||||
rs.Close() |
||||
return ErrNotFound |
||||
} |
||||
rs.usedItems[key] = struct{}{} |
||||
err = rs.conn.Scan(item, v) |
||||
if (err != nil) || (len(rs.items) == len(rs.usedItems)) { |
||||
rs.Close() |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Keys keys of result
|
||||
func (rs *Replies) Keys() (keys []string) { |
||||
keys = make([]string, 0, len(rs.items)) |
||||
for key := range rs.items { |
||||
keys = append(keys, key) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Touch updates the expiry for the given key.
|
||||
func (mc *Memcache) Touch(c context.Context, key string, timeout int32) (err error) { |
||||
conn := mc.pool.Get(c) |
||||
err = conn.Touch(key, timeout) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// Delete deletes the item with the provided key.
|
||||
func (mc *Memcache) Delete(c context.Context, key string) (err error) { |
||||
conn := mc.pool.Get(c) |
||||
err = conn.Delete(key) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// Increment atomically increments key by delta.
|
||||
func (mc *Memcache) Increment(c context.Context, key string, delta uint64) (newValue uint64, err error) { |
||||
conn := mc.pool.Get(c) |
||||
newValue, err = conn.Increment(key, delta) |
||||
conn.Close() |
||||
return |
||||
} |
||||
|
||||
// Decrement atomically decrements key by delta.
|
||||
func (mc *Memcache) Decrement(c context.Context, key string, delta uint64) (newValue uint64, err error) { |
||||
conn := mc.pool.Get(c) |
||||
newValue, err = conn.Decrement(key, delta) |
||||
conn.Close() |
||||
return |
||||
} |
||||
@ -0,0 +1,185 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"testing" |
||||
|
||||
test "github.com/bilibili/kratos/pkg/cache/memcache/test" |
||||
"github.com/gogo/protobuf/proto" |
||||
) |
||||
|
||||
func TestConnRaw(t *testing.T) { |
||||
item := &Item{ |
||||
Key: "test", |
||||
Value: []byte("test"), |
||||
Flags: FlagRAW, |
||||
Expiration: 60, |
||||
cas: 0, |
||||
} |
||||
if err := testConnASCII.Set(item); err != nil { |
||||
t.Errorf("conn.Store() error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestConnSerialization(t *testing.T) { |
||||
type TestObj struct { |
||||
Name string |
||||
Age int32 |
||||
} |
||||
|
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
e error |
||||
}{ |
||||
|
||||
{ |
||||
"JSON", |
||||
&Item{ |
||||
Key: "test_json", |
||||
Object: &TestObj{"json", 1}, |
||||
Expiration: 60, |
||||
Flags: FlagJSON, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"JSONGzip", |
||||
&Item{ |
||||
Key: "test_json_gzip", |
||||
Object: &TestObj{"jsongzip", 2}, |
||||
Expiration: 60, |
||||
Flags: FlagJSON | FlagGzip, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"GOB", |
||||
&Item{ |
||||
Key: "test_gob", |
||||
Object: &TestObj{"gob", 3}, |
||||
Expiration: 60, |
||||
Flags: FlagGOB, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"GOBGzip", |
||||
&Item{ |
||||
Key: "test_gob_gzip", |
||||
Object: &TestObj{"gobgzip", 4}, |
||||
Expiration: 60, |
||||
Flags: FlagGOB | FlagGzip, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"Protobuf", |
||||
&Item{ |
||||
Key: "test_protobuf", |
||||
Object: &test.TestItem{Name: "protobuf", Age: 6}, |
||||
Expiration: 60, |
||||
Flags: FlagProtobuf, |
||||
}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"ProtobufGzip", |
||||
&Item{ |
||||
Key: "test_protobuf_gzip", |
||||
Object: &test.TestItem{Name: "protobufgzip", Age: 7}, |
||||
Expiration: 60, |
||||
Flags: FlagProtobuf | FlagGzip, |
||||
}, |
||||
nil, |
||||
}, |
||||
} |
||||
for _, tc := range tests { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
if err := testConnASCII.Set(tc.a); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if r, err := testConnASCII.Get(tc.a.Key); err != tc.e { |
||||
t.Fatal(err) |
||||
} else { |
||||
if (tc.a.Flags & FlagProtobuf) > 0 { |
||||
var no test.TestItem |
||||
if err := testConnASCII.Scan(r, &no); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if (tc.a.Object.(*test.TestItem).Name != no.Name) || (tc.a.Object.(*test.TestItem).Age != no.Age) { |
||||
t.Fatalf("compare failed error, %v %v", tc.a.Object.(*test.TestItem), no) |
||||
} |
||||
} else { |
||||
var no TestObj |
||||
if err := testConnASCII.Scan(r, &no); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if (tc.a.Object.(*TestObj).Name != no.Name) || (tc.a.Object.(*TestObj).Age != no.Age) { |
||||
t.Fatalf("compare failed error, %v %v", tc.a.Object.(*TestObj), no) |
||||
} |
||||
} |
||||
|
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkConnJSON(b *testing.B) { |
||||
st := &struct { |
||||
Name string |
||||
Age int |
||||
}{"json", 10} |
||||
itemx := &Item{Key: "json", Object: st, Flags: FlagJSON} |
||||
var ( |
||||
eb bytes.Buffer |
||||
je *json.Encoder |
||||
ir bytes.Reader |
||||
jd *json.Decoder |
||||
jr reader |
||||
nst test.TestItem |
||||
) |
||||
jd = json.NewDecoder(&jr) |
||||
je = json.NewEncoder(&eb) |
||||
eb.Grow(_encodeBuf) |
||||
// NOTE reuse bytes.Buffer internal buf
|
||||
// DON'T concurrency call Scan
|
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
eb.Reset() |
||||
if err := je.Encode(itemx.Object); err != nil { |
||||
return |
||||
} |
||||
data := eb.Bytes() |
||||
ir.Reset(data) |
||||
jr.Reset(&ir) |
||||
jd.Decode(&nst) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkConnProtobuf(b *testing.B) { |
||||
st := &test.TestItem{Name: "protobuf", Age: 10} |
||||
itemx := &Item{Key: "protobuf", Object: st, Flags: FlagJSON} |
||||
var ( |
||||
eb bytes.Buffer |
||||
nst test.TestItem |
||||
ped *proto.Buffer |
||||
) |
||||
ped = proto.NewBuffer(eb.Bytes()) |
||||
eb.Grow(_encodeBuf) |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
ped.Reset() |
||||
pb, ok := itemx.Object.(proto.Message) |
||||
if !ok { |
||||
return |
||||
} |
||||
if err := ped.Marshal(pb); err != nil { |
||||
return |
||||
} |
||||
data := ped.Bytes() |
||||
ped.SetBuf(data) |
||||
ped.Unmarshal(&nst) |
||||
} |
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"bytes" |
||||
"compress/gzip" |
||||
"encoding/gob" |
||||
"encoding/json" |
||||
"io" |
||||
|
||||
"github.com/gogo/protobuf/proto" |
||||
) |
||||
|
||||
type reader struct { |
||||
io.Reader |
||||
} |
||||
|
||||
func (r *reader) Reset(rd io.Reader) { |
||||
r.Reader = rd |
||||
} |
||||
|
||||
const _encodeBuf = 4096 // 4kb
|
||||
|
||||
type encodeDecode struct { |
||||
// Item Reader
|
||||
ir bytes.Reader |
||||
// Compress
|
||||
gr gzip.Reader |
||||
gw *gzip.Writer |
||||
cb bytes.Buffer |
||||
// Encoding
|
||||
edb bytes.Buffer |
||||
// json
|
||||
jr reader |
||||
jd *json.Decoder |
||||
je *json.Encoder |
||||
// protobuffer
|
||||
ped *proto.Buffer |
||||
} |
||||
|
||||
func newEncodeDecoder() *encodeDecode { |
||||
ed := &encodeDecode{} |
||||
ed.jd = json.NewDecoder(&ed.jr) |
||||
ed.je = json.NewEncoder(&ed.edb) |
||||
ed.gw = gzip.NewWriter(&ed.cb) |
||||
ed.edb.Grow(_encodeBuf) |
||||
// NOTE reuse bytes.Buffer internal buf
|
||||
// DON'T concurrency call Scan
|
||||
ed.ped = proto.NewBuffer(ed.edb.Bytes()) |
||||
return ed |
||||
} |
||||
|
||||
func (ed *encodeDecode) encode(item *Item) (data []byte, err error) { |
||||
if (item.Flags | _flagEncoding) == _flagEncoding { |
||||
if item.Value == nil { |
||||
return nil, ErrItem |
||||
} |
||||
} else if item.Object == nil { |
||||
return nil, ErrItem |
||||
} |
||||
// encoding
|
||||
switch { |
||||
case item.Flags&FlagGOB == FlagGOB: |
||||
ed.edb.Reset() |
||||
if err = gob.NewEncoder(&ed.edb).Encode(item.Object); err != nil { |
||||
return |
||||
} |
||||
data = ed.edb.Bytes() |
||||
case item.Flags&FlagProtobuf == FlagProtobuf: |
||||
ed.edb.Reset() |
||||
ed.ped.SetBuf(ed.edb.Bytes()) |
||||
pb, ok := item.Object.(proto.Message) |
||||
if !ok { |
||||
err = ErrItemObject |
||||
return |
||||
} |
||||
if err = ed.ped.Marshal(pb); err != nil { |
||||
return |
||||
} |
||||
data = ed.ped.Bytes() |
||||
case item.Flags&FlagJSON == FlagJSON: |
||||
ed.edb.Reset() |
||||
if err = ed.je.Encode(item.Object); err != nil { |
||||
return |
||||
} |
||||
data = ed.edb.Bytes() |
||||
default: |
||||
data = item.Value |
||||
} |
||||
// compress
|
||||
if item.Flags&FlagGzip == FlagGzip { |
||||
ed.cb.Reset() |
||||
ed.gw.Reset(&ed.cb) |
||||
if _, err = ed.gw.Write(data); err != nil { |
||||
return |
||||
} |
||||
if err = ed.gw.Close(); err != nil { |
||||
return |
||||
} |
||||
data = ed.cb.Bytes() |
||||
} |
||||
if len(data) > 8000000 { |
||||
err = ErrValueSize |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (ed *encodeDecode) decode(item *Item, v interface{}) (err error) { |
||||
var ( |
||||
data []byte |
||||
rd io.Reader |
||||
) |
||||
ed.ir.Reset(item.Value) |
||||
rd = &ed.ir |
||||
if item.Flags&FlagGzip == FlagGzip { |
||||
rd = &ed.gr |
||||
if err = ed.gr.Reset(&ed.ir); err != nil { |
||||
return |
||||
} |
||||
defer func() { |
||||
if e := ed.gr.Close(); e != nil { |
||||
err = e |
||||
} |
||||
}() |
||||
} |
||||
switch { |
||||
case item.Flags&FlagGOB == FlagGOB: |
||||
err = gob.NewDecoder(rd).Decode(v) |
||||
case item.Flags&FlagJSON == FlagJSON: |
||||
ed.jr.Reset(rd) |
||||
err = ed.jd.Decode(v) |
||||
default: |
||||
data = item.Value |
||||
if item.Flags&FlagGzip == FlagGzip { |
||||
ed.edb.Reset() |
||||
if _, err = io.Copy(&ed.edb, rd); err != nil { |
||||
return |
||||
} |
||||
data = ed.edb.Bytes() |
||||
} |
||||
if item.Flags&FlagProtobuf == FlagProtobuf { |
||||
m, ok := v.(proto.Message) |
||||
if !ok { |
||||
err = ErrItemObject |
||||
return |
||||
} |
||||
ed.ped.SetBuf(data) |
||||
err = ed.ped.Unmarshal(m) |
||||
} else { |
||||
switch v.(type) { |
||||
case *[]byte: |
||||
d := v.(*[]byte) |
||||
*d = data |
||||
case *string: |
||||
d := v.(*string) |
||||
*d = string(data) |
||||
case interface{}: |
||||
err = json.Unmarshal(data, v) |
||||
} |
||||
} |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,220 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"bytes" |
||||
"testing" |
||||
|
||||
mt "github.com/bilibili/kratos/pkg/cache/memcache/test" |
||||
) |
||||
|
||||
func TestEncode(t *testing.T) { |
||||
type TestObj struct { |
||||
Name string |
||||
Age int32 |
||||
} |
||||
testObj := TestObj{"abc", 1} |
||||
|
||||
ed := newEncodeDecoder() |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
r []byte |
||||
e error |
||||
}{ |
||||
{ |
||||
"EncodeRawFlagErrItem", |
||||
&Item{ |
||||
Object: &TestObj{"abc", 1}, |
||||
Flags: FlagRAW, |
||||
}, |
||||
[]byte{}, |
||||
ErrItem, |
||||
}, |
||||
{ |
||||
"EncodeEncodeFlagErrItem", |
||||
&Item{ |
||||
Value: []byte("test"), |
||||
Flags: FlagJSON, |
||||
}, |
||||
[]byte{}, |
||||
ErrItem, |
||||
}, |
||||
{ |
||||
"EncodeEmpty", |
||||
&Item{ |
||||
Value: []byte(""), |
||||
Flags: FlagRAW, |
||||
}, |
||||
[]byte(""), |
||||
nil, |
||||
}, |
||||
{ |
||||
"EncodeMaxSize", |
||||
&Item{ |
||||
Value: bytes.Repeat([]byte("A"), 8000000), |
||||
Flags: FlagRAW, |
||||
}, |
||||
bytes.Repeat([]byte("A"), 8000000), |
||||
nil, |
||||
}, |
||||
{ |
||||
"EncodeExceededMaxSize", |
||||
&Item{ |
||||
Value: bytes.Repeat([]byte("A"), 8000000+1), |
||||
Flags: FlagRAW, |
||||
}, |
||||
nil, |
||||
ErrValueSize, |
||||
}, |
||||
{ |
||||
"EncodeGOB", |
||||
&Item{ |
||||
Object: testObj, |
||||
Flags: FlagGOB, |
||||
}, |
||||
[]byte{38, 255, 131, 3, 1, 1, 7, 84, 101, 115, 116, 79, 98, 106, 1, 255, 132, 0, 1, 2, 1, 4, 78, 97, 109, 101, 1, 12, 0, 1, 3, 65, 103, 101, 1, 4, 0, 0, 0, 10, 255, 132, 1, 3, 97, 98, 99, 1, 2, 0}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"EncodeJSON", |
||||
&Item{ |
||||
Object: testObj, |
||||
Flags: FlagJSON, |
||||
}, |
||||
[]byte{123, 34, 78, 97, 109, 101, 34, 58, 34, 97, 98, 99, 34, 44, 34, 65, 103, 101, 34, 58, 49, 125, 10}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"EncodeProtobuf", |
||||
&Item{ |
||||
Object: &mt.TestItem{Name: "abc", Age: 1}, |
||||
Flags: FlagProtobuf, |
||||
}, |
||||
[]byte{10, 3, 97, 98, 99, 16, 1}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"EncodeGzip", |
||||
&Item{ |
||||
Value: bytes.Repeat([]byte("B"), 50), |
||||
Flags: FlagGzip, |
||||
}, |
||||
[]byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 114, 34, 25, 0, 2, 0, 0, 255, 255, 252, 253, 67, 209, 50, 0, 0, 0}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"EncodeGOBGzip", |
||||
&Item{ |
||||
Object: testObj, |
||||
Flags: FlagGOB | FlagGzip, |
||||
}, |
||||
[]byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 82, 251, 223, 204, 204, 200, 200, 30, 146, 90, 92, 226, 159, 148, 197, 248, 191, 133, 129, 145, 137, 145, 197, 47, 49, 55, 149, 145, 135, 129, 145, 217, 49, 61, 149, 145, 133, 129, 129, 129, 235, 127, 11, 35, 115, 98, 82, 50, 35, 19, 3, 32, 0, 0, 255, 255, 211, 249, 1, 154, 50, 0, 0, 0}, |
||||
nil, |
||||
}, |
||||
} |
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if r, err := ed.encode(test.a); err != test.e { |
||||
t.Fatal(err) |
||||
} else { |
||||
if err == nil { |
||||
if !bytes.Equal(r, test.r) { |
||||
t.Fatalf("not equal, expect %v\n got %v", test.r, r) |
||||
} |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestDecode(t *testing.T) { |
||||
type TestObj struct { |
||||
Name string |
||||
Age int32 |
||||
} |
||||
testObj := &TestObj{"abc", 1} |
||||
|
||||
ed := newEncodeDecoder() |
||||
tests := []struct { |
||||
name string |
||||
a *Item |
||||
r interface{} |
||||
e error |
||||
}{ |
||||
{ |
||||
"DecodeGOB", |
||||
&Item{ |
||||
Flags: FlagGOB, |
||||
Value: []byte{38, 255, 131, 3, 1, 1, 7, 84, 101, 115, 116, 79, 98, 106, 1, 255, 132, 0, 1, 2, 1, 4, 78, 97, 109, 101, 1, 12, 0, 1, 3, 65, 103, 101, 1, 4, 0, 0, 0, 10, 255, 132, 1, 3, 97, 98, 99, 1, 2, 0}, |
||||
}, |
||||
testObj, |
||||
nil, |
||||
}, |
||||
{ |
||||
"DecodeJSON", |
||||
&Item{ |
||||
Value: []byte{123, 34, 78, 97, 109, 101, 34, 58, 34, 97, 98, 99, 34, 44, 34, 65, 103, 101, 34, 58, 49, 125, 10}, |
||||
Flags: FlagJSON, |
||||
}, |
||||
testObj, |
||||
nil, |
||||
}, |
||||
{ |
||||
"DecodeProtobuf", |
||||
&Item{ |
||||
Value: []byte{10, 3, 97, 98, 99, 16, 1}, |
||||
|
||||
Flags: FlagProtobuf, |
||||
}, |
||||
&mt.TestItem{Name: "abc", Age: 1}, |
||||
nil, |
||||
}, |
||||
{ |
||||
"DecodeGzip", |
||||
&Item{ |
||||
Value: []byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 114, 34, 25, 0, 2, 0, 0, 255, 255, 252, 253, 67, 209, 50, 0, 0, 0}, |
||||
Flags: FlagGzip, |
||||
}, |
||||
bytes.Repeat([]byte("B"), 50), |
||||
nil, |
||||
}, |
||||
{ |
||||
"DecodeGOBGzip", |
||||
&Item{ |
||||
Value: []byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 82, 251, 223, 204, 204, 200, 200, 30, 146, 90, 92, 226, 159, 148, 197, 248, 191, 133, 129, 145, 137, 145, 197, 47, 49, 55, 149, 145, 135, 129, 145, 217, 49, 61, 149, 145, 133, 129, 129, 129, 235, 127, 11, 35, 115, 98, 82, 50, 35, 19, 3, 32, 0, 0, 255, 255, 211, 249, 1, 154, 50, 0, 0, 0}, |
||||
Flags: FlagGOB | FlagGzip, |
||||
}, |
||||
testObj, |
||||
nil, |
||||
}, |
||||
} |
||||
for _, test := range tests { |
||||
t.Run(test.name, func(t *testing.T) { |
||||
if (test.a.Flags & FlagProtobuf) > 0 { |
||||
var dd mt.TestItem |
||||
if err := ed.decode(test.a, &dd); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if (test.r.(*mt.TestItem).Name != dd.Name) || (test.r.(*mt.TestItem).Age != dd.Age) { |
||||
t.Fatalf("compare failed error, expect %v\n got %v", test.r.(*mt.TestItem), dd) |
||||
} |
||||
} else if test.a.Flags == FlagGzip { |
||||
var dd []byte |
||||
if err := ed.decode(test.a, &dd); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if !bytes.Equal(dd, test.r.([]byte)) { |
||||
t.Fatalf("compare failed error, expect %v\n got %v", test.r, dd) |
||||
} |
||||
} else { |
||||
var dd TestObj |
||||
if err := ed.decode(test.a, &dd); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if (test.r.(*TestObj).Name != dd.Name) || (test.r.(*TestObj).Age != dd.Age) { |
||||
t.Fatalf("compare failed error, expect %v\n got %v", test.r.(*TestObj), dd) |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,177 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"time" |
||||
) |
||||
|
||||
var testExampleAddr string |
||||
|
||||
func ExampleConn_set() { |
||||
var ( |
||||
err error |
||||
value []byte |
||||
conn Conn |
||||
expire int32 = 100 |
||||
p = struct { |
||||
Name string |
||||
Age int64 |
||||
}{"golang", 10} |
||||
) |
||||
cnop := DialConnectTimeout(time.Duration(time.Second)) |
||||
rdop := DialReadTimeout(time.Duration(time.Second)) |
||||
wrop := DialWriteTimeout(time.Duration(time.Second)) |
||||
if value, err = json.Marshal(p); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
if conn, err = Dial("tcp", testExampleAddr, cnop, rdop, wrop); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
// FlagRAW test
|
||||
itemRaw := &Item{ |
||||
Key: "test_raw", |
||||
Value: value, |
||||
Expiration: expire, |
||||
} |
||||
if err = conn.Set(itemRaw); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
// FlagGzip
|
||||
itemGZip := &Item{ |
||||
Key: "test_gzip", |
||||
Value: value, |
||||
Flags: FlagGzip, |
||||
Expiration: expire, |
||||
} |
||||
if err = conn.Set(itemGZip); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
// FlagGOB
|
||||
itemGOB := &Item{ |
||||
Key: "test_gob", |
||||
Object: p, |
||||
Flags: FlagGOB, |
||||
Expiration: expire, |
||||
} |
||||
if err = conn.Set(itemGOB); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
// FlagJSON
|
||||
itemJSON := &Item{ |
||||
Key: "test_json", |
||||
Object: p, |
||||
Flags: FlagJSON, |
||||
Expiration: expire, |
||||
} |
||||
if err = conn.Set(itemJSON); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
// FlagJSON | FlagGzip
|
||||
itemJSONGzip := &Item{ |
||||
Key: "test_jsonGzip", |
||||
Object: p, |
||||
Flags: FlagJSON | FlagGzip, |
||||
Expiration: expire, |
||||
} |
||||
if err = conn.Set(itemJSONGzip); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
// Output:
|
||||
} |
||||
|
||||
func ExampleConn_get() { |
||||
var ( |
||||
err error |
||||
item2 *Item |
||||
conn Conn |
||||
p struct { |
||||
Name string |
||||
Age int64 |
||||
} |
||||
) |
||||
cnop := DialConnectTimeout(time.Duration(time.Second)) |
||||
rdop := DialReadTimeout(time.Duration(time.Second)) |
||||
wrop := DialWriteTimeout(time.Duration(time.Second)) |
||||
if conn, err = Dial("tcp", testExampleAddr, cnop, rdop, wrop); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
if item2, err = conn.Get("test_raw"); err != nil { |
||||
fmt.Println(err) |
||||
} else { |
||||
if err = conn.Scan(item2, &p); err != nil { |
||||
fmt.Printf("FlagRAW conn.Scan error(%v)\n", err) |
||||
return |
||||
} |
||||
} |
||||
// FlagGZip
|
||||
if item2, err = conn.Get("test_gzip"); err != nil { |
||||
fmt.Println(err) |
||||
} else { |
||||
if err = conn.Scan(item2, &p); err != nil { |
||||
fmt.Printf("FlagGZip conn.Scan error(%v)\n", err) |
||||
return |
||||
} |
||||
} |
||||
// FlagGOB
|
||||
if item2, err = conn.Get("test_gob"); err != nil { |
||||
fmt.Println(err) |
||||
} else { |
||||
if err = conn.Scan(item2, &p); err != nil { |
||||
fmt.Printf("FlagGOB conn.Scan error(%v)\n", err) |
||||
return |
||||
} |
||||
} |
||||
// FlagJSON
|
||||
if item2, err = conn.Get("test_json"); err != nil { |
||||
fmt.Println(err) |
||||
} else { |
||||
if err = conn.Scan(item2, &p); err != nil { |
||||
fmt.Printf("FlagJSON conn.Scan error(%v)\n", err) |
||||
return |
||||
} |
||||
} |
||||
// Output:
|
||||
} |
||||
|
||||
func ExampleConn_getMulti() { |
||||
var ( |
||||
err error |
||||
conn Conn |
||||
res map[string]*Item |
||||
keys = []string{"test_raw", "test_gzip"} |
||||
p struct { |
||||
Name string |
||||
Age int64 |
||||
} |
||||
) |
||||
cnop := DialConnectTimeout(time.Duration(time.Second)) |
||||
rdop := DialReadTimeout(time.Duration(time.Second)) |
||||
wrop := DialWriteTimeout(time.Duration(time.Second)) |
||||
if conn, err = Dial("tcp", testExampleAddr, cnop, rdop, wrop); err != nil { |
||||
fmt.Println(err) |
||||
return |
||||
} |
||||
if res, err = conn.GetMulti(keys); err != nil { |
||||
fmt.Printf("conn.GetMulti(%v) error(%v)", keys, err) |
||||
return |
||||
} |
||||
for _, v := range res { |
||||
if err = conn.Scan(v, &p); err != nil { |
||||
fmt.Printf("conn.Scan error(%v)\n", err) |
||||
return |
||||
} |
||||
fmt.Println(p) |
||||
} |
||||
// Output:
|
||||
//{golang 10}
|
||||
//{golang 10}
|
||||
} |
||||
@ -0,0 +1,85 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"log" |
||||
"os" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/container/pool" |
||||
xtime "github.com/bilibili/kratos/pkg/time" |
||||
) |
||||
|
||||
var testConnASCII Conn |
||||
var testMemcache *Memcache |
||||
var testPool *Pool |
||||
var testMemcacheAddr string |
||||
|
||||
func setupTestConnASCII(addr string) { |
||||
var err error |
||||
cnop := DialConnectTimeout(time.Duration(2 * time.Second)) |
||||
rdop := DialReadTimeout(time.Duration(2 * time.Second)) |
||||
wrop := DialWriteTimeout(time.Duration(2 * time.Second)) |
||||
testConnASCII, err = Dial("tcp", addr, cnop, rdop, wrop) |
||||
if err != nil { |
||||
log.Fatal(err) |
||||
} |
||||
testConnASCII.Delete("test") |
||||
testConnASCII.Delete("test1") |
||||
testConnASCII.Delete("test2") |
||||
if err != nil { |
||||
log.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
func setupTestMemcache(addr string) { |
||||
testConfig := &Config{ |
||||
Config: &pool.Config{ |
||||
Active: 10, |
||||
Idle: 10, |
||||
IdleTimeout: xtime.Duration(time.Second), |
||||
WaitTimeout: xtime.Duration(time.Second), |
||||
Wait: false, |
||||
}, |
||||
Addr: addr, |
||||
Proto: "tcp", |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
} |
||||
testMemcache = New(testConfig) |
||||
} |
||||
|
||||
func setupTestPool(addr string) { |
||||
config := &Config{ |
||||
Name: "test", |
||||
Proto: "tcp", |
||||
Addr: addr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
} |
||||
config.Config = &pool.Config{ |
||||
Active: 10, |
||||
Idle: 5, |
||||
IdleTimeout: xtime.Duration(90 * time.Second), |
||||
} |
||||
testPool = NewPool(config) |
||||
} |
||||
|
||||
func TestMain(m *testing.M) { |
||||
testMemcacheAddr = os.Getenv("TEST_MEMCACHE_ADDR") |
||||
if testExampleAddr == "" { |
||||
log.Print("TEST_MEMCACHE_ADDR not provide skip test.") |
||||
// ignored test.
|
||||
os.Exit(0) |
||||
} |
||||
setupTestConnASCII(testMemcacheAddr) |
||||
setupTestMemcache(testMemcacheAddr) |
||||
setupTestPool(testMemcacheAddr) |
||||
// TODO: add setupexample?
|
||||
testExampleAddr = testMemcacheAddr |
||||
|
||||
ret := m.Run() |
||||
os.Exit(ret) |
||||
} |
||||
@ -0,0 +1,300 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"reflect" |
||||
"testing" |
||||
"time" |
||||
) |
||||
|
||||
func Test_client_Set(t *testing.T) { |
||||
type args struct { |
||||
c context.Context |
||||
item *Item |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr bool |
||||
}{ |
||||
{name: "set value", args: args{c: context.Background(), item: &Item{Key: "Test_client_Set", Value: []byte("abc")}}, wantErr: false}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if err := testMemcache.Set(tt.args.c, tt.args.item); (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Set() error = %v, wantErr %v", err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Add(t *testing.T) { |
||||
type args struct { |
||||
c context.Context |
||||
item *Item |
||||
} |
||||
key := fmt.Sprintf("Test_client_Add_%d", time.Now().Unix()) |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr bool |
||||
}{ |
||||
{name: "add not exist value", args: args{c: context.Background(), item: &Item{Key: key, Value: []byte("abc")}}, wantErr: false}, |
||||
{name: "add exist value", args: args{c: context.Background(), item: &Item{Key: key, Value: []byte("abc")}}, wantErr: true}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if err := testMemcache.Add(tt.args.c, tt.args.item); (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Add() error = %v, wantErr %v", err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Replace(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_Replace_%d", time.Now().Unix()) |
||||
ekey := "Test_client_Replace_exist" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("ok")}) |
||||
type args struct { |
||||
c context.Context |
||||
item *Item |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), item: &Item{Key: key, Value: []byte("abc")}}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), item: &Item{Key: ekey, Value: []byte("abc")}}, wantErr: false}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if err := testMemcache.Replace(tt.args.c, tt.args.item); (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Replace() error = %v, wantErr %v", err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_CompareAndSwap(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_CompareAndSwap_%d", time.Now().Unix()) |
||||
ekey := "Test_client_CompareAndSwap_k" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("old")}) |
||||
cas := testMemcache.Get(context.Background(), ekey).Item().cas |
||||
type args struct { |
||||
c context.Context |
||||
item *Item |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), item: &Item{Key: key, Value: []byte("abc")}}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), item: &Item{Key: ekey, cas: cas, Value: []byte("abc")}}, wantErr: false}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if err := testMemcache.CompareAndSwap(tt.args.c, tt.args.item); (err != nil) != tt.wantErr { |
||||
t.Errorf("client.CompareAndSwap() error = %v, wantErr %v", err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Get(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_Get_%d", time.Now().Unix()) |
||||
ekey := "Test_client_Get_k" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("old")}) |
||||
type args struct { |
||||
c context.Context |
||||
key string |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
want string |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), key: key}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), key: ekey}, wantErr: false, want: "old"}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
var res string |
||||
if err := testMemcache.Get(tt.args.c, tt.args.key).Scan(&res); (err != nil) != tt.wantErr || res != tt.want { |
||||
t.Errorf("client.Get() = %v, want %v, got err: %v, want err: %v", err, tt.want, err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Touch(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_Touch_%d", time.Now().Unix()) |
||||
ekey := "Test_client_Touch_k" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("old")}) |
||||
type args struct { |
||||
c context.Context |
||||
key string |
||||
timeout int32 |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), key: key, timeout: 100000}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), key: ekey, timeout: 100000}, wantErr: false}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if err := testMemcache.Touch(tt.args.c, tt.args.key, tt.args.timeout); (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Touch() error = %v, wantErr %v", err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Delete(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_Delete_%d", time.Now().Unix()) |
||||
ekey := "Test_client_Delete_k" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("old")}) |
||||
type args struct { |
||||
c context.Context |
||||
key string |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), key: key}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), key: ekey}, wantErr: false}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if err := testMemcache.Delete(tt.args.c, tt.args.key); (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Delete() error = %v, wantErr %v", err, tt.wantErr) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Increment(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_Increment_%d", time.Now().Unix()) |
||||
ekey := "Test_client_Increment_k" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("1")}) |
||||
type args struct { |
||||
c context.Context |
||||
key string |
||||
delta uint64 |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantNewValue uint64 |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), key: key, delta: 10}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), key: ekey, delta: 10}, wantErr: false, wantNewValue: 11}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
gotNewValue, err := testMemcache.Increment(tt.args.c, tt.args.key, tt.args.delta) |
||||
if (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Increment() error = %v, wantErr %v", err, tt.wantErr) |
||||
return |
||||
} |
||||
if gotNewValue != tt.wantNewValue { |
||||
t.Errorf("client.Increment() = %v, want %v", gotNewValue, tt.wantNewValue) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Decrement(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_Decrement_%d", time.Now().Unix()) |
||||
ekey := "Test_client_Decrement_k" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey, Value: []byte("100")}) |
||||
type args struct { |
||||
c context.Context |
||||
key string |
||||
delta uint64 |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantNewValue uint64 |
||||
wantErr bool |
||||
}{ |
||||
{name: "not exist value", args: args{c: context.Background(), key: key, delta: 10}, wantErr: true}, |
||||
{name: "exist value", args: args{c: context.Background(), key: ekey, delta: 10}, wantErr: false, wantNewValue: 90}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
gotNewValue, err := testMemcache.Decrement(tt.args.c, tt.args.key, tt.args.delta) |
||||
if (err != nil) != tt.wantErr { |
||||
t.Errorf("client.Decrement() error = %v, wantErr %v", err, tt.wantErr) |
||||
return |
||||
} |
||||
if gotNewValue != tt.wantNewValue { |
||||
t.Errorf("client.Decrement() = %v, want %v", gotNewValue, tt.wantNewValue) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_client_GetMulti(t *testing.T) { |
||||
key := fmt.Sprintf("Test_client_GetMulti_%d", time.Now().Unix()) |
||||
ekey1 := "Test_client_GetMulti_k1" |
||||
ekey2 := "Test_client_GetMulti_k2" |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey1, Value: []byte("1")}) |
||||
testMemcache.Set(context.Background(), &Item{Key: ekey2, Value: []byte("2")}) |
||||
keys := []string{key, ekey1, ekey2} |
||||
rows, err := testMemcache.GetMulti(context.Background(), keys) |
||||
if err != nil { |
||||
t.Errorf("client.GetMulti() error = %v, wantErr %v", err, nil) |
||||
} |
||||
tests := []struct { |
||||
key string |
||||
wantNewValue string |
||||
wantErr bool |
||||
nilItem bool |
||||
}{ |
||||
{key: ekey1, wantErr: false, wantNewValue: "1", nilItem: false}, |
||||
{key: ekey2, wantErr: false, wantNewValue: "2", nilItem: false}, |
||||
{key: key, wantErr: true, nilItem: true}, |
||||
} |
||||
if reflect.DeepEqual(keys, rows.Keys()) { |
||||
t.Errorf("got %v, expect: %v", rows.Keys(), keys) |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.key, func(t *testing.T) { |
||||
var gotNewValue string |
||||
err = rows.Scan(tt.key, &gotNewValue) |
||||
if (err != nil) != tt.wantErr { |
||||
t.Errorf("rows.Scan() error = %v, wantErr %v", err, tt.wantErr) |
||||
return |
||||
} |
||||
if gotNewValue != tt.wantNewValue { |
||||
t.Errorf("rows.Value() = %v, want %v", gotNewValue, tt.wantNewValue) |
||||
} |
||||
if (rows.Item(tt.key) == nil) != tt.nilItem { |
||||
t.Errorf("rows.Item() = %v, want %v", rows.Item(tt.key) == nil, tt.nilItem) |
||||
} |
||||
}) |
||||
} |
||||
err = rows.Close() |
||||
if err != nil { |
||||
t.Errorf("client.Replies.Close() error = %v, wantErr %v", err, nil) |
||||
} |
||||
} |
||||
|
||||
func Test_client_Conn(t *testing.T) { |
||||
conn := testMemcache.Conn(context.Background()) |
||||
defer conn.Close() |
||||
if conn == nil { |
||||
t.Errorf("expect get conn, get nil") |
||||
} |
||||
} |
||||
@ -1,59 +0,0 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
) |
||||
|
||||
// MockErr for unit test.
|
||||
type MockErr struct { |
||||
Error error |
||||
} |
||||
|
||||
var _ Conn = MockErr{} |
||||
|
||||
// MockWith return a mock conn.
|
||||
func MockWith(err error) MockErr { |
||||
return MockErr{Error: err} |
||||
} |
||||
|
||||
// Err .
|
||||
func (m MockErr) Err() error { return m.Error } |
||||
|
||||
// Close .
|
||||
func (m MockErr) Close() error { return m.Error } |
||||
|
||||
// Add .
|
||||
func (m MockErr) Add(item *Item) error { return m.Error } |
||||
|
||||
// Set .
|
||||
func (m MockErr) Set(item *Item) error { return m.Error } |
||||
|
||||
// Replace .
|
||||
func (m MockErr) Replace(item *Item) error { return m.Error } |
||||
|
||||
// CompareAndSwap .
|
||||
func (m MockErr) CompareAndSwap(item *Item) error { return m.Error } |
||||
|
||||
// Get .
|
||||
func (m MockErr) Get(key string) (*Item, error) { return nil, m.Error } |
||||
|
||||
// GetMulti .
|
||||
func (m MockErr) GetMulti(keys []string) (map[string]*Item, error) { return nil, m.Error } |
||||
|
||||
// Touch .
|
||||
func (m MockErr) Touch(key string, timeout int32) error { return m.Error } |
||||
|
||||
// Delete .
|
||||
func (m MockErr) Delete(key string) error { return m.Error } |
||||
|
||||
// Increment .
|
||||
func (m MockErr) Increment(key string, delta uint64) (uint64, error) { return 0, m.Error } |
||||
|
||||
// Decrement .
|
||||
func (m MockErr) Decrement(key string, delta uint64) (uint64, error) { return 0, m.Error } |
||||
|
||||
// Scan .
|
||||
func (m MockErr) Scan(item *Item, v interface{}) error { return m.Error } |
||||
|
||||
// WithContext .
|
||||
func (m MockErr) WithContext(ctx context.Context) Conn { return m } |
||||
@ -1,197 +0,0 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/container/pool" |
||||
"github.com/bilibili/kratos/pkg/stat" |
||||
xtime "github.com/bilibili/kratos/pkg/time" |
||||
) |
||||
|
||||
var stats = stat.Cache |
||||
|
||||
// Config memcache config.
|
||||
type Config struct { |
||||
*pool.Config |
||||
|
||||
Name string // memcache name, for trace
|
||||
Proto string |
||||
Addr string |
||||
DialTimeout xtime.Duration |
||||
ReadTimeout xtime.Duration |
||||
WriteTimeout xtime.Duration |
||||
} |
||||
|
||||
// Pool memcache connection pool struct.
|
||||
type Pool struct { |
||||
p pool.Pool |
||||
c *Config |
||||
} |
||||
|
||||
// NewPool new a memcache conn pool.
|
||||
func NewPool(c *Config) (p *Pool) { |
||||
if c.DialTimeout <= 0 || c.ReadTimeout <= 0 || c.WriteTimeout <= 0 { |
||||
panic("must config memcache timeout") |
||||
} |
||||
p1 := pool.NewList(c.Config) |
||||
cnop := DialConnectTimeout(time.Duration(c.DialTimeout)) |
||||
rdop := DialReadTimeout(time.Duration(c.ReadTimeout)) |
||||
wrop := DialWriteTimeout(time.Duration(c.WriteTimeout)) |
||||
p1.New = func(ctx context.Context) (io.Closer, error) { |
||||
conn, err := Dial(c.Proto, c.Addr, cnop, rdop, wrop) |
||||
return &traceConn{Conn: conn, address: c.Addr}, err |
||||
} |
||||
p = &Pool{p: p1, c: c} |
||||
return |
||||
} |
||||
|
||||
// Get gets a connection. The application must close the returned connection.
|
||||
// This method always returns a valid connection so that applications can defer
|
||||
// error handling to the first use of the connection. If there is an error
|
||||
// getting an underlying connection, then the connection Err, Do, Send, Flush
|
||||
// and Receive methods return that error.
|
||||
func (p *Pool) Get(ctx context.Context) Conn { |
||||
c, err := p.p.Get(ctx) |
||||
if err != nil { |
||||
return errorConnection{err} |
||||
} |
||||
c1, _ := c.(Conn) |
||||
return &pooledConnection{p: p, c: c1.WithContext(ctx), ctx: ctx} |
||||
} |
||||
|
||||
// Close release the resources used by the pool.
|
||||
func (p *Pool) Close() error { |
||||
return p.p.Close() |
||||
} |
||||
|
||||
type pooledConnection struct { |
||||
p *Pool |
||||
c Conn |
||||
ctx context.Context |
||||
} |
||||
|
||||
func pstat(key string, t time.Time, err error) { |
||||
stats.Timing(key, int64(time.Since(t)/time.Millisecond)) |
||||
if err != nil { |
||||
if msg := formatErr(err); msg != "" { |
||||
stats.Incr("memcache", msg) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (pc *pooledConnection) Close() error { |
||||
c := pc.c |
||||
if _, ok := c.(errorConnection); ok { |
||||
return nil |
||||
} |
||||
pc.c = errorConnection{ErrConnClosed} |
||||
pc.p.p.Put(context.Background(), c, c.Err() != nil) |
||||
return nil |
||||
} |
||||
|
||||
func (pc *pooledConnection) Err() error { |
||||
return pc.c.Err() |
||||
} |
||||
|
||||
func (pc *pooledConnection) Set(item *Item) (err error) { |
||||
now := time.Now() |
||||
err = pc.c.Set(item) |
||||
pstat("memcache:set", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Add(item *Item) (err error) { |
||||
now := time.Now() |
||||
err = pc.c.Add(item) |
||||
pstat("memcache:add", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Replace(item *Item) (err error) { |
||||
now := time.Now() |
||||
err = pc.c.Replace(item) |
||||
pstat("memcache:replace", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) CompareAndSwap(item *Item) (err error) { |
||||
now := time.Now() |
||||
err = pc.c.CompareAndSwap(item) |
||||
pstat("memcache:cas", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Get(key string) (r *Item, err error) { |
||||
now := time.Now() |
||||
r, err = pc.c.Get(key) |
||||
pstat("memcache:get", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) GetMulti(keys []string) (res map[string]*Item, err error) { |
||||
// if keys is empty slice returns empty map direct
|
||||
if len(keys) == 0 { |
||||
return make(map[string]*Item), nil |
||||
} |
||||
now := time.Now() |
||||
res, err = pc.c.GetMulti(keys) |
||||
pstat("memcache:gets", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Touch(key string, timeout int32) (err error) { |
||||
now := time.Now() |
||||
err = pc.c.Touch(key, timeout) |
||||
pstat("memcache:touch", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Scan(item *Item, v interface{}) error { |
||||
return pc.c.Scan(item, v) |
||||
} |
||||
|
||||
func (pc *pooledConnection) WithContext(ctx context.Context) Conn { |
||||
// TODO: set context
|
||||
pc.ctx = ctx |
||||
return pc |
||||
} |
||||
|
||||
func (pc *pooledConnection) Delete(key string) (err error) { |
||||
now := time.Now() |
||||
err = pc.c.Delete(key) |
||||
pstat("memcache:delete", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Increment(key string, delta uint64) (newValue uint64, err error) { |
||||
now := time.Now() |
||||
newValue, err = pc.c.Increment(key, delta) |
||||
pstat("memcache:increment", now, err) |
||||
return |
||||
} |
||||
|
||||
func (pc *pooledConnection) Decrement(key string, delta uint64) (newValue uint64, err error) { |
||||
now := time.Now() |
||||
newValue, err = pc.c.Decrement(key, delta) |
||||
pstat("memcache:decrement", now, err) |
||||
return |
||||
} |
||||
|
||||
type errorConnection struct{ err error } |
||||
|
||||
func (ec errorConnection) Err() error { return ec.err } |
||||
func (ec errorConnection) Close() error { return ec.err } |
||||
func (ec errorConnection) Add(item *Item) error { return ec.err } |
||||
func (ec errorConnection) Set(item *Item) error { return ec.err } |
||||
func (ec errorConnection) Replace(item *Item) error { return ec.err } |
||||
func (ec errorConnection) CompareAndSwap(item *Item) error { return ec.err } |
||||
func (ec errorConnection) Get(key string) (*Item, error) { return nil, ec.err } |
||||
func (ec errorConnection) GetMulti(keys []string) (map[string]*Item, error) { return nil, ec.err } |
||||
func (ec errorConnection) Touch(key string, timeout int32) error { return ec.err } |
||||
func (ec errorConnection) Delete(key string) error { return ec.err } |
||||
func (ec errorConnection) Increment(key string, delta uint64) (uint64, error) { return 0, ec.err } |
||||
func (ec errorConnection) Decrement(key string, delta uint64) (uint64, error) { return 0, ec.err } |
||||
func (ec errorConnection) Scan(item *Item, v interface{}) error { return ec.err } |
||||
func (ec errorConnection) WithContext(ctx context.Context) Conn { return ec } |
||||
@ -0,0 +1,204 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/container/pool" |
||||
"github.com/bilibili/kratos/pkg/stat" |
||||
) |
||||
|
||||
var stats = stat.Cache |
||||
|
||||
// Pool memcache connection pool struct.
|
||||
// Deprecated: Use Memcache instead
|
||||
type Pool struct { |
||||
p pool.Pool |
||||
c *Config |
||||
} |
||||
|
||||
// NewPool new a memcache conn pool.
|
||||
// Deprecated: Use New instead
|
||||
func NewPool(cfg *Config) (p *Pool) { |
||||
if cfg.DialTimeout <= 0 || cfg.ReadTimeout <= 0 || cfg.WriteTimeout <= 0 { |
||||
panic("must config memcache timeout") |
||||
} |
||||
p1 := pool.NewList(cfg.Config) |
||||
cnop := DialConnectTimeout(time.Duration(cfg.DialTimeout)) |
||||
rdop := DialReadTimeout(time.Duration(cfg.ReadTimeout)) |
||||
wrop := DialWriteTimeout(time.Duration(cfg.WriteTimeout)) |
||||
p1.New = func(ctx context.Context) (io.Closer, error) { |
||||
conn, err := Dial(cfg.Proto, cfg.Addr, cnop, rdop, wrop) |
||||
return newTraceConn(conn, fmt.Sprintf("%s://%s", cfg.Proto, cfg.Addr)), err |
||||
} |
||||
p = &Pool{p: p1, c: cfg} |
||||
return |
||||
} |
||||
|
||||
// Get gets a connection. The application must close the returned connection.
|
||||
// This method always returns a valid connection so that applications can defer
|
||||
// error handling to the first use of the connection. If there is an error
|
||||
// getting an underlying connection, then the connection Err, Do, Send, Flush
|
||||
// and Receive methods return that error.
|
||||
func (p *Pool) Get(ctx context.Context) Conn { |
||||
c, err := p.p.Get(ctx) |
||||
if err != nil { |
||||
return errConn{err} |
||||
} |
||||
c1, _ := c.(Conn) |
||||
return &poolConn{p: p, c: c1, ctx: ctx} |
||||
} |
||||
|
||||
// Close release the resources used by the pool.
|
||||
func (p *Pool) Close() error { |
||||
return p.p.Close() |
||||
} |
||||
|
||||
type poolConn struct { |
||||
c Conn |
||||
p *Pool |
||||
ctx context.Context |
||||
} |
||||
|
||||
func pstat(key string, t time.Time, err error) { |
||||
stats.Timing(key, int64(time.Since(t)/time.Millisecond)) |
||||
if err != nil { |
||||
if msg := formatErr(err); msg != "" { |
||||
stats.Incr("memcache", msg) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (pc *poolConn) Close() error { |
||||
c := pc.c |
||||
if _, ok := c.(errConn); ok { |
||||
return nil |
||||
} |
||||
pc.c = errConn{ErrConnClosed} |
||||
pc.p.p.Put(context.Background(), c, c.Err() != nil) |
||||
return nil |
||||
} |
||||
|
||||
func (pc *poolConn) Err() error { |
||||
return pc.c.Err() |
||||
} |
||||
|
||||
func (pc *poolConn) Set(item *Item) (err error) { |
||||
return pc.c.SetContext(pc.ctx, item) |
||||
} |
||||
|
||||
func (pc *poolConn) Add(item *Item) (err error) { |
||||
return pc.AddContext(pc.ctx, item) |
||||
} |
||||
|
||||
func (pc *poolConn) Replace(item *Item) (err error) { |
||||
return pc.ReplaceContext(pc.ctx, item) |
||||
} |
||||
|
||||
func (pc *poolConn) CompareAndSwap(item *Item) (err error) { |
||||
return pc.CompareAndSwapContext(pc.ctx, item) |
||||
} |
||||
|
||||
func (pc *poolConn) Get(key string) (r *Item, err error) { |
||||
return pc.c.GetContext(pc.ctx, key) |
||||
} |
||||
|
||||
func (pc *poolConn) GetMulti(keys []string) (res map[string]*Item, err error) { |
||||
return pc.c.GetMultiContext(pc.ctx, keys) |
||||
} |
||||
|
||||
func (pc *poolConn) Touch(key string, timeout int32) (err error) { |
||||
return pc.c.TouchContext(pc.ctx, key, timeout) |
||||
} |
||||
|
||||
func (pc *poolConn) Scan(item *Item, v interface{}) error { |
||||
return pc.c.Scan(item, v) |
||||
} |
||||
|
||||
func (pc *poolConn) Delete(key string) (err error) { |
||||
return pc.c.DeleteContext(pc.ctx, key) |
||||
} |
||||
|
||||
func (pc *poolConn) Increment(key string, delta uint64) (newValue uint64, err error) { |
||||
return pc.c.IncrementContext(pc.ctx, key, delta) |
||||
} |
||||
|
||||
func (pc *poolConn) Decrement(key string, delta uint64) (newValue uint64, err error) { |
||||
return pc.c.DecrementContext(pc.ctx, key, delta) |
||||
} |
||||
|
||||
func (pc *poolConn) AddContext(ctx context.Context, item *Item) error { |
||||
now := time.Now() |
||||
err := pc.c.AddContext(ctx, item) |
||||
pstat("memcache:add", now, err) |
||||
return err |
||||
} |
||||
|
||||
func (pc *poolConn) SetContext(ctx context.Context, item *Item) error { |
||||
now := time.Now() |
||||
err := pc.c.SetContext(ctx, item) |
||||
pstat("memcache:set", now, err) |
||||
return err |
||||
} |
||||
|
||||
func (pc *poolConn) ReplaceContext(ctx context.Context, item *Item) error { |
||||
now := time.Now() |
||||
err := pc.c.ReplaceContext(ctx, item) |
||||
pstat("memcache:replace", now, err) |
||||
return err |
||||
} |
||||
|
||||
func (pc *poolConn) GetContext(ctx context.Context, key string) (*Item, error) { |
||||
now := time.Now() |
||||
item, err := pc.c.Get(key) |
||||
pstat("memcache:get", now, err) |
||||
return item, err |
||||
} |
||||
|
||||
func (pc *poolConn) GetMultiContext(ctx context.Context, keys []string) (map[string]*Item, error) { |
||||
// if keys is empty slice returns empty map direct
|
||||
if len(keys) == 0 { |
||||
return make(map[string]*Item), nil |
||||
} |
||||
now := time.Now() |
||||
items, err := pc.c.GetMulti(keys) |
||||
pstat("memcache:gets", now, err) |
||||
return items, err |
||||
} |
||||
|
||||
func (pc *poolConn) DeleteContext(ctx context.Context, key string) error { |
||||
now := time.Now() |
||||
err := pc.c.Delete(key) |
||||
pstat("memcache:delete", now, err) |
||||
return err |
||||
} |
||||
|
||||
func (pc *poolConn) IncrementContext(ctx context.Context, key string, delta uint64) (uint64, error) { |
||||
now := time.Now() |
||||
newValue, err := pc.c.IncrementContext(ctx, key, delta) |
||||
pstat("memcache:increment", now, err) |
||||
return newValue, err |
||||
} |
||||
|
||||
func (pc *poolConn) DecrementContext(ctx context.Context, key string, delta uint64) (uint64, error) { |
||||
now := time.Now() |
||||
newValue, err := pc.c.DecrementContext(ctx, key, delta) |
||||
pstat("memcache:decrement", now, err) |
||||
return newValue, err |
||||
} |
||||
|
||||
func (pc *poolConn) CompareAndSwapContext(ctx context.Context, item *Item) error { |
||||
now := time.Now() |
||||
err := pc.c.CompareAndSwap(item) |
||||
pstat("memcache:cas", now, err) |
||||
return err |
||||
} |
||||
|
||||
func (pc *poolConn) TouchContext(ctx context.Context, key string, seconds int32) error { |
||||
now := time.Now() |
||||
err := pc.c.Touch(key, seconds) |
||||
pstat("memcache:touch", now, err) |
||||
return err |
||||
} |
||||
@ -0,0 +1,545 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"reflect" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/container/pool" |
||||
xtime "github.com/bilibili/kratos/pkg/time" |
||||
) |
||||
|
||||
var itempool = &Item{ |
||||
Key: "testpool", |
||||
Value: []byte("testpool"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
cas: 0, |
||||
} |
||||
var itempool2 = &Item{ |
||||
Key: "test_count", |
||||
Value: []byte("0"), |
||||
Flags: 0, |
||||
Expiration: 1000, |
||||
cas: 0, |
||||
} |
||||
|
||||
type testObject struct { |
||||
Mid int64 |
||||
Value []byte |
||||
} |
||||
|
||||
var largeValue = &Item{ |
||||
Key: "large_value", |
||||
Flags: FlagGOB | FlagGzip, |
||||
Expiration: 1000, |
||||
cas: 0, |
||||
} |
||||
|
||||
var largeValueBoundary = &Item{ |
||||
Key: "large_value", |
||||
Flags: FlagGOB | FlagGzip, |
||||
Expiration: 1000, |
||||
cas: 0, |
||||
} |
||||
|
||||
func TestPoolSet(t *testing.T) { |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// set
|
||||
if err := conn.Set(itempool); err != nil { |
||||
t.Errorf("memcache: set error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: set value: %s", itempool.Value) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolGet(t *testing.T) { |
||||
key := "testpool" |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// get
|
||||
if res, err := conn.Get(key); err != nil { |
||||
t.Errorf("memcache: get error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: get value: %s", res.Value) |
||||
} |
||||
if _, err := conn.Get("not_found"); err != ErrNotFound { |
||||
t.Errorf("memcache: expceted err is not found but got: %v", err) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolGetMulti(t *testing.T) { |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
s := []string{"testpool", "test1"} |
||||
// get
|
||||
if res, err := conn.GetMulti(s); err != nil { |
||||
t.Errorf("memcache: gets error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: gets value: %d", len(res)) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolTouch(t *testing.T) { |
||||
key := "testpool" |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// touch
|
||||
if err := conn.Touch(key, 10); err != nil { |
||||
t.Errorf("memcache: touch error(%v)", err) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolIncrement(t *testing.T) { |
||||
key := "test_count" |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// set
|
||||
if err := conn.Set(itempool2); err != nil { |
||||
t.Errorf("memcache: set error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: set value: 0") |
||||
} |
||||
// incr
|
||||
if res, err := conn.Increment(key, 1); err != nil { |
||||
t.Errorf("memcache: incr error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: incr n: %d", res) |
||||
if res != 1 { |
||||
t.Errorf("memcache: expected res=1 but got %d", res) |
||||
} |
||||
} |
||||
// decr
|
||||
if res, err := conn.Decrement(key, 1); err != nil { |
||||
t.Errorf("memcache: decr error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: decr n: %d", res) |
||||
if res != 0 { |
||||
t.Errorf("memcache: expected res=0 but got %d", res) |
||||
} |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolErr(t *testing.T) { |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
if err := conn.Err(); err == nil { |
||||
t.Errorf("memcache: err not nil") |
||||
} else { |
||||
t.Logf("memcache: err: %v", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolCompareAndSwap(t *testing.T) { |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
key := "testpool" |
||||
//cas
|
||||
if r, err := conn.Get(key); err != nil { |
||||
t.Errorf("conn.Get() error(%v)", err) |
||||
} else { |
||||
r.Value = []byte("shit") |
||||
if err := conn.CompareAndSwap(r); err != nil { |
||||
t.Errorf("conn.Get() error(%v)", err) |
||||
} |
||||
r, _ := conn.Get("testpool") |
||||
if r.Key != "testpool" || !bytes.Equal(r.Value, []byte("shit")) || r.Flags != 0 { |
||||
t.Error("conn.Get() error, value") |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestPoolDel(t *testing.T) { |
||||
key := "testpool" |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// delete
|
||||
if err := conn.Delete(key); err != nil { |
||||
t.Errorf("memcache: delete error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: delete key: %s", key) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkMemcache(b *testing.B) { |
||||
c := &Config{ |
||||
Name: "test", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
} |
||||
c.Config = &pool.Config{ |
||||
Active: 10, |
||||
Idle: 5, |
||||
IdleTimeout: xtime.Duration(90 * time.Second), |
||||
} |
||||
testPool = NewPool(c) |
||||
b.ResetTimer() |
||||
b.RunParallel(func(pb *testing.PB) { |
||||
for pb.Next() { |
||||
conn := testPool.Get(context.Background()) |
||||
if err := conn.Close(); err != nil { |
||||
b.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
}) |
||||
if err := testPool.Close(); err != nil { |
||||
b.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolSetLargeValue(t *testing.T) { |
||||
var b bytes.Buffer |
||||
for i := 0; i < 4000000; i++ { |
||||
b.WriteByte(1) |
||||
} |
||||
obj := &testObject{} |
||||
obj.Mid = 1000 |
||||
obj.Value = b.Bytes() |
||||
largeValue.Object = obj |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// set
|
||||
if err := conn.Set(largeValue); err != nil { |
||||
t.Errorf("memcache: set error(%v)", err) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolGetLargeValue(t *testing.T) { |
||||
key := largeValue.Key |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// get
|
||||
var err error |
||||
if _, err = conn.Get(key); err != nil { |
||||
t.Errorf("memcache: large get error(%+v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolGetMultiLargeValue(t *testing.T) { |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
s := []string{largeValue.Key, largeValue.Key} |
||||
// get
|
||||
if res, err := conn.GetMulti(s); err != nil { |
||||
t.Errorf("memcache: gets error(%v)", err) |
||||
} else { |
||||
t.Logf("memcache: gets value: %d", len(res)) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolSetLargeValueBoundary(t *testing.T) { |
||||
var b bytes.Buffer |
||||
for i := 0; i < _largeValue; i++ { |
||||
b.WriteByte(1) |
||||
} |
||||
obj := &testObject{} |
||||
obj.Mid = 1000 |
||||
obj.Value = b.Bytes() |
||||
largeValueBoundary.Object = obj |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// set
|
||||
if err := conn.Set(largeValueBoundary); err != nil { |
||||
t.Errorf("memcache: set error(%v)", err) |
||||
} |
||||
if err := conn.Close(); err != nil { |
||||
t.Errorf("memcache: close error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolGetLargeValueBoundary(t *testing.T) { |
||||
key := largeValueBoundary.Key |
||||
conn := testPool.Get(context.Background()) |
||||
defer conn.Close() |
||||
// get
|
||||
var err error |
||||
if _, err = conn.Get(key); err != nil { |
||||
t.Errorf("memcache: large get error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestPoolAdd(t *testing.T) { |
||||
var ( |
||||
key = "test_add" |
||||
item = &Item{ |
||||
Key: key, |
||||
Value: []byte("0"), |
||||
Flags: 0, |
||||
Expiration: 60, |
||||
cas: 0, |
||||
} |
||||
conn = testPool.Get(context.Background()) |
||||
) |
||||
defer conn.Close() |
||||
conn.Delete(key) |
||||
if err := conn.Add(item); err != nil { |
||||
t.Errorf("memcache: add error(%v)", err) |
||||
} |
||||
if err := conn.Add(item); err != ErrNotStored { |
||||
t.Errorf("memcache: add error(%v)", err) |
||||
} |
||||
} |
||||
|
||||
func TestNewPool(t *testing.T) { |
||||
type args struct { |
||||
cfg *Config |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
wantErr error |
||||
wantPanic bool |
||||
}{ |
||||
{ |
||||
"NewPoolIllegalDialTimeout", |
||||
args{ |
||||
&Config{ |
||||
Name: "test_illegal_dial_timeout", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(-time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}, |
||||
}, |
||||
nil, |
||||
true, |
||||
}, |
||||
{ |
||||
"NewPoolIllegalReadTimeout", |
||||
args{ |
||||
&Config{ |
||||
Name: "test_illegal_read_timeout", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(-time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}, |
||||
}, |
||||
nil, |
||||
true, |
||||
}, |
||||
{ |
||||
"NewPoolIllegalWriteTimeout", |
||||
args{ |
||||
&Config{ |
||||
Name: "test_illegal_write_timeout", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(-time.Second), |
||||
}, |
||||
}, |
||||
nil, |
||||
true, |
||||
}, |
||||
{ |
||||
"NewPool", |
||||
args{ |
||||
&Config{ |
||||
Name: "test_new", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}, |
||||
}, |
||||
nil, |
||||
true, |
||||
}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
defer func() { |
||||
r := recover() |
||||
if (r != nil) != tt.wantPanic { |
||||
t.Errorf("wantPanic recover = %v, wantPanic = %v", r, tt.wantPanic) |
||||
} |
||||
}() |
||||
|
||||
if gotP := NewPool(tt.args.cfg); gotP == nil { |
||||
t.Error("NewPool() failed, got nil") |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestPool_Get(t *testing.T) { |
||||
|
||||
type args struct { |
||||
ctx context.Context |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
p *Pool |
||||
args args |
||||
wantErr bool |
||||
n int |
||||
}{ |
||||
{ |
||||
"Get", |
||||
NewPool(&Config{ |
||||
Config: &pool.Config{ |
||||
Active: 3, |
||||
Idle: 2, |
||||
}, |
||||
Name: "test_get", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}), |
||||
args{context.TODO()}, |
||||
false, |
||||
3, |
||||
}, |
||||
{ |
||||
"GetExceededPoolSize", |
||||
NewPool(&Config{ |
||||
Config: &pool.Config{ |
||||
Active: 3, |
||||
Idle: 2, |
||||
}, |
||||
Name: "test_get_out", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}), |
||||
args{context.TODO()}, |
||||
true, |
||||
6, |
||||
}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
for i := 1; i <= tt.n; i++ { |
||||
got := tt.p.Get(tt.args.ctx) |
||||
if reflect.TypeOf(got) == reflect.TypeOf(errConn{}) { |
||||
if !tt.wantErr { |
||||
t.Errorf("got errConn, export Conn") |
||||
} |
||||
return |
||||
} else { |
||||
if tt.wantErr { |
||||
if i > tt.p.c.Active { |
||||
t.Errorf("got Conn, export errConn") |
||||
} |
||||
} |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestPool_Close(t *testing.T) { |
||||
|
||||
type args struct { |
||||
ctx context.Context |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
p *Pool |
||||
args args |
||||
wantErr bool |
||||
g int |
||||
c int |
||||
}{ |
||||
{ |
||||
"Close", |
||||
NewPool(&Config{ |
||||
Config: &pool.Config{ |
||||
Active: 1, |
||||
Idle: 1, |
||||
}, |
||||
Name: "test_get", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}), |
||||
args{context.TODO()}, |
||||
false, |
||||
3, |
||||
3, |
||||
}, |
||||
{ |
||||
"CloseExceededPoolSize", |
||||
NewPool(&Config{ |
||||
Config: &pool.Config{ |
||||
Active: 1, |
||||
Idle: 1, |
||||
}, |
||||
Name: "test_get_out", |
||||
Proto: "tcp", |
||||
Addr: testMemcacheAddr, |
||||
DialTimeout: xtime.Duration(time.Second), |
||||
ReadTimeout: xtime.Duration(time.Second), |
||||
WriteTimeout: xtime.Duration(time.Second), |
||||
}), |
||||
args{context.TODO()}, |
||||
true, |
||||
5, |
||||
3, |
||||
}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
for i := 1; i <= tt.g; i++ { |
||||
got := tt.p.Get(tt.args.ctx) |
||||
if err := got.Close(); err != nil { |
||||
if !tt.wantErr { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
if i <= tt.c { |
||||
if err := got.Close(); err != nil { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,48 @@ |
||||
load( |
||||
"@io_bazel_rules_go//go:def.bzl", |
||||
"go_library", |
||||
) |
||||
load( |
||||
"@io_bazel_rules_go//proto:def.bzl", |
||||
"go_proto_library", |
||||
) |
||||
|
||||
go_library( |
||||
name = "go_default_library", |
||||
srcs = [], |
||||
embed = [":proto_go_proto"], |
||||
importpath = "go-common/library/cache/memcache/test", |
||||
tags = ["automanaged"], |
||||
visibility = ["//visibility:public"], |
||||
deps = ["@com_github_golang_protobuf//proto:go_default_library"], |
||||
) |
||||
|
||||
filegroup( |
||||
name = "package-srcs", |
||||
srcs = glob(["**"]), |
||||
tags = ["automanaged"], |
||||
visibility = ["//visibility:private"], |
||||
) |
||||
|
||||
filegroup( |
||||
name = "all-srcs", |
||||
srcs = [":package-srcs"], |
||||
tags = ["automanaged"], |
||||
visibility = ["//visibility:public"], |
||||
) |
||||
|
||||
proto_library( |
||||
name = "test_proto", |
||||
srcs = ["test.proto"], |
||||
import_prefix = "go-common/library/cache/memcache/test", |
||||
strip_import_prefix = "", |
||||
tags = ["automanaged"], |
||||
) |
||||
|
||||
go_proto_library( |
||||
name = "proto_go_proto", |
||||
compilers = ["@io_bazel_rules_go//proto:go_proto"], |
||||
importpath = "go-common/library/cache/memcache/test", |
||||
proto = ":test_proto", |
||||
tags = ["automanaged"], |
||||
) |
||||
@ -0,0 +1,375 @@ |
||||
// Code generated by protoc-gen-gogo. DO NOT EDIT.
|
||||
// source: test.proto
|
||||
|
||||
/* |
||||
Package proto is a generated protocol buffer package. |
||||
|
||||
It is generated from these files: |
||||
test.proto |
||||
|
||||
It has these top-level messages: |
||||
TestItem |
||||
*/ |
||||
package proto |
||||
|
||||
import proto1 "github.com/golang/protobuf/proto" |
||||
import fmt "fmt" |
||||
import math "math" |
||||
|
||||
import io "io" |
||||
|
||||
// Reference imports to suppress errors if they are not otherwise used.
|
||||
var _ = proto1.Marshal |
||||
var _ = fmt.Errorf |
||||
var _ = math.Inf |
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the proto package it is being compiled against.
|
||||
// A compilation error at this line likely means your copy of the
|
||||
// proto package needs to be updated.
|
||||
const _ = proto1.ProtoPackageIsVersion2 // please upgrade the proto package
|
||||
|
||||
type FOO int32 |
||||
|
||||
const ( |
||||
FOO_X FOO = 0 |
||||
) |
||||
|
||||
var FOO_name = map[int32]string{ |
||||
0: "X", |
||||
} |
||||
var FOO_value = map[string]int32{ |
||||
"X": 0, |
||||
} |
||||
|
||||
func (x FOO) String() string { |
||||
return proto1.EnumName(FOO_name, int32(x)) |
||||
} |
||||
func (FOO) EnumDescriptor() ([]byte, []int) { return fileDescriptorTest, []int{0} } |
||||
|
||||
type TestItem struct { |
||||
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` |
||||
Age int32 `protobuf:"varint,2,opt,name=Age,proto3" json:"Age,omitempty"` |
||||
} |
||||
|
||||
func (m *TestItem) Reset() { *m = TestItem{} } |
||||
func (m *TestItem) String() string { return proto1.CompactTextString(m) } |
||||
func (*TestItem) ProtoMessage() {} |
||||
func (*TestItem) Descriptor() ([]byte, []int) { return fileDescriptorTest, []int{0} } |
||||
|
||||
func (m *TestItem) GetName() string { |
||||
if m != nil { |
||||
return m.Name |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func (m *TestItem) GetAge() int32 { |
||||
if m != nil { |
||||
return m.Age |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func init() { |
||||
proto1.RegisterType((*TestItem)(nil), "proto.TestItem") |
||||
proto1.RegisterEnum("proto.FOO", FOO_name, FOO_value) |
||||
} |
||||
func (m *TestItem) Marshal() (dAtA []byte, err error) { |
||||
size := m.Size() |
||||
dAtA = make([]byte, size) |
||||
n, err := m.MarshalTo(dAtA) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return dAtA[:n], nil |
||||
} |
||||
|
||||
func (m *TestItem) MarshalTo(dAtA []byte) (int, error) { |
||||
var i int |
||||
_ = i |
||||
var l int |
||||
_ = l |
||||
if len(m.Name) > 0 { |
||||
dAtA[i] = 0xa |
||||
i++ |
||||
i = encodeVarintTest(dAtA, i, uint64(len(m.Name))) |
||||
i += copy(dAtA[i:], m.Name) |
||||
} |
||||
if m.Age != 0 { |
||||
dAtA[i] = 0x10 |
||||
i++ |
||||
i = encodeVarintTest(dAtA, i, uint64(m.Age)) |
||||
} |
||||
return i, nil |
||||
} |
||||
|
||||
func encodeFixed64Test(dAtA []byte, offset int, v uint64) int { |
||||
dAtA[offset] = uint8(v) |
||||
dAtA[offset+1] = uint8(v >> 8) |
||||
dAtA[offset+2] = uint8(v >> 16) |
||||
dAtA[offset+3] = uint8(v >> 24) |
||||
dAtA[offset+4] = uint8(v >> 32) |
||||
dAtA[offset+5] = uint8(v >> 40) |
||||
dAtA[offset+6] = uint8(v >> 48) |
||||
dAtA[offset+7] = uint8(v >> 56) |
||||
return offset + 8 |
||||
} |
||||
func encodeFixed32Test(dAtA []byte, offset int, v uint32) int { |
||||
dAtA[offset] = uint8(v) |
||||
dAtA[offset+1] = uint8(v >> 8) |
||||
dAtA[offset+2] = uint8(v >> 16) |
||||
dAtA[offset+3] = uint8(v >> 24) |
||||
return offset + 4 |
||||
} |
||||
func encodeVarintTest(dAtA []byte, offset int, v uint64) int { |
||||
for v >= 1<<7 { |
||||
dAtA[offset] = uint8(v&0x7f | 0x80) |
||||
v >>= 7 |
||||
offset++ |
||||
} |
||||
dAtA[offset] = uint8(v) |
||||
return offset + 1 |
||||
} |
||||
func (m *TestItem) Size() (n int) { |
||||
var l int |
||||
_ = l |
||||
l = len(m.Name) |
||||
if l > 0 { |
||||
n += 1 + l + sovTest(uint64(l)) |
||||
} |
||||
if m.Age != 0 { |
||||
n += 1 + sovTest(uint64(m.Age)) |
||||
} |
||||
return n |
||||
} |
||||
|
||||
func sovTest(x uint64) (n int) { |
||||
for { |
||||
n++ |
||||
x >>= 7 |
||||
if x == 0 { |
||||
break |
||||
} |
||||
} |
||||
return n |
||||
} |
||||
func sozTest(x uint64) (n int) { |
||||
return sovTest(uint64((x << 1) ^ uint64((int64(x) >> 63)))) |
||||
} |
||||
func (m *TestItem) Unmarshal(dAtA []byte) error { |
||||
l := len(dAtA) |
||||
iNdEx := 0 |
||||
for iNdEx < l { |
||||
preIndex := iNdEx |
||||
var wire uint64 |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
b := dAtA[iNdEx] |
||||
iNdEx++ |
||||
wire |= (uint64(b) & 0x7F) << shift |
||||
if b < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
fieldNum := int32(wire >> 3) |
||||
wireType := int(wire & 0x7) |
||||
if wireType == 4 { |
||||
return fmt.Errorf("proto: TestItem: wiretype end group for non-group") |
||||
} |
||||
if fieldNum <= 0 { |
||||
return fmt.Errorf("proto: TestItem: illegal tag %d (wire type %d)", fieldNum, wire) |
||||
} |
||||
switch fieldNum { |
||||
case 1: |
||||
if wireType != 2 { |
||||
return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) |
||||
} |
||||
var stringLen uint64 |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
b := dAtA[iNdEx] |
||||
iNdEx++ |
||||
stringLen |= (uint64(b) & 0x7F) << shift |
||||
if b < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
intStringLen := int(stringLen) |
||||
if intStringLen < 0 { |
||||
return ErrInvalidLengthTest |
||||
} |
||||
postIndex := iNdEx + intStringLen |
||||
if postIndex > l { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
m.Name = string(dAtA[iNdEx:postIndex]) |
||||
iNdEx = postIndex |
||||
case 2: |
||||
if wireType != 0 { |
||||
return fmt.Errorf("proto: wrong wireType = %d for field Age", wireType) |
||||
} |
||||
m.Age = 0 |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
b := dAtA[iNdEx] |
||||
iNdEx++ |
||||
m.Age |= (int32(b) & 0x7F) << shift |
||||
if b < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
default: |
||||
iNdEx = preIndex |
||||
skippy, err := skipTest(dAtA[iNdEx:]) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if skippy < 0 { |
||||
return ErrInvalidLengthTest |
||||
} |
||||
if (iNdEx + skippy) > l { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
iNdEx += skippy |
||||
} |
||||
} |
||||
|
||||
if iNdEx > l { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
return nil |
||||
} |
||||
func skipTest(dAtA []byte) (n int, err error) { |
||||
l := len(dAtA) |
||||
iNdEx := 0 |
||||
for iNdEx < l { |
||||
var wire uint64 |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return 0, ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return 0, io.ErrUnexpectedEOF |
||||
} |
||||
b := dAtA[iNdEx] |
||||
iNdEx++ |
||||
wire |= (uint64(b) & 0x7F) << shift |
||||
if b < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
wireType := int(wire & 0x7) |
||||
switch wireType { |
||||
case 0: |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return 0, ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return 0, io.ErrUnexpectedEOF |
||||
} |
||||
iNdEx++ |
||||
if dAtA[iNdEx-1] < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
return iNdEx, nil |
||||
case 1: |
||||
iNdEx += 8 |
||||
return iNdEx, nil |
||||
case 2: |
||||
var length int |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return 0, ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return 0, io.ErrUnexpectedEOF |
||||
} |
||||
b := dAtA[iNdEx] |
||||
iNdEx++ |
||||
length |= (int(b) & 0x7F) << shift |
||||
if b < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
iNdEx += length |
||||
if length < 0 { |
||||
return 0, ErrInvalidLengthTest |
||||
} |
||||
return iNdEx, nil |
||||
case 3: |
||||
for { |
||||
var innerWire uint64 |
||||
var start int = iNdEx |
||||
for shift := uint(0); ; shift += 7 { |
||||
if shift >= 64 { |
||||
return 0, ErrIntOverflowTest |
||||
} |
||||
if iNdEx >= l { |
||||
return 0, io.ErrUnexpectedEOF |
||||
} |
||||
b := dAtA[iNdEx] |
||||
iNdEx++ |
||||
innerWire |= (uint64(b) & 0x7F) << shift |
||||
if b < 0x80 { |
||||
break |
||||
} |
||||
} |
||||
innerWireType := int(innerWire & 0x7) |
||||
if innerWireType == 4 { |
||||
break |
||||
} |
||||
next, err := skipTest(dAtA[start:]) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
iNdEx = start + next |
||||
} |
||||
return iNdEx, nil |
||||
case 4: |
||||
return iNdEx, nil |
||||
case 5: |
||||
iNdEx += 4 |
||||
return iNdEx, nil |
||||
default: |
||||
return 0, fmt.Errorf("proto: illegal wireType %d", wireType) |
||||
} |
||||
} |
||||
panic("unreachable") |
||||
} |
||||
|
||||
var ( |
||||
ErrInvalidLengthTest = fmt.Errorf("proto: negative length found during unmarshaling") |
||||
ErrIntOverflowTest = fmt.Errorf("proto: integer overflow") |
||||
) |
||||
|
||||
func init() { proto1.RegisterFile("test.proto", fileDescriptorTest) } |
||||
|
||||
var fileDescriptorTest = []byte{ |
||||
// 122 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2a, 0x49, 0x2d, 0x2e, |
||||
0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0x06, 0x5c, 0x1c, 0x21, 0xa9, |
||||
0xc5, 0x25, 0x9e, 0x25, 0xa9, 0xb9, 0x42, 0x42, 0x5c, 0x2c, 0x7e, 0x89, 0xb9, 0xa9, 0x12, 0x8c, |
||||
0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x60, 0xb6, 0x90, 0x00, 0x17, 0xb3, 0x63, 0x7a, 0xaa, 0x04, 0x93, |
||||
0x02, 0xa3, 0x06, 0x6b, 0x10, 0x88, 0xa9, 0xc5, 0xc3, 0xc5, 0xec, 0xe6, 0xef, 0x2f, 0xc4, 0xca, |
||||
0xc5, 0x18, 0x21, 0xc0, 0xe0, 0x24, 0x70, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, |
||||
0x1e, 0xc9, 0x31, 0xce, 0x78, 0x2c, 0xc7, 0x90, 0xc4, 0x06, 0x36, 0xd8, 0x18, 0x10, 0x00, 0x00, |
||||
0xff, 0xff, 0x16, 0x80, 0x60, 0x15, 0x6d, 0x00, 0x00, 0x00, |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
syntax = "proto3"; |
||||
package proto; |
||||
|
||||
enum FOO |
||||
{ |
||||
X = 0; |
||||
}; |
||||
|
||||
message TestItem{ |
||||
string Name = 1; |
||||
int32 Age = 2; |
||||
} |
||||
@ -1,109 +0,0 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/log" |
||||
"github.com/bilibili/kratos/pkg/net/trace" |
||||
) |
||||
|
||||
const ( |
||||
_traceFamily = "memcache" |
||||
_traceSpanKind = "client" |
||||
_traceComponentName = "library/cache/memcache" |
||||
_tracePeerService = "memcache" |
||||
_slowLogDuration = time.Millisecond * 250 |
||||
) |
||||
|
||||
type traceConn struct { |
||||
Conn |
||||
ctx context.Context |
||||
address string |
||||
} |
||||
|
||||
func (t *traceConn) setTrace(action, statement string) func(error) error { |
||||
now := time.Now() |
||||
parent, ok := trace.FromContext(t.ctx) |
||||
if !ok { |
||||
return func(err error) error { return err } |
||||
} |
||||
span := parent.Fork(_traceFamily, "Memcache:"+action) |
||||
span.SetTag( |
||||
trace.String(trace.TagSpanKind, _traceSpanKind), |
||||
trace.String(trace.TagComponent, _traceComponentName), |
||||
trace.String(trace.TagPeerService, _tracePeerService), |
||||
trace.String(trace.TagPeerAddress, t.address), |
||||
trace.String(trace.TagDBStatement, action+" "+statement), |
||||
) |
||||
return func(err error) error { |
||||
span.Finish(&err) |
||||
t := time.Since(now) |
||||
if t > _slowLogDuration { |
||||
log.Warn("%s slow log action: %s key: %s time: %v", _traceFamily, action, statement, t) |
||||
} |
||||
return err |
||||
} |
||||
} |
||||
|
||||
func (t *traceConn) WithContext(ctx context.Context) Conn { |
||||
t.ctx = ctx |
||||
t.Conn = t.Conn.WithContext(ctx) |
||||
return t |
||||
} |
||||
|
||||
func (t *traceConn) Add(item *Item) error { |
||||
finishFn := t.setTrace("Add", item.Key) |
||||
return finishFn(t.Conn.Add(item)) |
||||
} |
||||
|
||||
func (t *traceConn) Set(item *Item) error { |
||||
finishFn := t.setTrace("Set", item.Key) |
||||
return finishFn(t.Conn.Set(item)) |
||||
} |
||||
|
||||
func (t *traceConn) Replace(item *Item) error { |
||||
finishFn := t.setTrace("Replace", item.Key) |
||||
return finishFn(t.Conn.Replace(item)) |
||||
} |
||||
|
||||
func (t *traceConn) Get(key string) (*Item, error) { |
||||
finishFn := t.setTrace("Get", key) |
||||
item, err := t.Conn.Get(key) |
||||
return item, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) GetMulti(keys []string) (map[string]*Item, error) { |
||||
finishFn := t.setTrace("GetMulti", strings.Join(keys, " ")) |
||||
items, err := t.Conn.GetMulti(keys) |
||||
return items, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) Delete(key string) error { |
||||
finishFn := t.setTrace("Delete", key) |
||||
return finishFn(t.Conn.Delete(key)) |
||||
} |
||||
|
||||
func (t *traceConn) Increment(key string, delta uint64) (newValue uint64, err error) { |
||||
finishFn := t.setTrace("Increment", key+" "+strconv.FormatUint(delta, 10)) |
||||
newValue, err = t.Conn.Increment(key, delta) |
||||
return newValue, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) Decrement(key string, delta uint64) (newValue uint64, err error) { |
||||
finishFn := t.setTrace("Decrement", key+" "+strconv.FormatUint(delta, 10)) |
||||
newValue, err = t.Conn.Decrement(key, delta) |
||||
return newValue, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) CompareAndSwap(item *Item) error { |
||||
finishFn := t.setTrace("CompareAndSwap", item.Key) |
||||
return finishFn(t.Conn.CompareAndSwap(item)) |
||||
} |
||||
|
||||
func (t *traceConn) Touch(key string, seconds int32) (err error) { |
||||
finishFn := t.setTrace("Touch", key+" "+strconv.Itoa(int(seconds))) |
||||
return finishFn(t.Conn.Touch(key, seconds)) |
||||
} |
||||
@ -0,0 +1,103 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"context" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/log" |
||||
"github.com/bilibili/kratos/pkg/net/trace" |
||||
) |
||||
|
||||
const ( |
||||
_slowLogDuration = time.Millisecond * 250 |
||||
) |
||||
|
||||
func newTraceConn(conn Conn, address string) Conn { |
||||
tags := []trace.Tag{ |
||||
trace.String(trace.TagSpanKind, "client"), |
||||
trace.String(trace.TagComponent, "cache/memcache"), |
||||
trace.String(trace.TagPeerService, "memcache"), |
||||
trace.String(trace.TagPeerAddress, address), |
||||
} |
||||
return &traceConn{Conn: conn, tags: tags} |
||||
} |
||||
|
||||
type traceConn struct { |
||||
Conn |
||||
tags []trace.Tag |
||||
} |
||||
|
||||
func (t *traceConn) setTrace(ctx context.Context, action, statement string) func(error) error { |
||||
now := time.Now() |
||||
parent, ok := trace.FromContext(ctx) |
||||
if !ok { |
||||
return func(err error) error { return err } |
||||
} |
||||
span := parent.Fork("", "Memcache:"+action) |
||||
span.SetTag(t.tags...) |
||||
span.SetTag(trace.String(trace.TagDBStatement, action+" "+statement)) |
||||
return func(err error) error { |
||||
span.Finish(&err) |
||||
t := time.Since(now) |
||||
if t > _slowLogDuration { |
||||
log.Warn("memcache slow log action: %s key: %s time: %v", action, statement, t) |
||||
} |
||||
return err |
||||
} |
||||
} |
||||
|
||||
func (t *traceConn) AddContext(ctx context.Context, item *Item) error { |
||||
finishFn := t.setTrace(ctx, "Add", item.Key) |
||||
return finishFn(t.Conn.Add(item)) |
||||
} |
||||
|
||||
func (t *traceConn) SetContext(ctx context.Context, item *Item) error { |
||||
finishFn := t.setTrace(ctx, "Set", item.Key) |
||||
return finishFn(t.Conn.Set(item)) |
||||
} |
||||
|
||||
func (t *traceConn) ReplaceContext(ctx context.Context, item *Item) error { |
||||
finishFn := t.setTrace(ctx, "Replace", item.Key) |
||||
return finishFn(t.Conn.Replace(item)) |
||||
} |
||||
|
||||
func (t *traceConn) GetContext(ctx context.Context, key string) (*Item, error) { |
||||
finishFn := t.setTrace(ctx, "Get", key) |
||||
item, err := t.Conn.Get(key) |
||||
return item, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) GetMultiContext(ctx context.Context, keys []string) (map[string]*Item, error) { |
||||
finishFn := t.setTrace(ctx, "GetMulti", strings.Join(keys, " ")) |
||||
items, err := t.Conn.GetMulti(keys) |
||||
return items, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) DeleteContext(ctx context.Context, key string) error { |
||||
finishFn := t.setTrace(ctx, "Delete", key) |
||||
return finishFn(t.Conn.Delete(key)) |
||||
} |
||||
|
||||
func (t *traceConn) IncrementContext(ctx context.Context, key string, delta uint64) (newValue uint64, err error) { |
||||
finishFn := t.setTrace(ctx, "Increment", key+" "+strconv.FormatUint(delta, 10)) |
||||
newValue, err = t.Conn.Increment(key, delta) |
||||
return newValue, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) DecrementContext(ctx context.Context, key string, delta uint64) (newValue uint64, err error) { |
||||
finishFn := t.setTrace(ctx, "Decrement", key+" "+strconv.FormatUint(delta, 10)) |
||||
newValue, err = t.Conn.Decrement(key, delta) |
||||
return newValue, finishFn(err) |
||||
} |
||||
|
||||
func (t *traceConn) CompareAndSwapContext(ctx context.Context, item *Item) error { |
||||
finishFn := t.setTrace(ctx, "CompareAndSwap", item.Key) |
||||
return finishFn(t.Conn.CompareAndSwap(item)) |
||||
} |
||||
|
||||
func (t *traceConn) TouchContext(ctx context.Context, key string, seconds int32) (err error) { |
||||
finishFn := t.setTrace(ctx, "Touch", key+" "+strconv.Itoa(int(seconds))) |
||||
return finishFn(t.Conn.Touch(key, seconds)) |
||||
} |
||||
@ -0,0 +1,75 @@ |
||||
package memcache |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
pb "github.com/bilibili/kratos/pkg/cache/memcache/test" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestItemUtil(t *testing.T) { |
||||
item1 := RawItem("test", []byte("hh"), 0, 0) |
||||
assert.Equal(t, "test", item1.Key) |
||||
assert.Equal(t, []byte("hh"), item1.Value) |
||||
assert.Equal(t, FlagRAW, FlagRAW&item1.Flags) |
||||
|
||||
item1 = JSONItem("test", &Item{}, 0, 0) |
||||
assert.Equal(t, "test", item1.Key) |
||||
assert.NotNil(t, item1.Object) |
||||
assert.Equal(t, FlagJSON, FlagJSON&item1.Flags) |
||||
|
||||
item1 = ProtobufItem("test", &pb.TestItem{}, 0, 0) |
||||
assert.Equal(t, "test", item1.Key) |
||||
assert.NotNil(t, item1.Object) |
||||
assert.Equal(t, FlagProtobuf, FlagProtobuf&item1.Flags) |
||||
} |
||||
|
||||
func TestLegalKey(t *testing.T) { |
||||
type args struct { |
||||
key string |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
want bool |
||||
}{ |
||||
{ |
||||
name: "test empty key", |
||||
want: false, |
||||
}, |
||||
{ |
||||
name: "test too large key", |
||||
args: args{func() string { |
||||
var data []byte |
||||
for i := 0; i < 255; i++ { |
||||
data = append(data, 'k') |
||||
} |
||||
return string(data) |
||||
}()}, |
||||
want: false, |
||||
}, |
||||
{ |
||||
name: "test invalid char", |
||||
args: args{"hello world"}, |
||||
want: false, |
||||
}, |
||||
{ |
||||
name: "test invalid char", |
||||
args: args{string([]byte{0x7f})}, |
||||
want: false, |
||||
}, |
||||
{ |
||||
name: "test normal key", |
||||
args: args{"hello"}, |
||||
want: true, |
||||
}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if got := legalKey(tt.args.key); got != tt.want { |
||||
t.Errorf("legalKey() = %v, want %v", got, tt.want) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
#### group |
||||
|
||||
##### 项目简介 |
||||
|
||||
懒加载对象容器 |
||||
|
||||
##### 编译环境 |
||||
|
||||
- **推荐 Golang v1.12.1 以上版本编译执行** |
||||
|
||||
##### 依赖包 |
||||
|
||||
@ -0,0 +1,46 @@ |
||||
package group |
||||
|
||||
import "fmt" |
||||
|
||||
type Counter struct { |
||||
Value int |
||||
} |
||||
|
||||
func (c *Counter) Incr() { |
||||
c.Value++ |
||||
} |
||||
|
||||
func ExampleGroup_Get() { |
||||
new := func() interface{} { |
||||
fmt.Println("Only Once") |
||||
return &Counter{} |
||||
} |
||||
group := NewGroup(new) |
||||
|
||||
// Create a new Counter
|
||||
group.Get("pass").(*Counter).Incr() |
||||
|
||||
// Get the created Counter again.
|
||||
group.Get("pass").(*Counter).Incr() |
||||
// Output:
|
||||
// Only Once
|
||||
} |
||||
|
||||
func ExampleGroup_Reset() { |
||||
new := func() interface{} { |
||||
return &Counter{} |
||||
} |
||||
group := NewGroup(new) |
||||
|
||||
newV2 := func() interface{} { |
||||
fmt.Println("New V2") |
||||
return &Counter{} |
||||
} |
||||
// Reset the new function and clear all created objects.
|
||||
group.Reset(newV2) |
||||
|
||||
// Create a new Counter
|
||||
group.Get("pass").(*Counter).Incr() |
||||
// Output:
|
||||
// New V2
|
||||
} |
||||
@ -0,0 +1,55 @@ |
||||
// Package group provides a sample lazy load container.
|
||||
// The group only creating a new object not until the object is needed by user.
|
||||
// And it will cache all the objects to reduce the creation of object.
|
||||
package group |
||||
|
||||
import "sync" |
||||
|
||||
// Group is a lazy load container.
|
||||
type Group struct { |
||||
new func() interface{} |
||||
objs sync.Map |
||||
sync.RWMutex |
||||
} |
||||
|
||||
// NewGroup news a group container.
|
||||
func NewGroup(new func() interface{}) *Group { |
||||
if new == nil { |
||||
panic("container.group: can't assign a nil to the new function") |
||||
} |
||||
return &Group{ |
||||
new: new, |
||||
} |
||||
} |
||||
|
||||
// Get gets the object by the given key.
|
||||
func (g *Group) Get(key string) interface{} { |
||||
g.RLock() |
||||
new := g.new |
||||
g.RUnlock() |
||||
obj, ok := g.objs.Load(key) |
||||
if !ok { |
||||
obj = new() |
||||
g.objs.Store(key, obj) |
||||
} |
||||
return obj |
||||
} |
||||
|
||||
// Reset resets the new function and deletes all existing objects.
|
||||
func (g *Group) Reset(new func() interface{}) { |
||||
if new == nil { |
||||
panic("container.group: can't assign a nil to the new function") |
||||
} |
||||
g.Lock() |
||||
g.new = new |
||||
g.Unlock() |
||||
g.Clear() |
||||
} |
||||
|
||||
// Clear deletes all objects.
|
||||
func (g *Group) Clear() { |
||||
g.objs.Range(func(key, value interface{}) bool { |
||||
g.objs.Delete(key) |
||||
return true |
||||
}) |
||||
} |
||||
@ -0,0 +1,69 @@ |
||||
package group |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestGroupGet(t *testing.T) { |
||||
count := 0 |
||||
g := NewGroup(func() interface{} { |
||||
count++ |
||||
return count |
||||
}) |
||||
v := g.Get("/x/internal/dummy/user") |
||||
assert.Equal(t, 1, v.(int)) |
||||
|
||||
v = g.Get("/x/internal/dummy/avatar") |
||||
assert.Equal(t, 2, v.(int)) |
||||
|
||||
v = g.Get("/x/internal/dummy/user") |
||||
assert.Equal(t, 1, v.(int)) |
||||
assert.Equal(t, 2, count) |
||||
|
||||
} |
||||
|
||||
func TestGroupReset(t *testing.T) { |
||||
g := NewGroup(func() interface{} { |
||||
return 1 |
||||
}) |
||||
g.Get("/x/internal/dummy/user") |
||||
call := false |
||||
g.Reset(func() interface{} { |
||||
call = true |
||||
return 1 |
||||
}) |
||||
|
||||
length := 0 |
||||
g.objs.Range(func(_, _ interface{}) bool { |
||||
length++ |
||||
return true |
||||
}) |
||||
assert.Equal(t, 0, length) |
||||
|
||||
g.Get("/x/internal/dummy/user") |
||||
assert.Equal(t, true, call) |
||||
} |
||||
|
||||
func TestGroupClear(t *testing.T) { |
||||
g := NewGroup(func() interface{} { |
||||
return 1 |
||||
}) |
||||
g.Get("/x/internal/dummy/user") |
||||
length := 0 |
||||
g.objs.Range(func(_, _ interface{}) bool { |
||||
length++ |
||||
return true |
||||
}) |
||||
assert.Equal(t, 1, length) |
||||
|
||||
g.Clear() |
||||
length = 0 |
||||
g.objs.Range(func(_, _ interface{}) bool { |
||||
length++ |
||||
return true |
||||
}) |
||||
assert.Equal(t, 0, length) |
||||
|
||||
} |
||||
@ -0,0 +1,50 @@ |
||||
package log |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"strconv" |
||||
"strings" |
||||
) |
||||
|
||||
type verboseModule map[string]int32 |
||||
|
||||
type logFilter []string |
||||
|
||||
func (f *logFilter) String() string { |
||||
return fmt.Sprint(*f) |
||||
} |
||||
|
||||
// Set sets the value of the named command-line flag.
|
||||
// format: -log.filter key1,key2
|
||||
func (f *logFilter) Set(value string) error { |
||||
for _, i := range strings.Split(value, ",") { |
||||
*f = append(*f, strings.TrimSpace(i)) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (m verboseModule) String() string { |
||||
// FIXME strings.Builder
|
||||
var buf bytes.Buffer |
||||
for k, v := range m { |
||||
buf.WriteString(k) |
||||
buf.WriteString(strconv.FormatInt(int64(v), 10)) |
||||
buf.WriteString(",") |
||||
} |
||||
return buf.String() |
||||
} |
||||
|
||||
// Set sets the value of the named command-line flag.
|
||||
// format: -log.module file=1,file2=2
|
||||
func (m verboseModule) Set(value string) error { |
||||
for _, i := range strings.Split(value, ",") { |
||||
kv := strings.Split(i, "=") |
||||
if len(kv) == 2 { |
||||
if v, err := strconv.ParseInt(kv[1], 10, 64); err == nil { |
||||
m[strings.TrimSpace(kv[0])] = int32(v) |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
package log |
||||
|
||||
import ( |
||||
"context" |
||||
"io/ioutil" |
||||
"os" |
||||
|
||||
"github.com/sirupsen/logrus" |
||||
) |
||||
|
||||
func init() { |
||||
redirectLogrus() |
||||
} |
||||
|
||||
func redirectLogrus() { |
||||
// FIXME: because of different stack depth call runtime.Caller will get error function name.
|
||||
logrus.AddHook(redirectHook{}) |
||||
if os.Getenv("LOGRUS_STDOUT") == "" { |
||||
logrus.SetOutput(ioutil.Discard) |
||||
} |
||||
} |
||||
|
||||
type redirectHook struct{} |
||||
|
||||
func (redirectHook) Levels() []logrus.Level { |
||||
return logrus.AllLevels |
||||
} |
||||
|
||||
func (redirectHook) Fire(entry *logrus.Entry) error { |
||||
lv := _infoLevel |
||||
var logrusLv string |
||||
var verbose int32 |
||||
switch entry.Level { |
||||
case logrus.FatalLevel, logrus.PanicLevel: |
||||
logrusLv = entry.Level.String() |
||||
fallthrough |
||||
case logrus.ErrorLevel: |
||||
lv = _errorLevel |
||||
case logrus.WarnLevel: |
||||
lv = _warnLevel |
||||
case logrus.InfoLevel: |
||||
lv = _infoLevel |
||||
case logrus.DebugLevel: |
||||
// use verbose log replace of debuglevel
|
||||
verbose = 10 |
||||
} |
||||
args := make([]D, 0, len(entry.Data)+1) |
||||
args = append(args, D{Key: _log, Value: entry.Message}) |
||||
for k, v := range entry.Data { |
||||
args = append(args, D{Key: k, Value: v}) |
||||
} |
||||
if logrusLv != "" { |
||||
args = append(args, D{Key: "logrus_lv", Value: logrusLv}) |
||||
} |
||||
if verbose != 0 { |
||||
V(verbose).Infov(context.Background(), args...) |
||||
} else { |
||||
h.Log(context.Background(), lv, args...) |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,54 @@ |
||||
package log |
||||
|
||||
import ( |
||||
"reflect" |
||||
"strings" |
||||
"testing" |
||||
"time" |
||||
) |
||||
|
||||
func TestFuncName(t *testing.T) { |
||||
name := funcName(1) |
||||
if !strings.Contains(name, "util_test.go:11") { |
||||
t.Errorf("expect contains util_test.go:11 got %s", name) |
||||
} |
||||
} |
||||
|
||||
func Test_toMap(t *testing.T) { |
||||
type args struct { |
||||
args []D |
||||
} |
||||
tests := []struct { |
||||
name string |
||||
args args |
||||
want map[string]interface{} |
||||
}{ |
||||
{ |
||||
args: args{[]D{KVString("test", "hello")}}, |
||||
want: map[string]interface{}{"test": "hello"}, |
||||
}, |
||||
{ |
||||
args: args{[]D{KVInt64("test", 123)}}, |
||||
want: map[string]interface{}{"test": int64(123)}, |
||||
}, |
||||
{ |
||||
args: args{[]D{KVFloat32("test", float32(1.01))}}, |
||||
want: map[string]interface{}{"test": float32(1.01)}, |
||||
}, |
||||
{ |
||||
args: args{[]D{KVFloat32("test", float32(1.01))}}, |
||||
want: map[string]interface{}{"test": float32(1.01)}, |
||||
}, |
||||
{ |
||||
args: args{[]D{KVDuration("test", time.Second)}}, |
||||
want: map[string]interface{}{"test": time.Second}, |
||||
}, |
||||
} |
||||
for _, tt := range tests { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
if got := toMap(tt.args.args...); !reflect.DeepEqual(got, tt.want) { |
||||
t.Errorf("toMap() = %v, want %v", got, tt.want) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,57 @@ |
||||
package criticality |
||||
|
||||
// Criticality is
|
||||
type Criticality string |
||||
|
||||
// criticality
|
||||
var ( |
||||
// EmptyCriticality is used to mark any invalid criticality, and the empty criticality will be parsed as the default criticality later.
|
||||
EmptyCriticality = Criticality("") |
||||
// CriticalPlus is reserved for the most critical requests, those that will result in serious user-visible impact if they fail.
|
||||
CriticalPlus = Criticality("CRITICAL_PLUS") |
||||
// Critical is the default value for requests sent from production jobs. These requests will result in user-visible impact, but the impact may be less severe than those of CRITICAL_PLUS. Services are expected to provision enough capacity for all expected CRITICAL and CRITICAL_PLUS traffic.
|
||||
Critical = Criticality("CRITICAL") |
||||
// SheddablePlus is traffic for which partial unavailability is expected. This is the default for batch jobs, which can retry requests minutes or even hours later.
|
||||
SheddablePlus = Criticality("SHEDDABLE_PLUS") |
||||
// Sheddable is traffic for which frequent partial unavailability and occasional full unavailability is expected.
|
||||
Sheddable = Criticality("SHEDDABLE") |
||||
|
||||
// higher is more critical
|
||||
_criticalityEnum = map[Criticality]int{ |
||||
CriticalPlus: 40, |
||||
Critical: 30, |
||||
SheddablePlus: 20, |
||||
Sheddable: 10, |
||||
} |
||||
|
||||
_defaultCriticality = Critical |
||||
) |
||||
|
||||
// Value is used to get criticality value, higher value is more critical.
|
||||
func Value(in Criticality) int { |
||||
v, ok := _criticalityEnum[in] |
||||
if !ok { |
||||
return _criticalityEnum[_defaultCriticality] |
||||
} |
||||
return v |
||||
} |
||||
|
||||
// Higher will compare the input criticality with self, return true if the input is more critical than self.
|
||||
func (c Criticality) Higher(in Criticality) bool { |
||||
return Value(in) > Value(c) |
||||
} |
||||
|
||||
// Parse will parse raw criticality string as valid critality. Any invalid input will parse as empty criticality.
|
||||
func Parse(raw string) Criticality { |
||||
crtl := Criticality(raw) |
||||
if _, ok := _criticalityEnum[crtl]; ok { |
||||
return crtl |
||||
} |
||||
return EmptyCriticality |
||||
} |
||||
|
||||
// Exist is used to check criticality is exist in several enumeration.
|
||||
func Exist(c Criticality) bool { |
||||
_, ok := _criticalityEnum[c] |
||||
return ok |
||||
} |
||||
@ -0,0 +1,21 @@ |
||||
package blademaster |
||||
|
||||
import ( |
||||
criticalityPkg "github.com/bilibili/kratos/pkg/net/criticality" |
||||
"github.com/bilibili/kratos/pkg/net/metadata" |
||||
|
||||
"github.com/pkg/errors" |
||||
) |
||||
|
||||
// Criticality is
|
||||
func Criticality(pathCriticality criticalityPkg.Criticality) HandlerFunc { |
||||
if !criticalityPkg.Exist(pathCriticality) { |
||||
panic(errors.Errorf("This criticality is not exist: %s", pathCriticality)) |
||||
} |
||||
return func(ctx *Context) { |
||||
md, ok := metadata.FromContext(ctx) |
||||
if ok { |
||||
md[metadata.Criticality] = string(pathCriticality) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,62 @@ |
||||
package blademaster |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/bilibili/kratos/pkg/log" |
||||
limit "github.com/bilibili/kratos/pkg/ratelimit" |
||||
"github.com/bilibili/kratos/pkg/ratelimit/bbr" |
||||
"github.com/bilibili/kratos/pkg/stat/prom" |
||||
) |
||||
|
||||
const ( |
||||
_statName = "go_http_bbr" |
||||
) |
||||
|
||||
var ( |
||||
bbrStats = prom.New().WithState("go_http_bbr", []string{"url"}) |
||||
) |
||||
|
||||
// RateLimiter bbr middleware.
|
||||
type RateLimiter struct { |
||||
group *bbr.Group |
||||
logTime int64 |
||||
} |
||||
|
||||
// New return a ratelimit middleware.
|
||||
func NewRateLimiter(conf *bbr.Config) (s *RateLimiter) { |
||||
return &RateLimiter{ |
||||
group: bbr.NewGroup(conf), |
||||
logTime: time.Now().UnixNano(), |
||||
} |
||||
} |
||||
|
||||
func (b *RateLimiter) printStats(routePath string, limiter limit.Limiter) { |
||||
now := time.Now().UnixNano() |
||||
if now-atomic.LoadInt64(&b.logTime) > int64(time.Second*3) { |
||||
atomic.StoreInt64(&b.logTime, now) |
||||
log.Info("http.bbr path:%s stat:%+v", routePath, limiter.(*bbr.BBR).Stat()) |
||||
} |
||||
} |
||||
|
||||
// Limit return a bm handler func.
|
||||
func (b *RateLimiter) Limit() HandlerFunc { |
||||
return func(c *Context) { |
||||
uri := fmt.Sprintf("%s://%s%s", c.Request.URL.Scheme, c.Request.Host, c.Request.URL.Path) |
||||
limiter := b.group.Get(uri) |
||||
done, err := limiter.Allow(c) |
||||
if err != nil { |
||||
bbrStats.Incr(_statName, uri) |
||||
c.JSON(nil, err) |
||||
c.Abort() |
||||
return |
||||
} |
||||
defer func() { |
||||
done(limit.DoneInfo{Op: limit.Success}) |
||||
b.printStats(uri, limiter) |
||||
}() |
||||
c.Next() |
||||
} |
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue