From 0b2a2a4d446b14c16760d57eb9d32a04309af6d5 Mon Sep 17 00:00:00 2001 From: wujianhua <1107015496@qq.com> Date: Tue, 18 Jul 2023 11:32:21 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E7=BB=9F=E4=B8=80sls?= =?UTF-8?q?=E6=8E=A5=E5=85=A5=E5=92=8C=E8=AF=B7=E6=B1=82=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=86=99=E5=85=A5sls=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sls/log.go | 104 ++++++++++++++++++++++++++++++++++++ sls/producer.go | 123 +++++++++++++++++++++++++++++++++++++++++++ sls/producer_test.go | 60 +++++++++++++++++++++ sls/types.go | 46 ++++++++++++++++ 4 files changed, 333 insertions(+) create mode 100644 sls/log.go create mode 100644 sls/producer.go create mode 100644 sls/producer_test.go create mode 100644 sls/types.go diff --git a/sls/log.go b/sls/log.go new file mode 100644 index 0000000..da3662d --- /dev/null +++ b/sls/log.go @@ -0,0 +1,104 @@ +package sls + +import ( + "encoding/json" + "time" + + sls "github.com/aliyun/aliyun-log-go-sdk" + "google.golang.org/protobuf/proto" +) + +// GenerateLog 传入固定请求日志，生成sls对应格式日志 +func GenerateLog(request *Request) *sls.Log { + var log sls.Log + // 获取当前时间 + log.Time = proto.Uint32(uint32(time.Now().Unix())) + contents := make([]*sls.LogContent, 0) + // time + contents = append(contents, &sls.LogContent{ + Key: proto.String("time"), + Value: proto.String(request.Time.Format("2006-01-02 15:04:05")), + }) + // remote_addr + contents = append(contents, &sls.LogContent{ + Key: proto.String("remote_addr"), + Value: proto.String(request.RemoteAddr), + }) + // query_string + contents = append(contents, &sls.LogContent{ + Key: proto.String("query_string"), + Value: proto.String(request.QueryString), + }) + // request_method + contents = append(contents, &sls.LogContent{ + Key: proto.String("request_method"), + Value: proto.String(request.RequestMethod), + }) + // request_uri + contents = append(contents, &sls.LogContent{ + Key: proto.String("request_uri"), + Value: proto.String(request.RequestUri), + }) + // host + contents = append(contents, &sls.LogContent{ + Key: proto.String("host"), + Value: proto.String(request.Host), + }) + // request_body + contents = append(contents, &sls.LogContent{ + Key: proto.String("request_body"), + Value: proto.String(request.RequestBody), + }) + // http_user_agent + contents = append(contents, &sls.LogContent{ + Key: proto.String("http_user_agent"), + Value: proto.String(request.HttpUserAgent), + }) + // http_finger + contents = append(contents, &sls.LogContent{ + Key: proto.String("http_finger"), + Value: proto.String(request.HttpFinger), + }) + // http_referrer + contents = append(contents, &sls.LogContent{ + Key: proto.String("http_referrer"), + Value: proto.String(request.HttpReferrer), + }) + // http_trace_id + contents = append(contents, &sls.LogContent{ + Key: proto.String("http_trace_id"), + Value: proto.String(request.HttpTraceId), + }) + // x_app_id + contents = append(contents, &sls.LogContent{ + Key: proto.String("x_app_id"), + Value: proto.String(request.XAppId), + }) + // x_user_data + userData, _ := json.Marshal(request.XUserData) + contents = append(contents, &sls.LogContent{ + Key: proto.String("x_user_data"), + Value: proto.String(string(userData)), + }) + // x_dsm_data + dsmData, _ := json.Marshal(request.XDsmData) + contents = append(contents, &sls.LogContent{ + Key: proto.String("x_dsm_data"), + Value: proto.String(string(dsmData)), + }) + // http_trace_id + customData, _ := json.Marshal(request.XCustomData) + contents = append(contents, &sls.LogContent{ + Key: proto.String("x_custom_data"), + Value: proto.String(string(customData)), + }) + // response_data + respData, _ := json.Marshal(request.ResponseData) + contents = append(contents, &sls.LogContent{ + Key: proto.String("response_data"), + Value: proto.String(string(respData)), + }) + + log.Contents = contents + return &log +} diff --git a/sls/producer.go b/sls/producer.go new file mode 100644 index 0000000..7c6a273 --- /dev/null +++ b/sls/producer.go @@ -0,0 +1,123 @@ +package sls + +import ( + "gitea.drugeyes.vip/pharnexbase/utils/enum" + sls "github.com/aliyun/aliyun-log-go-sdk" + "github.com/aliyun/aliyun-log-go-sdk/producer" +) + +type options struct { + accessKey string + accessSecret string + endpoint string + project string + logstore string +} + +func defaultOptions() *options { + return &options{ + project: "bcpm-log", + logstore: "request-develop", + } +} + +// WithEndpoint 指定端口 +func WithEndpoint(endpoint string) Option { + return func(alc *options) { + alc.endpoint = endpoint + } +} + +// WithProject 指定发送普通日志的project +func WithProject(project string) Option { + return func(alc *options) { + alc.project = project + } +} + +// WithLogstore 指定发送普通日志的logstore +func WithLogstore(logstore string) Option { + return func(alc *options) { + alc.logstore = logstore + } +} + +// WithAccessKey 设置访问access id +func WithAccessKey(ak string) Option { + return func(alc *options) { + alc.accessKey = ak + } +} + +// WithAccessSecret 设置访问access secret +func WithAccessSecret(as string) Option { + return func(alc *options) { + alc.accessSecret = as + } +} + +type Option func(alc *options) + +type slsProducer struct { + pr *producer.Producer + opts *options +} + +// NewSLSProducer 创建一个高性能的sls制造者 +func NewSLSProducer(opts ...Option) *slsProducer { + opt := defaultOptions() + + for _, o := range opts { + o(opt) + } + + producerConfig := producer.GetDefaultProducerConfig() + producerConfig.Endpoint = opt.endpoint + producerConfig.AccessKeyID = opt.accessKey + producerConfig.AccessKeySecret = opt.accessSecret + producerInstance := producer.InitProducer(producerConfig) + + producerInstance.Start() + return &slsProducer{ + pr: producerInstance, + opts: opt, + } +} + +// Close 程序结束时，必须调用该方法，不然可能导致部分日志丢失 +func (s *slsProducer) Close() error { + return s.pr.Close(5000) +} + +// SendLog 发送普通日志 +func (s *slsProducer) SendLog(log *sls.Log) error { + err := s.pr.SendLog(s.opts.project, s.opts.logstore, "", "", log) + if err != nil { + return err + } + return nil +} + +// SendRequestLog 发送特定的请求日志，使用该方法时，不需要指定project和logstore +func (s *slsProducer) SendRequestLog(env enum.Env, req *Request) error { + log := GenerateLog(req) + + err := s.pr.SendLog("bcpm-log", getLogStore(env), "", "", log) + if err != nil { + return err + } + return nil +} + +func getLogStore(env enum.Env) string { + switch env { + case enum.Env_local, enum.Env_develop: + return "request-develop" + case enum.Env_preview: + return "request-preview" + case enum.Env_production: + return "request" + default: + return "request-develop" + } +} diff --git a/sls/producer_test.go b/sls/producer_test.go new file mode 100644 index 0000000..c480592 --- /dev/null +++ b/sls/producer_test.go @@ -0,0 +1,60 @@ +package sls + +import ( + "testing" + "time" + + "gitea.drugeyes.vip/pharnexbase/utils/enum" +) + +var pd *slsProducer + +func init() { + pd = NewSLSProducer( + WithEndpoint("cn-hangzhou.log.aliyuncs.com"), + WithAccessKey("-"), // 传入阿里云sls的key + WithAccessSecret("-"), // 传入阿里云sls的Secret + ) +} + +func TestProducer(t *testing.T) { + err := pd.SendRequestLog(enum.Env_develop, &Request{ + Time: time.Now(), + RemoteAddr: "118.112.98.208", + QueryString: "test001", + RequestMethod: "POST", + RequestUri: "/dc/analyse/senior_aggregation", + Host: "intsynth-server.drugeyes.vip", + RequestBody: "{}", + HttpUserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0", + HttpFinger: "59246b1e1639f4753bcd010bac2ddb01", + HttpReferrer: "http://intsynth-web.drugeyes.vip:7031/", + HttpTraceId: "0121", + XAppId: "LtEolTiV", + XUserData: &UserData{ + ID: "1075", + Username: "用户名_IDdXdk4xSU", + OrganizeID: "223", + OrganizeName: "新数据-机构06-权限", + Email: "", + Phone: "17800000051", + }, + XDsmData: &DSMData{ + AppKey: "ce2f8bcc2f64466d", + PageType: "LIST", + DataCount: 1, + DBData: make([]*DBCount, 0), + }, + XCustomData: nil, + ResponseData: nil, + }) + if err != nil { + t.Fail() + } + + err = pd.Close() + if err != nil { + t.Fail() + } + +} diff --git a/sls/types.go b/sls/types.go new file mode 100644 index 0000000..47b4fcd --- /dev/null +++ b/sls/types.go @@ -0,0 +1,46 @@ +package sls + +import "time" + +// Request HTTP请求 +type Request struct { + Time time.Time `json:"time"` // 请求时间 + RemoteAddr string `json:"remote_addr"` // 客户IP + QueryString string `json:"query_string"` // Query参数 + RequestMethod string `json:"request_method"` // 请求方法 + RequestUri string `json:"request_uri"` // 请求地址 + Host string `json:"host"` // host地址 + RequestBody string `json:"request_body"` // 请求body + HttpUserAgent string `json:"http_user_agent"` // 浏览器标识 + HttpFinger string `json:"http_finger"` // 浏览器指纹 + HttpReferrer string `json:"http_referrer"` // referer信息 + HttpTraceId string `json:"http_trace_id"` // trance信息 + XAppId string `json:"x_app_id"` // 用户中心的应用ID + XUserData *UserData `json:"x_user_data"` // 用户信息 + XDsmData *DSMData `json:"x_dsm_data"` // 风控信息，不接人风控可不传入 + XCustomData any `json:"x_custom_data"` // 自定义信息，需要能够转为json字符串 + ResponseData any `json:"response_data"` // 返回信息 +} + +// UserData 用户数据 +type UserData struct { + ID string `json:"id"` // 用户ID + Username string `json:"username"` // 用户名称 + OrganizeID string `json:"organize_id"` // 组织ID + OrganizeName string `json:"organize_name"` // 组织名称 + Phone string `json:"phone"` // 手机号 + Email string `json:"email"` // 邮箱 +} + +type DBCount struct { + DB string `json:"db"` // 数据库标识 + UniqueKey string `json:"unique_key"` // 该条记录唯一标识 +} + +// DSMData 风控需要数据 +type DSMData struct { + AppKey string `json:"app_key"` // 项目在风控平台的appid + PageType string `json:"page_type"` // 访问类型 + DataCount int32 `json:"data_count"` // 请求数量，list的时为列表数据量 + DBData []*DBCount `json:"db_data"` // 数据库监控时，对应数据相关内容 +}