From 94182a21993b2e7ec9a5e9332032c7bc769109ef Mon Sep 17 00:00:00 2001 From: joeybloggs Date: Fri, 27 Nov 2015 10:13:49 -0500 Subject: [PATCH 1/2] init struct cache changes --- cache.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ util.go | 51 +++++++++++++++++++++++++++++++++++++++ validator.go | 42 ++++---------------------------- 3 files changed, 123 insertions(+), 37 deletions(-) create mode 100644 cache.go diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..8318c84 --- /dev/null +++ b/cache.go @@ -0,0 +1,67 @@ +package validator + +import "sync" + +type cachedField struct { + Idx int + Name string + AltName string + CachedTag *cachedTag +} + +type cachedStruct struct { + Name string + fields map[int]cachedField +} + +type structCacheMap struct { + lock sync.RWMutex + m map[string]*cachedStruct +} + +func (s *structCacheMap) Get(key string) (*cachedStruct, bool) { + s.lock.RLock() + value, ok := s.m[key] + s.lock.RUnlock() + return value, ok +} + +func (s *structCacheMap) Set(key string, value *cachedStruct) { + s.lock.Lock() + s.m[key] = value + s.lock.Unlock() +} + +type cachedTag struct { + isOmitEmpty bool + isNoStructLevel bool + isStructOnly bool + diveTag string + tags []*tagVals +} + +type tagVals struct { + tagVals [][]string + isOrVal bool + isAlias bool + tag string +} + +type tagCacheMap struct { + lock sync.RWMutex + m map[string]*cachedTag +} + +func (s *tagCacheMap) Get(key string) (*cachedTag, bool) { + s.lock.RLock() + value, ok := s.m[key] + s.lock.RUnlock() + + return value, ok +} + +func (s *tagCacheMap) Set(key string, value *cachedTag) { + s.lock.Lock() + s.m[key] = value + s.lock.Unlock() +} diff --git a/util.go b/util.go index 10626bc..3a35fa8 100644 --- a/util.go +++ b/util.go @@ -247,11 +247,62 @@ func panicIf(err error) { } } +func (v *Validate) parseStruct(topStruct reflect.Type, sName string) *cachedStruct { + + s := &cachedStruct{Name: sName, fields: map[int]cachedField{}} + + numFields := topStruct.NumField() + + var fld reflect.StructField + var tag string + var customName string + + for i := 0; i < numFields; i++ { + + fld = topStruct.Field(i) + + if len(fld.PkgPath) != 0 { + continue + } + + tag = fld.Tag.Get(v.tagName) + + if tag == skipValidationTag { + continue + } + + customName = fld.Name + if len(v.fieldNameTag) != 0 { + + name := strings.SplitN(fld.Tag.Get(v.fieldNameTag), ",", 2)[0] + + // dash check is for json "-" (aka skipValidationTag) means don't output in json + if name != "" && name != skipValidationTag { + customName = name + } + } + + cTag, ok := v.tagCache.Get(tag) + if !ok { + cTag = v.parseTags(tag, fld.Name) + } + + s.fields[i] = cachedField{Idx: i, Name: fld.Name, AltName: customName, CachedTag: cTag} + } + + v.structCache.Set(sName, s) + + return s +} + func (v *Validate) parseTags(tag, fieldName string) *cachedTag { cTag := &cachedTag{} v.parseTagsRecursive(cTag, tag, fieldName, blank, false) + + v.tagCache.Set(tag, cTag) + return cTag } diff --git a/validator.go b/validator.go index 4347095..0b13bae 100644 --- a/validator.go +++ b/validator.go @@ -47,39 +47,6 @@ var ( emptyStructPtr = new(struct{}) ) -type cachedTag struct { - isOmitEmpty bool - isNoStructLevel bool - isStructOnly bool - diveTag string - tags []*tagVals -} - -type tagVals struct { - tagVals [][]string - isOrVal bool - isAlias bool - tag string -} - -type tagCacheMap struct { - lock sync.RWMutex - m map[string]*cachedTag -} - -func (s *tagCacheMap) Get(key string) (*cachedTag, bool) { - s.lock.RLock() - value, ok := s.m[key] - s.lock.RUnlock() - return value, ok -} - -func (s *tagCacheMap) Set(key string, value *cachedTag) { - s.lock.Lock() - s.m[key] = value - s.lock.Unlock() -} - // StructLevel contains all of the information and helper methods // for reporting errors during struct level validation type StructLevel struct { @@ -154,7 +121,8 @@ type Validate struct { hasCustomFuncs bool hasAliasValidators bool hasStructLevelFuncs bool - tagsCache *tagCacheMap + tagCache *tagCacheMap + structCache *structCacheMap errsPool *sync.Pool } @@ -227,7 +195,8 @@ func New(config *Config) *Validate { v := &Validate{ tagName: config.TagName, fieldNameTag: config.FieldNameTag, - tagsCache: &tagCacheMap{m: map[string]*cachedTag{}}, + tagCache: &tagCacheMap{m: map[string]*cachedTag{}}, + structCache: &structCacheMap{m: map[string]*cachedStruct{}}, errsPool: &sync.Pool{New: func() interface{} { return ValidationErrors{} }}} @@ -545,11 +514,10 @@ func (v *Validate) traverseField(topStruct reflect.Value, currentStruct reflect. return } - cTag, isCached := v.tagsCache.Get(tag) + cTag, isCached := v.tagCache.Get(tag) if !isCached { cTag = v.parseTags(tag, name) - v.tagsCache.Set(tag, cTag) } current, kind := v.ExtractType(current) From e019c2854253f5526fb664efcd0726dbb0c8a45b Mon Sep 17 00:00:00 2001 From: joeybloggs Date: Fri, 27 Nov 2015 15:19:15 -0500 Subject: [PATCH 2/2] Add struct field + associated tags caching * This essentially reduces the number of cache tag lookups for a structs fields to one. --- README.md | 56 ++++++++++++++--------------- cache.go | 12 ++++--- util.go | 11 +++--- validator.go | 100 +++++++++++++++++++++++++++++++++------------------ 4 files changed, 108 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index c6ea5a0..4731304 100644 --- a/README.md +++ b/README.md @@ -310,34 +310,34 @@ Benchmarks ```go $ go test -cpu=4 -bench=. -benchmem=true PASS -BenchmarkFieldSuccess-4 10000000 163 ns/op 0 B/op 0 allocs/op -BenchmarkFieldFailure-4 2000000 673 ns/op 400 B/op 4 allocs/op -BenchmarkFieldDiveSuccess-4 500000 3019 ns/op 480 B/op 27 allocs/op -BenchmarkFieldDiveFailure-4 500000 3553 ns/op 880 B/op 31 allocs/op -BenchmarkFieldCustomTypeSuccess-4 5000000 347 ns/op 32 B/op 2 allocs/op -BenchmarkFieldCustomTypeFailure-4 2000000 645 ns/op 400 B/op 4 allocs/op -BenchmarkFieldOrTagSuccess-4 1000000 1177 ns/op 16 B/op 1 allocs/op -BenchmarkFieldOrTagFailure-4 1000000 1093 ns/op 432 B/op 6 allocs/op -BenchmarkStructLevelValidationSuccess-4 2000000 702 ns/op 160 B/op 6 allocs/op -BenchmarkStructLevelValidationFailure-4 1000000 1279 ns/op 592 B/op 11 allocs/op -BenchmarkStructSimpleCustomTypeSuccess-4 1000000 1010 ns/op 80 B/op 5 allocs/op -BenchmarkStructSimpleCustomTypeFailure-4 1000000 1544 ns/op 624 B/op 11 allocs/op -BenchmarkStructPartialSuccess-4 1000000 1249 ns/op 400 B/op 11 allocs/op -BenchmarkStructPartialFailure-4 1000000 1797 ns/op 816 B/op 16 allocs/op -BenchmarkStructExceptSuccess-4 2000000 927 ns/op 368 B/op 9 allocs/op -BenchmarkStructExceptFailure-4 1000000 1259 ns/op 400 B/op 11 allocs/op -BenchmarkStructSimpleCrossFieldSuccess-4 1000000 1076 ns/op 128 B/op 6 allocs/op -BenchmarkStructSimpleCrossFieldFailure-4 1000000 1623 ns/op 560 B/op 11 allocs/op -BenchmarkStructSimpleCrossStructCrossFieldSuccess-4 1000000 1582 ns/op 176 B/op 9 allocs/op -BenchmarkStructSimpleCrossStructCrossFieldFailure-4 1000000 2139 ns/op 608 B/op 14 allocs/op -BenchmarkStructSimpleSuccess-4 1000000 1040 ns/op 48 B/op 3 allocs/op -BenchmarkStructSimpleFailure-4 1000000 1683 ns/op 624 B/op 11 allocs/op -BenchmarkStructSimpleSuccessParallel-4 5000000 356 ns/op 48 B/op 3 allocs/op -BenchmarkStructSimpleFailureParallel-4 2000000 831 ns/op 624 B/op 11 allocs/op -BenchmarkStructComplexSuccess-4 200000 6738 ns/op 512 B/op 30 allocs/op -BenchmarkStructComplexFailure-4 200000 11387 ns/op 3415 B/op 72 allocs/op -BenchmarkStructComplexSuccessParallel-4 500000 2330 ns/op 512 B/op 30 allocs/op -BenchmarkStructComplexFailureParallel-4 300000 4857 ns/op 3416 B/op 72 allocs/op +BenchmarkFieldSuccess-4 10000000 162 ns/op 0 B/op 0 allocs/op +BenchmarkFieldFailure-4 2000000 678 ns/op 400 B/op 4 allocs/op +BenchmarkFieldDiveSuccess-4 500000 3079 ns/op 480 B/op 27 allocs/op +BenchmarkFieldDiveFailure-4 300000 3584 ns/op 880 B/op 31 allocs/op +BenchmarkFieldCustomTypeSuccess-4 5000000 345 ns/op 32 B/op 2 allocs/op +BenchmarkFieldCustomTypeFailure-4 2000000 650 ns/op 400 B/op 4 allocs/op +BenchmarkFieldOrTagSuccess-4 1000000 1188 ns/op 16 B/op 1 allocs/op +BenchmarkFieldOrTagFailure-4 1000000 1088 ns/op 432 B/op 6 allocs/op +BenchmarkStructLevelValidationSuccess-4 2000000 689 ns/op 160 B/op 6 allocs/op +BenchmarkStructLevelValidationFailure-4 1000000 1290 ns/op 592 B/op 11 allocs/op +BenchmarkStructSimpleCustomTypeSuccess-4 2000000 911 ns/op 80 B/op 5 allocs/op +BenchmarkStructSimpleCustomTypeFailure-4 1000000 1446 ns/op 624 B/op 11 allocs/op +BenchmarkStructPartialSuccess-4 1000000 1221 ns/op 384 B/op 10 allocs/op +BenchmarkStructPartialFailure-4 1000000 1764 ns/op 800 B/op 15 allocs/op +BenchmarkStructExceptSuccess-4 2000000 941 ns/op 336 B/op 7 allocs/op +BenchmarkStructExceptFailure-4 1000000 1237 ns/op 384 B/op 10 allocs/op +BenchmarkStructSimpleCrossFieldSuccess-4 2000000 970 ns/op 128 B/op 6 allocs/op +BenchmarkStructSimpleCrossFieldFailure-4 1000000 1560 ns/op 560 B/op 11 allocs/op +BenchmarkStructSimpleCrossStructCrossFieldSuccess-4 1000000 1542 ns/op 176 B/op 9 allocs/op +BenchmarkStructSimpleCrossStructCrossFieldFailure-4 1000000 2147 ns/op 608 B/op 14 allocs/op +BenchmarkStructSimpleSuccess-4 2000000 847 ns/op 48 B/op 3 allocs/op +BenchmarkStructSimpleFailure-4 1000000 1497 ns/op 624 B/op 11 allocs/op +BenchmarkStructSimpleSuccessParallel-4 5000000 257 ns/op 48 B/op 3 allocs/op +BenchmarkStructSimpleFailureParallel-4 2000000 586 ns/op 624 B/op 11 allocs/op +BenchmarkStructComplexSuccess-4 300000 5104 ns/op 496 B/op 29 allocs/op +BenchmarkStructComplexFailure-4 200000 9840 ns/op 3400 B/op 71 allocs/op +BenchmarkStructComplexSuccessParallel-4 1000000 1540 ns/op 496 B/op 29 allocs/op +BenchmarkStructComplexFailureParallel-4 500000 3478 ns/op 3400 B/op 71 allocs/op ``` How to Contribute diff --git a/cache.go b/cache.go index 8318c84..289226e 100644 --- a/cache.go +++ b/cache.go @@ -1,6 +1,9 @@ package validator -import "sync" +import ( + "reflect" + "sync" +) type cachedField struct { Idx int @@ -16,23 +19,24 @@ type cachedStruct struct { type structCacheMap struct { lock sync.RWMutex - m map[string]*cachedStruct + m map[reflect.Type]*cachedStruct } -func (s *structCacheMap) Get(key string) (*cachedStruct, bool) { +func (s *structCacheMap) Get(key reflect.Type) (*cachedStruct, bool) { s.lock.RLock() value, ok := s.m[key] s.lock.RUnlock() return value, ok } -func (s *structCacheMap) Set(key string, value *cachedStruct) { +func (s *structCacheMap) Set(key reflect.Type, value *cachedStruct) { s.lock.Lock() s.m[key] = value s.lock.Unlock() } type cachedTag struct { + tag string isOmitEmpty bool isNoStructLevel bool isStructOnly bool diff --git a/util.go b/util.go index 3a35fa8..abe6b03 100644 --- a/util.go +++ b/util.go @@ -247,11 +247,12 @@ func panicIf(err error) { } } -func (v *Validate) parseStruct(topStruct reflect.Type, sName string) *cachedStruct { +func (v *Validate) parseStruct(current reflect.Value, sName string) *cachedStruct { + typ := current.Type() s := &cachedStruct{Name: sName, fields: map[int]cachedField{}} - numFields := topStruct.NumField() + numFields := current.NumField() var fld reflect.StructField var tag string @@ -259,7 +260,7 @@ func (v *Validate) parseStruct(topStruct reflect.Type, sName string) *cachedStru for i := 0; i < numFields; i++ { - fld = topStruct.Field(i) + fld = typ.Field(i) if len(fld.PkgPath) != 0 { continue @@ -290,14 +291,14 @@ func (v *Validate) parseStruct(topStruct reflect.Type, sName string) *cachedStru s.fields[i] = cachedField{Idx: i, Name: fld.Name, AltName: customName, CachedTag: cTag} } - v.structCache.Set(sName, s) + v.structCache.Set(typ, s) return s } func (v *Validate) parseTags(tag, fieldName string) *cachedTag { - cTag := &cachedTag{} + cTag := &cachedTag{tag: tag} v.parseTagsRecursive(cTag, tag, fieldName, blank, false) diff --git a/validator.go b/validator.go index 0b13bae..65a0914 100644 --- a/validator.go +++ b/validator.go @@ -16,7 +16,6 @@ import ( "strings" "sync" "time" - "unicode" ) const ( @@ -196,7 +195,7 @@ func New(config *Config) *Validate { tagName: config.TagName, fieldNameTag: config.FieldNameTag, tagCache: &tagCacheMap{m: map[string]*cachedTag{}}, - structCache: &structCacheMap{m: map[string]*cachedStruct{}}, + structCache: &structCacheMap{m: map[reflect.Type]*cachedStruct{}}, errsPool: &sync.Pool{New: func() interface{} { return ValidationErrors{} }}} @@ -307,7 +306,7 @@ func (v *Validate) Field(field interface{}, tag string) error { errs := v.errsPool.Get().(ValidationErrors) fieldVal := reflect.ValueOf(field) - v.traverseField(fieldVal, fieldVal, fieldVal, blank, errs, false, tag, blank, blank, false, false, nil) + v.traverseField(fieldVal, fieldVal, fieldVal, blank, errs, false, tag, blank, blank, false, false, nil, nil) if len(errs) == 0 { v.errsPool.Put(errs) @@ -327,7 +326,7 @@ func (v *Validate) FieldWithValue(val interface{}, field interface{}, tag string errs := v.errsPool.Get().(ValidationErrors) topVal := reflect.ValueOf(val) - v.traverseField(topVal, topVal, reflect.ValueOf(field), blank, errs, false, tag, blank, blank, false, false, nil) + v.traverseField(topVal, topVal, reflect.ValueOf(field), blank, errs, false, tag, blank, blank, false, false, nil, nil) if len(errs) == 0 { v.errsPool.Put(errs) @@ -452,50 +451,80 @@ func (v *Validate) tranverseStruct(topStruct reflect.Value, currentStruct reflec panic("value passed for validation is not a struct") } - var ok bool + // var ok bool typ := current.Type() + sName := typ.Name() + if useStructName { - errPrefix += typ.Name() + "." + errPrefix += sName + "." } // structonly tag present don't tranverseFields // but must still check and run below struct level validation // if present if !isStructOnly { - numFields := current.NumField() var fld reflect.StructField - var customName string - for i := 0; i < numFields; i++ { - fld = typ.Field(i) + // is anonymous struct, cannot parse or cache as + // it has no name to index by + if len(sName) == 0 { - if !unicode.IsUpper(rune(fld.Name[0])) { - continue - } + var customName string + var ok bool + numFields := current.NumField() - if partial { + for i := 0; i < numFields; i++ { - _, ok = includeExclude[errPrefix+fld.Name] + fld = typ.Field(i) - if (ok && exclude) || (!ok && !exclude) { + if len(fld.PkgPath) != 0 { continue } - } - customName = fld.Name - if v.fieldNameTag != "" { + if partial { - name := strings.SplitN(fld.Tag.Get(v.fieldNameTag), ",", 2)[0] + _, ok = includeExclude[errPrefix+fld.Name] - // dash check is for json "-" means don't output in json - if name != "" && name != "-" { - customName = name + if (ok && exclude) || (!ok && !exclude) { + continue + } } + + customName = fld.Name + if v.fieldNameTag != "" { + + name := strings.SplitN(fld.Tag.Get(v.fieldNameTag), ",", 2)[0] + + // dash check is for json "-" means don't output in json + if name != "" && name != "-" { + customName = name + } + } + + v.traverseField(topStruct, currentStruct, current.Field(i), errPrefix, errs, true, fld.Tag.Get(v.tagName), fld.Name, customName, partial, exclude, includeExclude, nil) } + } else { + s, ok := v.structCache.Get(typ) + if !ok { + s = v.parseStruct(current, sName) + } + + for i, f := range s.fields { + + if partial { + + _, ok = includeExclude[errPrefix+f.Name] - v.traverseField(topStruct, currentStruct, current.Field(i), errPrefix, errs, true, fld.Tag.Get(v.tagName), fld.Name, customName, partial, exclude, includeExclude) + if (ok && exclude) || (!ok && !exclude) { + continue + } + } + fld = typ.Field(i) + + v.traverseField(topStruct, currentStruct, current.Field(i), errPrefix, errs, true, f.CachedTag.tag, fld.Name, f.AltName, partial, exclude, includeExclude, f.CachedTag) + } } } @@ -508,16 +537,19 @@ func (v *Validate) tranverseStruct(topStruct reflect.Value, currentStruct reflec } // traverseField validates any field, be it a struct or single field, ensures it's validity and passes it along to be validated via it's tag options -func (v *Validate) traverseField(topStruct reflect.Value, currentStruct reflect.Value, current reflect.Value, errPrefix string, errs ValidationErrors, isStructField bool, tag, name, customName string, partial bool, exclude bool, includeExclude map[string]*struct{}) { +func (v *Validate) traverseField(topStruct reflect.Value, currentStruct reflect.Value, current reflect.Value, errPrefix string, errs ValidationErrors, isStructField bool, tag, name, customName string, partial bool, exclude bool, includeExclude map[string]*struct{}, cTag *cachedTag) { if tag == skipValidationTag { return } - cTag, isCached := v.tagCache.Get(tag) + if cTag == nil { + var isCached bool + cTag, isCached = v.tagCache.Get(tag) - if !isCached { - cTag = v.parseTags(tag, name) + if !isCached { + cTag = v.parseTags(tag, name) + } } current, kind := v.ExtractType(current) @@ -615,9 +647,9 @@ func (v *Validate) traverseField(topStruct reflect.Value, currentStruct reflect. // or panic ;) switch kind { case reflect.Slice, reflect.Array: - v.traverseSlice(topStruct, currentStruct, current, errPrefix, errs, diveSubTag, name, customName, partial, exclude, includeExclude) + v.traverseSlice(topStruct, currentStruct, current, errPrefix, errs, diveSubTag, name, customName, partial, exclude, includeExclude, nil) case reflect.Map: - v.traverseMap(topStruct, currentStruct, current, errPrefix, errs, diveSubTag, name, customName, partial, exclude, includeExclude) + v.traverseMap(topStruct, currentStruct, current, errPrefix, errs, diveSubTag, name, customName, partial, exclude, includeExclude, nil) default: // throw error, if not a slice or map then should not have gotten here // bad dive tag @@ -627,18 +659,18 @@ func (v *Validate) traverseField(topStruct reflect.Value, currentStruct reflect. } // traverseSlice traverses a Slice or Array's elements and passes them to traverseField for validation -func (v *Validate) traverseSlice(topStruct reflect.Value, currentStruct reflect.Value, current reflect.Value, errPrefix string, errs ValidationErrors, tag, name, customName string, partial bool, exclude bool, includeExclude map[string]*struct{}) { +func (v *Validate) traverseSlice(topStruct reflect.Value, currentStruct reflect.Value, current reflect.Value, errPrefix string, errs ValidationErrors, tag, name, customName string, partial bool, exclude bool, includeExclude map[string]*struct{}, cTag *cachedTag) { for i := 0; i < current.Len(); i++ { - v.traverseField(topStruct, currentStruct, current.Index(i), errPrefix, errs, false, tag, fmt.Sprintf(arrayIndexFieldName, name, i), fmt.Sprintf(arrayIndexFieldName, customName, i), partial, exclude, includeExclude) + v.traverseField(topStruct, currentStruct, current.Index(i), errPrefix, errs, false, tag, fmt.Sprintf(arrayIndexFieldName, name, i), fmt.Sprintf(arrayIndexFieldName, customName, i), partial, exclude, includeExclude, cTag) } } // traverseMap traverses a map's elements and passes them to traverseField for validation -func (v *Validate) traverseMap(topStruct reflect.Value, currentStruct reflect.Value, current reflect.Value, errPrefix string, errs ValidationErrors, tag, name, customName string, partial bool, exclude bool, includeExclude map[string]*struct{}) { +func (v *Validate) traverseMap(topStruct reflect.Value, currentStruct reflect.Value, current reflect.Value, errPrefix string, errs ValidationErrors, tag, name, customName string, partial bool, exclude bool, includeExclude map[string]*struct{}, cTag *cachedTag) { for _, key := range current.MapKeys() { - v.traverseField(topStruct, currentStruct, current.MapIndex(key), errPrefix, errs, false, tag, fmt.Sprintf(mapIndexFieldName, name, key.Interface()), fmt.Sprintf(mapIndexFieldName, customName, key.Interface()), partial, exclude, includeExclude) + v.traverseField(topStruct, currentStruct, current.MapIndex(key), errPrefix, errs, false, tag, fmt.Sprintf(mapIndexFieldName, name, key.Interface()), fmt.Sprintf(mapIndexFieldName, customName, key.Interface()), partial, exclude, includeExclude, cTag) } }