Compare commits

...

11 Commits

Author SHA1 Message Date
9cc0abf9e0 v0.0.278 DYN-166 bugfix jsonfilter
Some checks failed
Build Docker and Deploy / Run goext test-suite (push) Failing after 52s
2023-10-05 12:54:07 +02:00
7c40bcfd3c v0.0.277 DYN-166 json marshal filter in ginext Write
Some checks failed
Build Docker and Deploy / Run goext test-suite (push) Failing after 54s
2023-10-05 12:00:51 +02:00
05636a1e4d v0.0.276
All checks were successful
Build Docker and Deploy / Run goext test-suite (push) Successful in 1m12s
2023-10-05 10:59:20 +02:00
0f52b860ea DYN-166 add jsonfilter to json library
All checks were successful
Build Docker and Deploy / Run goext test-suite (push) Successful in 56s
2023-10-05 10:57:34 +02:00
b5cd116219 DYN-166 add jsonfilter to json library
All checks were successful
Build Docker and Deploy / Run goext test-suite (push) Successful in 47s
2023-10-05 10:45:09 +02:00
98486842ae v0.0.275 fix missing returns in (v MetaValue) ShortString
All checks were successful
Build Docker and Deploy / Run goext test-suite (push) Successful in 54s
2023-09-29 16:00:40 +02:00
7577a2dd47 v0.0.274 limit exerr log meta values (shortlog) to 240 chars
All checks were successful
Build Docker and Deploy / Run goext test-suite (push) Successful in 50s
2023-09-27 16:18:21 +02:00
08681756b6 v0.0.273 add stack to PanicWrappedErr
All checks were successful
Build Docker and Deploy / Run goext test-suite (push) Successful in 1m9s
2023-09-27 14:15:59 +02:00
64772d0474 v0.0.272 WMO: fix FindOneAndReplace not using FindOneAndReplace
Some checks failed
Build Docker and Deploy / Run goext test-suite (push) Failing after 50s
2023-09-26 14:41:15 +02:00
127764556e Merge branch 'master' of ssh://gogs.mikescher.com:8022/BlackForestBytes/goext 2023-09-26 14:41:06 +02:00
170f43d806 WMO: fix FindOneAndReplace not using FindOneAndReplace 2023-09-26 14:40:56 +02:00
17 changed files with 140 additions and 137 deletions

View File

@@ -164,7 +164,7 @@ func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
}
func (ee *ExErr) ShortLog(evt *zerolog.Event) {
ee.Meta.Apply(evt).Msg(ee.FormatLog(LogPrintShort))
ee.Meta.Apply(evt, langext.Ptr(240)).Msg(ee.FormatLog(LogPrintShort))
}
// RecursiveMessage returns the message to show

View File

@@ -217,23 +217,35 @@ func (v MetaValue) ShortString(lim int) string {
return "(err)"
}
func (v MetaValue) Apply(key string, evt *zerolog.Event) *zerolog.Event {
func (v MetaValue) Apply(key string, evt *zerolog.Event, limitLen *int) *zerolog.Event {
switch v.DataType {
case MDTString:
return evt.Str(key, v.Value.(string))
if limitLen == nil {
return evt.Str(key, v.Value.(string))
} else {
return evt.Str(key, langext.StrLimit(v.Value.(string), *limitLen, "..."))
}
case MDTID:
return evt.Str(key, v.Value.(IDWrap).Value)
case MDTAny:
if v.Value.(AnyWrap).IsError {
return evt.Str(key, "(err)")
} else {
return evt.Str(key, v.Value.(AnyWrap).Json)
if limitLen == nil {
return evt.Str(key, v.Value.(AnyWrap).Json)
} else {
return evt.Str(key, langext.StrLimit(v.Value.(AnyWrap).Json, *limitLen, "..."))
}
}
case MDTStringPtr:
if langext.IsNil(v.Value) {
return evt.Str(key, "<<null>>")
}
return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
if limitLen == nil {
return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
} else {
return evt.Str(key, langext.StrLimit(langext.CoalesceString(v.Value.(*string), "<<null>>"), *limitLen, "..."))
}
case MDTInt:
return evt.Int(key, v.Value.(int))
case MDTInt8:
@@ -702,9 +714,9 @@ func (mm MetaMap) Any() bool {
return len(mm) > 0
}
func (mm MetaMap) Apply(evt *zerolog.Event) *zerolog.Event {
func (mm MetaMap) Apply(evt *zerolog.Event, limitLen *int) *zerolog.Event {
for key, val := range mm {
evt = val.Apply(key, evt)
evt = val.Apply(key, evt, limitLen)
}
return evt
}

View File

@@ -27,7 +27,11 @@ func (j jsonHTTPResponse) Write(g *gin.Context) {
for _, v := range j.headers {
g.Header(v.Key, v.Val)
}
g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true})
var f *string
if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
f = &jsonfilter
}
g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f})
}
func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {

2
go.mod
View File

@@ -21,7 +21,7 @@ require (
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.15.4 // indirect
github.com/go-playground/validator/v10 v10.15.5 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/json-iterator/go v1.1.12 // indirect

2
go.sum
View File

@@ -38,6 +38,8 @@ github.com/go-playground/validator/v10 v10.15.3 h1:S+sSpunYjNPDuXkWbK+x+bA7iXiW2
github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs=
github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

View File

@@ -1,5 +1,5 @@
package goext
const GoextVersion = "0.0.271"
const GoextVersion = "0.0.278"
const GoextVersionTimestamp = "2023-09-26T14:32:45+0200"
const GoextVersionTimestamp = "2023-10-05T12:54:07+0200"

View File

@@ -156,7 +156,6 @@ import (
// an error.
func Marshal(v any) ([]byte, error) {
e := newEncodeState()
defer encodeStatePool.Put(e)
err := e.marshal(v, encOpts{escapeHTML: true})
if err != nil {
@@ -164,6 +163,8 @@ func Marshal(v any) ([]byte, error) {
}
buf := append([]byte(nil), e.Bytes()...)
encodeStatePool.Put(e)
return buf, nil
}
@@ -174,9 +175,9 @@ type IndentOpt struct {
// MarshalSafeCollections is like Marshal except it will marshal nil maps and
// slices as '{}' and '[]' respectfully instead of 'null'
func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt) ([]byte, error) {
func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt, filter *string) ([]byte, error) {
e := &encodeState{}
err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps})
err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps, filter: filter})
if err != nil {
return nil, err
}
@@ -393,6 +394,9 @@ type encOpts struct {
nilSafeSlices bool
// nilSafeMaps marshals a nil maps '{}' instead of 'null'
nilSafeMaps bool
// filter matches jsonfilter tag of struct
// marshals if no jsonfilter is set or otherwise if jsonfilter has the filter value
filter *string
}
type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
@@ -777,6 +781,8 @@ FieldLoop:
if f.omitEmpty && isEmptyValue(fv) {
continue
} else if opts.filter != nil && len(f.jsonfilter) > 0 && !f.jsonfilter.Contains(*opts.filter) {
continue
}
e.WriteByte(next)
next = ','
@@ -1220,15 +1226,28 @@ type field struct {
nameNonEsc string // `"` + name + `":`
nameEscHTML string // `"` + HTMLEscape(name) + `":`
tag bool
index []int
typ reflect.Type
omitEmpty bool
quoted bool
tag bool
index []int
typ reflect.Type
omitEmpty bool
jsonfilter jsonfilter
quoted bool
encoder encoderFunc
}
// jsonfilter stores the value of the jsonfilter struct tag
type jsonfilter []string
func (j jsonfilter) Contains(t string) bool {
for _, tag := range j {
if t == tag {
return true
}
}
return false
}
// byIndex sorts field by index sequence.
type byIndex []field
@@ -1304,6 +1323,13 @@ func typeFields(t reflect.Type) structFields {
if !isValidTag(name) {
name = ""
}
var jsonfilter []string
jsonfilterTag := sf.Tag.Get("jsonfilter")
if jsonfilterTag != "" && jsonfilterTag != "-" {
jsonfilter = strings.Split(jsonfilterTag, ",")
}
index := make([]int, len(f.index)+1)
copy(index, f.index)
index[len(f.index)] = i
@@ -1334,12 +1360,13 @@ func typeFields(t reflect.Type) structFields {
name = sf.Name
}
field := field{
name: name,
tag: tagged,
index: index,
typ: ft,
omitEmpty: opts.Contains("omitempty"),
quoted: quoted,
name: name,
tag: tagged,
index: index,
typ: ft,
omitEmpty: opts.Contains("omitempty"),
jsonfilter: jsonfilter,
quoted: quoted,
}
field.nameBytes = []byte(field.name)
field.equalFold = foldFunc(field.nameBytes)

View File

@@ -1253,6 +1253,10 @@ func TestMarshalSafeCollections(t *testing.T) {
nilMapStruct struct {
NilMap map[string]interface{} `json:"nil_map"`
}
testWithFilter struct {
Test1 string `json:"test1" jsonfilter:"FILTERONE"`
Test2 string `json:"test2" jsonfilter:"FILTERTWO"`
}
)
tests := []struct {
@@ -1271,10 +1275,12 @@ func TestMarshalSafeCollections(t *testing.T) {
{map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}"},
{pNilMap, "null"},
{nilMapStruct{}, "{\"nil_map\":{}}"},
{testWithFilter{}, "{\"test1\":\"\"}"},
}
filter := "FILTERONE"
for i, tt := range tests {
b, err := MarshalSafeCollections(tt.in, true, true, nil)
b, err := MarshalSafeCollections(tt.in, true, true, nil, &filter)
if err != nil {
t.Errorf("test %d, unexpected failure: %v", i, err)
}

View File

@@ -97,7 +97,10 @@ func equalFoldRight(s, t []byte) bool {
t = t[size:]
}
return len(t) == 0
if len(t) > 0 {
return false
}
return true
}
// asciiEqualFold is a specialization of bytes.EqualFold for use when

View File

@@ -52,7 +52,9 @@ func TestFold(t *testing.T) {
}
func TestFoldAgainstUnicode(t *testing.T) {
var buf1, buf2 []byte
const bufSize = 5
buf1 := make([]byte, 0, bufSize)
buf2 := make([]byte, 0, bufSize)
var runes []rune
for i := 0x20; i <= 0x7f; i++ {
runes = append(runes, rune(i))
@@ -94,8 +96,12 @@ func TestFoldAgainstUnicode(t *testing.T) {
continue
}
for _, r2 := range runes {
buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x')
buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x')
buf1 := append(buf1[:0], 'x')
buf2 := append(buf2[:0], 'x')
buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
buf1 = append(buf1, 'x')
buf2 = append(buf2, 'x')
want := bytes.EqualFold(buf1, buf2)
if got := ff.fold(buf1, buf2); got != want {
t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)

View File

@@ -17,6 +17,7 @@ type GoJsonRender struct {
NilSafeSlices bool
NilSafeMaps bool
Indent *IndentOpt
Filter *string
}
func (r GoJsonRender) Render(w http.ResponseWriter) error {
@@ -25,7 +26,7 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
header["Content-Type"] = []string{"application/json; charset=utf-8"}
}
jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent)
jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
if err != nil {
panic(err)
}

View File

@@ -116,3 +116,18 @@ func TestNumberIsValid(t *testing.T) {
}
}
}
func BenchmarkNumberIsValid(b *testing.B) {
s := "-61657.61667E+61673"
for i := 0; i < b.N; i++ {
isValidNumber(s)
}
}
func BenchmarkNumberIsValidRegexp(b *testing.B) {
var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
s := "-61657.61667E+61673"
for i := 0; i < b.N; i++ {
jsonNumberRegexp.MatchString(s)
}
}

View File

@@ -594,7 +594,7 @@ func (s *scanner) error(c byte, context string) int {
return scanError
}
// quoteChar formats c as a quoted character literal.
// quoteChar formats c as a quoted character literal
func quoteChar(c byte) string {
// special cases - different from quoted strings
if c == '\'' {

View File

@@ -179,11 +179,9 @@ func nonSpace(b []byte) bool {
// An Encoder writes JSON values to an output stream.
type Encoder struct {
w io.Writer
err error
escapeHTML bool
nilSafeSlices bool
nilSafeMaps bool
w io.Writer
err error
escapeHTML bool
indentBuf *bytes.Buffer
indentPrefix string
@@ -204,11 +202,8 @@ func (enc *Encoder) Encode(v any) error {
if enc.err != nil {
return enc.err
}
e := newEncodeState()
defer encodeStatePool.Put(e)
err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
if err != nil {
return err
}
@@ -236,6 +231,7 @@ func (enc *Encoder) Encode(v any) error {
if _, err = enc.w.Write(b); err != nil {
enc.err = err
}
encodeStatePool.Put(e)
return err
}
@@ -247,13 +243,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) {
enc.indentValue = indent
}
// SetNilSafeCollection specifies whether to represent nil slices and maps as
// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
enc.nilSafeSlices = nilSafeSlices
enc.nilSafeMaps = nilSafeMaps
}
// SetEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e

View File

@@ -12,7 +12,6 @@ import (
"net/http"
"net/http/httptest"
"reflect"
"runtime/debug"
"strings"
"testing"
)
@@ -42,7 +41,7 @@ false
func TestEncoder(t *testing.T) {
for i := 0; i <= len(streamTest); i++ {
var buf strings.Builder
var buf bytes.Buffer
enc := NewEncoder(&buf)
// Check that enc.SetIndent("", "") turns off indentation.
enc.SetIndent(">", ".")
@@ -60,43 +59,6 @@ func TestEncoder(t *testing.T) {
}
}
func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
// Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test.
percent := debug.SetGCPercent(-1)
defer debug.SetGCPercent(percent)
// Trigger an error in Marshal with cyclic data.
type Dummy struct {
Name string
Next *Dummy
}
dummy := Dummy{Name: "Dummy"}
dummy.Next = &dummy
var buf bytes.Buffer
enc := NewEncoder(&buf)
if err := enc.Encode(dummy); err == nil {
t.Errorf("Encode(dummy) == nil; want error")
}
type Data struct {
A string
I int
}
data := Data{A: "a", I: 1}
if err := enc.Encode(data); err != nil {
t.Errorf("Marshal(%v) = %v", data, err)
}
var data2 Data
if err := Unmarshal(buf.Bytes(), &data2); err != nil {
t.Errorf("Unmarshal(%v) = %v", data2, err)
}
if data2 != data {
t.Errorf("expect: %v, but get: %v", data, data2)
}
}
var streamEncodedIndent = `0.1
"hello"
null
@@ -115,7 +77,7 @@ false
`
func TestEncoderIndent(t *testing.T) {
var buf strings.Builder
var buf bytes.Buffer
enc := NewEncoder(&buf)
enc.SetIndent(">", ".")
for _, v := range streamTest {
@@ -185,7 +147,7 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
`{"bar":"\"<html>foobar</html>\""}`,
},
} {
var buf strings.Builder
var buf bytes.Buffer
enc := NewEncoder(&buf)
if err := enc.Encode(tt.v); err != nil {
t.Errorf("Encode(%s): %s", tt.name, err)
@@ -347,6 +309,21 @@ func TestBlocking(t *testing.T) {
}
}
func BenchmarkEncoderEncode(b *testing.B) {
b.ReportAllocs()
type T struct {
X, Y string
}
v := &T{"foo", "bar"}
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if err := NewEncoder(io.Discard).Encode(v); err != nil {
b.Fatal(err)
}
}
})
}
type tokenStreamCase struct {
json string
expTokens []any
@@ -495,45 +472,3 @@ func TestHTTPDecoding(t *testing.T) {
t.Errorf("err = %v; want io.EOF", err)
}
}
func TestEncoderSetNilSafeCollection(t *testing.T) {
var (
nilSlice []interface{}
pNilSlice *[]interface{}
nilMap map[string]interface{}
pNilMap *map[string]interface{}
)
for _, tt := range []struct {
name string
v interface{}
want string
rescuedWant string
}{
{"nilSlice", nilSlice, "null", "[]"},
{"nonNilSlice", []interface{}{}, "[]", "[]"},
{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
{"pNilSlice", pNilSlice, "null", "null"},
{"nilMap", nilMap, "null", "{}"},
{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
{"pNilMap", pNilMap, "null", "null"},
} {
var buf bytes.Buffer
enc := NewEncoder(&buf)
if err := enc.Encode(tt.v); err != nil {
t.Fatalf("Encode(%s): %s", tt.name, err)
}
if got := strings.TrimSpace(buf.String()); got != tt.want {
t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
}
buf.Reset()
enc.SetNilSafeCollection(true, true)
if err := enc.Encode(tt.v); err != nil {
t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
}
if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
tt.name, got, tt.want)
}
}
}

View File

@@ -1,7 +1,10 @@
package langext
import "runtime/debug"
type PanicWrappedErr struct {
panic any
Stack string
}
func (p PanicWrappedErr) Error() string {
@@ -15,7 +18,7 @@ func (p PanicWrappedErr) ReoveredObj() any {
func RunPanicSafe(fn func()) (err error) {
defer func() {
if rec := recover(); rec != nil {
err = PanicWrappedErr{panic: rec}
err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
}
}()
@@ -27,7 +30,7 @@ func RunPanicSafe(fn func()) (err error) {
func RunPanicSafeR1(fn func() error) (err error) {
defer func() {
if rec := recover(); rec != nil {
err = PanicWrappedErr{panic: rec}
err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
}
}()
@@ -38,7 +41,7 @@ func RunPanicSafeR2[T1 any](fn func() (T1, error)) (r1 T1, err error) {
defer func() {
if rec := recover(); rec != nil {
r1 = *new(T1)
err = PanicWrappedErr{panic: rec}
err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
}
}()
@@ -50,7 +53,7 @@ func RunPanicSafeR3[T1 any, T2 any](fn func() (T1, T2, error)) (r1 T1, r2 T2, er
if rec := recover(); rec != nil {
r1 = *new(T1)
r2 = *new(T2)
err = PanicWrappedErr{panic: rec}
err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
}
}()
@@ -63,7 +66,7 @@ func RunPanicSafeR4[T1 any, T2 any, T3 any](fn func() (T1, T2, T3, error)) (r1 T
r1 = *new(T1)
r2 = *new(T2)
r3 = *new(T3)
err = PanicWrappedErr{panic: rec}
err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
}
}()

View File

@@ -73,7 +73,7 @@ func (c *Coll[TData]) ReplaceOne(ctx context.Context, filterQuery bson.M, value
}
func (c *Coll[TData]) FindOneAndReplace(ctx context.Context, filterQuery bson.M, value TData) (TData, error) {
mongoRes := c.coll.FindOneAndUpdate(ctx, filterQuery, bson.M{"$set": value}, options.FindOneAndUpdate().SetReturnDocument(options.After))
mongoRes := c.coll.FindOneAndReplace(ctx, filterQuery, value, options.FindOneAndReplace().SetReturnDocument(options.After))
if err := mongoRes.Err(); err != nil {
return *new(TData), exerr.Wrap(err, "mongo-query[find-one-and-update] failed").
Str("collection", c.Name()).