v0.0.277 DYN-166 json marshal filter in ginext Write

exerr/exerr.go

@@ -164,7 +164,7 @@ func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
 }
 
 func (ee *ExErr) ShortLog(evt *zerolog.Event) {
-	ee.Meta.Apply(evt).Msg(ee.FormatLog(LogPrintShort))
+	ee.Meta.Apply(evt, langext.Ptr(240)).Msg(ee.FormatLog(LogPrintShort))
 }
 
 // RecursiveMessage returns the message to show

exerr/meta.go

@@ -217,23 +217,35 @@ func (v MetaValue) ShortString(lim int) string {
 	return "(err)"
 }
 
-func (v MetaValue) Apply(key string, evt *zerolog.Event) *zerolog.Event {
+func (v MetaValue) Apply(key string, evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	switch v.DataType {
 	case MDTString:
-		return evt.Str(key, v.Value.(string))
+		if limitLen == nil {
+			return evt.Str(key, v.Value.(string))
+		} else {
+			return evt.Str(key, langext.StrLimit(v.Value.(string), *limitLen, "..."))
+		}
 	case MDTID:
 		return evt.Str(key, v.Value.(IDWrap).Value)
 	case MDTAny:
 		if v.Value.(AnyWrap).IsError {
 			return evt.Str(key, "(err)")
 		} else {
-			return evt.Str(key, v.Value.(AnyWrap).Json)
+			if limitLen == nil {
+				return evt.Str(key, v.Value.(AnyWrap).Json)
+			} else {
+				return evt.Str(key, langext.StrLimit(v.Value.(AnyWrap).Json, *limitLen, "..."))
+			}
 		}
 	case MDTStringPtr:
 		if langext.IsNil(v.Value) {
 			return evt.Str(key, "<<null>>")
 		}
-		return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		if limitLen == nil {
+			return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		} else {
+			return evt.Str(key, langext.StrLimit(langext.CoalesceString(v.Value.(*string), "<<null>>"), *limitLen, "..."))
+		}
 	case MDTInt:
 		return evt.Int(key, v.Value.(int))
 	case MDTInt8:
@@ -702,9 +714,9 @@ func (mm MetaMap) Any() bool {
 	return len(mm) > 0
 }
 
-func (mm MetaMap) Apply(evt *zerolog.Event) *zerolog.Event {
+func (mm MetaMap) Apply(evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	for key, val := range mm {
-		evt = val.Apply(key, evt)
+		evt = val.Apply(key, evt, limitLen)
 	}
 	return evt
 }

ginext/response.go

@@ -27,7 +27,11 @@ func (j jsonHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
-	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true})
+	var f *string
+	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
+		f = &jsonfilter
+	}
+	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f})
 }
 
 func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.15.4 // indirect
+	github.com/go-playground/validator/v10 v10.15.5 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

go.sum

@@ -38,6 +38,8 @@ github.com/go-playground/validator/v10 v10.15.3 h1:S+sSpunYjNPDuXkWbK+x+bA7iXiW2
 github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs=
 github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
+github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.273"
+const GoextVersion = "0.0.277"
 
-const GoextVersionTimestamp = "2023-09-27T14:15:59+0200"
+const GoextVersionTimestamp = "2023-10-05T12:00:51+0200"

gojson/encode.go

@@ -156,7 +156,6 @@ import (
 // an error.
 func Marshal(v any) ([]byte, error) {
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
 
 	err := e.marshal(v, encOpts{escapeHTML: true})
 	if err != nil {
@@ -164,6 +163,8 @@ func Marshal(v any) ([]byte, error) {
 	}
 	buf := append([]byte(nil), e.Bytes()...)
 
+	encodeStatePool.Put(e)
+
 	return buf, nil
 }
 
@@ -174,9 +175,9 @@ type IndentOpt struct {
 
 // MarshalSafeCollections is like Marshal except it will marshal nil maps and
 // slices as '{}' and '[]' respectfully instead of 'null'
-func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt) ([]byte, error) {
+func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt, filter *string) ([]byte, error) {
 	e := &encodeState{}
-	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps})
+	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps, filter: filter})
 	if err != nil {
 		return nil, err
 	}
@@ -393,6 +394,9 @@ type encOpts struct {
 	nilSafeSlices bool
 	// nilSafeMaps marshals a nil maps '{}' instead of 'null'
 	nilSafeMaps bool
+	// filter matches jsonfilter tag of struct
+	// marshals if no jsonfilter is set or otherwise if jsonfilter has the filter value
+	filter *string
 }
 
 type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
@@ -777,6 +781,8 @@ FieldLoop:
 
 		if f.omitEmpty && isEmptyValue(fv) {
 			continue
+		} else if opts.filter != nil && len(f.jsonfilter) > 0 && !f.jsonfilter.Contains(*opts.filter) {
+			continue
 		}
 		e.WriteByte(next)
 		next = ','
@@ -1220,15 +1226,28 @@ type field struct {
 	nameNonEsc  string // `"` + name + `":`
 	nameEscHTML string // `"` + HTMLEscape(name) + `":`
 
-	tag       bool
+	tag        bool
-	index     []int
+	index      []int
-	typ       reflect.Type
+	typ        reflect.Type
-	omitEmpty bool
+	omitEmpty  bool
-	quoted    bool
+	jsonfilter jsonfilter
+	quoted     bool
 
 	encoder encoderFunc
 }
 
+// jsonfilter stores the value of the jsonfilter struct tag
+type jsonfilter []string
+
+func (j jsonfilter) Contains(t string) bool {
+	for _, tag := range j {
+		if t == tag {
+			return true
+		}
+	}
+	return false
+}
+
 // byIndex sorts field by index sequence.
 type byIndex []field
 
@@ -1304,6 +1323,13 @@ func typeFields(t reflect.Type) structFields {
 				if !isValidTag(name) {
 					name = ""
 				}
+
+				var jsonfilter []string
+				jsonfilterTag := sf.Tag.Get("jsonfilter")
+				if isValidTag(jsonfilterTag) {
+					jsonfilter = strings.Split(jsonfilterTag, ",")
+				}
+
 				index := make([]int, len(f.index)+1)
 				copy(index, f.index)
 				index[len(f.index)] = i
@@ -1334,12 +1360,13 @@ func typeFields(t reflect.Type) structFields {
 						name = sf.Name
 					}
 					field := field{
-						name:      name,
+						name:       name,
-						tag:       tagged,
+						tag:        tagged,
-						index:     index,
+						index:      index,
-						typ:       ft,
+						typ:        ft,
-						omitEmpty: opts.Contains("omitempty"),
+						omitEmpty:  opts.Contains("omitempty"),
-						quoted:    quoted,
+						jsonfilter: jsonfilter,
+						quoted:     quoted,
 					}
 					field.nameBytes = []byte(field.name)
 					field.equalFold = foldFunc(field.nameBytes)

gojson/encode_test.go

@@ -1253,6 +1253,10 @@ func TestMarshalSafeCollections(t *testing.T) {
 		nilMapStruct struct {
 			NilMap map[string]interface{} `json:"nil_map"`
 		}
+		testWithFilter struct {
+			Test1 string `json:"test1" jsonfilter:"FILTERONE"`
+			Test2 string `json:"test2" jsonfilter:"FILTERTWO"`
+		}
 	)
 
 	tests := []struct {
@@ -1271,10 +1275,12 @@ func TestMarshalSafeCollections(t *testing.T) {
 		{map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}"},
 		{pNilMap, "null"},
 		{nilMapStruct{}, "{\"nil_map\":{}}"},
+		{testWithFilter{}, "{\"test1\":\"\"}"},
 	}
 
+	filter := "FILTERONE"
 	for i, tt := range tests {
-		b, err := MarshalSafeCollections(tt.in, true, true, nil)
+		b, err := MarshalSafeCollections(tt.in, true, true, nil, &filter)
 		if err != nil {
 			t.Errorf("test %d, unexpected failure: %v", i, err)
 		}

gojson/fold.go

@@ -97,7 +97,10 @@ func equalFoldRight(s, t []byte) bool {
 		t = t[size:]
 
 	}
-	return len(t) == 0
+	if len(t) > 0 {
+		return false
+	}
+	return true
 }
 
 // asciiEqualFold is a specialization of bytes.EqualFold for use when

gojson/fold_test.go

@@ -52,7 +52,9 @@ func TestFold(t *testing.T) {
 }
 
 func TestFoldAgainstUnicode(t *testing.T) {
-	var buf1, buf2 []byte
+	const bufSize = 5
+	buf1 := make([]byte, 0, bufSize)
+	buf2 := make([]byte, 0, bufSize)
 	var runes []rune
 	for i := 0x20; i <= 0x7f; i++ {
 		runes = append(runes, rune(i))
@@ -94,8 +96,12 @@ func TestFoldAgainstUnicode(t *testing.T) {
 				continue
 			}
 			for _, r2 := range runes {
-				buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x')
+				buf1 := append(buf1[:0], 'x')
-				buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x')
+				buf2 := append(buf2[:0], 'x')
+				buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
+				buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
+				buf1 = append(buf1, 'x')
+				buf2 = append(buf2, 'x')
 				want := bytes.EqualFold(buf1, buf2)
 				if got := ff.fold(buf1, buf2); got != want {
 					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)

gojson/gionic.go

@@ -17,6 +17,7 @@ type GoJsonRender struct {
 	NilSafeSlices bool
 	NilSafeMaps   bool
 	Indent        *IndentOpt
+	Filter        *string
 }
 
 func (r GoJsonRender) Render(w http.ResponseWriter) error {
@@ -25,7 +26,7 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
 		header["Content-Type"] = []string{"application/json; charset=utf-8"}
 	}
 
-	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent)
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
 	if err != nil {
 		panic(err)
 	}

gojson/number_test.go

@@ -116,3 +116,18 @@ func TestNumberIsValid(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkNumberIsValid(b *testing.B) {
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		isValidNumber(s)
+	}
+}
+
+func BenchmarkNumberIsValidRegexp(b *testing.B) {
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		jsonNumberRegexp.MatchString(s)
+	}
+}

gojson/scanner.go

@@ -594,7 +594,7 @@ func (s *scanner) error(c byte, context string) int {
 	return scanError
 }
 
-// quoteChar formats c as a quoted character literal.
+// quoteChar formats c as a quoted character literal
 func quoteChar(c byte) string {
 	// special cases - different from quoted strings
 	if c == '\'' {

gojson/stream.go

@@ -179,11 +179,9 @@ func nonSpace(b []byte) bool {
 
 // An Encoder writes JSON values to an output stream.
 type Encoder struct {
-	w             io.Writer
+	w          io.Writer
-	err           error
+	err        error
-	escapeHTML    bool
+	escapeHTML bool
-	nilSafeSlices bool
-	nilSafeMaps   bool
 
 	indentBuf    *bytes.Buffer
 	indentPrefix string
@@ -204,11 +202,8 @@ func (enc *Encoder) Encode(v any) error {
 	if enc.err != nil {
 		return enc.err
 	}
-
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
+	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
-
-	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
 	if err != nil {
 		return err
 	}
@@ -236,6 +231,7 @@ func (enc *Encoder) Encode(v any) error {
 	if _, err = enc.w.Write(b); err != nil {
 		enc.err = err
 	}
+	encodeStatePool.Put(e)
 	return err
 }
 
@@ -247,13 +243,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) {
 	enc.indentValue = indent
 }
 
-// SetNilSafeCollection specifies whether to represent nil slices and maps as
-// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
-func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
-	enc.nilSafeSlices = nilSafeSlices
-	enc.nilSafeMaps = nilSafeMaps
-}
-
 // SetEscapeHTML specifies whether problematic HTML characters
 // should be escaped inside JSON quoted strings.
 // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e

gojson/stream_test.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
-	"runtime/debug"
 	"strings"
 	"testing"
 )
@@ -42,7 +41,7 @@ false
 
 func TestEncoder(t *testing.T) {
 	for i := 0; i <= len(streamTest); i++ {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		// Check that enc.SetIndent("", "") turns off indentation.
 		enc.SetIndent(">", ".")
@@ -60,43 +59,6 @@ func TestEncoder(t *testing.T) {
 	}
 }
 
-func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
-	// Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test.
-	percent := debug.SetGCPercent(-1)
-	defer debug.SetGCPercent(percent)
-
-	// Trigger an error in Marshal with cyclic data.
-	type Dummy struct {
-		Name string
-		Next *Dummy
-	}
-	dummy := Dummy{Name: "Dummy"}
-	dummy.Next = &dummy
-
-	var buf bytes.Buffer
-	enc := NewEncoder(&buf)
-	if err := enc.Encode(dummy); err == nil {
-		t.Errorf("Encode(dummy) == nil; want error")
-	}
-
-	type Data struct {
-		A string
-		I int
-	}
-	data := Data{A: "a", I: 1}
-	if err := enc.Encode(data); err != nil {
-		t.Errorf("Marshal(%v) = %v", data, err)
-	}
-
-	var data2 Data
-	if err := Unmarshal(buf.Bytes(), &data2); err != nil {
-		t.Errorf("Unmarshal(%v) = %v", data2, err)
-	}
-	if data2 != data {
-		t.Errorf("expect: %v, but get: %v", data, data2)
-	}
-}
-
 var streamEncodedIndent = `0.1
 "hello"
 null
@@ -115,7 +77,7 @@ false
 `
 
 func TestEncoderIndent(t *testing.T) {
-	var buf strings.Builder
+	var buf bytes.Buffer
 	enc := NewEncoder(&buf)
 	enc.SetIndent(">", ".")
 	for _, v := range streamTest {
@@ -185,7 +147,7 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
 			`{"bar":"\"<html>foobar</html>\""}`,
 		},
 	} {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		if err := enc.Encode(tt.v); err != nil {
 			t.Errorf("Encode(%s): %s", tt.name, err)
@@ -347,6 +309,21 @@ func TestBlocking(t *testing.T) {
 	}
 }
 
+func BenchmarkEncoderEncode(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X, Y string
+	}
+	v := &T{"foo", "bar"}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			if err := NewEncoder(io.Discard).Encode(v); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
 type tokenStreamCase struct {
 	json      string
 	expTokens []any
@@ -495,45 +472,3 @@ func TestHTTPDecoding(t *testing.T) {
 		t.Errorf("err = %v; want io.EOF", err)
 	}
 }
-
-func TestEncoderSetNilSafeCollection(t *testing.T) {
-	var (
-		nilSlice  []interface{}
-		pNilSlice *[]interface{}
-		nilMap    map[string]interface{}
-		pNilMap   *map[string]interface{}
-	)
-	for _, tt := range []struct {
-		name        string
-		v           interface{}
-		want        string
-		rescuedWant string
-	}{
-		{"nilSlice", nilSlice, "null", "[]"},
-		{"nonNilSlice", []interface{}{}, "[]", "[]"},
-		{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
-		{"pNilSlice", pNilSlice, "null", "null"},
-		{"nilMap", nilMap, "null", "{}"},
-		{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
-		{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
-		{"pNilMap", pNilMap, "null", "null"},
-	} {
-		var buf bytes.Buffer
-		enc := NewEncoder(&buf)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.want {
-			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
-		}
-		buf.Reset()
-		enc.SetNilSafeCollection(true, true)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
-			t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
-				tt.name, got, tt.want)
-		}
-	}
-}