v0.0.278 DYN-166 bugfix jsonfilter

Makefile

@@ -7,5 +7,11 @@ test:
 	which gotestsum || go install gotest.tools/gotestsum@latest
 	gotestsum --format "testname" -- -tags="timetzdata sqlite_fts5 sqlite_foreign_keys" "./..."
 
+test-in-docker:
+	tag="goext_temp_test_image:$(shell uuidgen | tr -d '-')";        \
+	docker build --tag $$tag . -f .gitea/workflows/Dockerfile_tests; \
+	docker run --rm $$tag;                                           \
+	docker rmi $$tag
+
 version:
 	_data/version.sh

TODO.md

@@ -2,12 +2,6 @@
 
  - cronext
 
- - cursortoken
-
- - typed/geenric mongo wrapper
-
- - error package
-
-- rfctime.DateOnly
-- rfctime.HMSTimeOnly
-- rfctime.NanoTimeOnly
+ - rfctime.DateOnly
+ - rfctime.HMSTimeOnly
+ - rfctime.NanoTimeOnly

bfcodegen/enum-generate.go

@@ -31,13 +31,13 @@ type EnumDef struct {
 	Values       []EnumDefVal
 }
 
-var rexPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
 
-var rexValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:]+"|[0-9]+))\s*(//(?P<descr>.*))?.*$`))
+var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:]+"|[0-9]+))\s*(//(?P<descr>.*))?.*$`))
 
-var rexChecksumConst = rext.W(regexp.MustCompile(`const ChecksumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
 
 func GenerateEnumSpecs(sourceDir string, destFile string) error {
 
@@ -52,13 +52,14 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		if err != nil {
 			return err
 		}
-		if m, ok := rexChecksumConst.MatchFirst(string(content)); ok {
+		if m, ok := rexEnumChecksumConst.MatchFirst(string(content)); ok {
 			oldChecksum = m.GroupByName("cs").Value()
 		}
 	}
 
 	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
 	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
 	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
 
 	newChecksumStr := goext.GoextVersion
@@ -85,7 +86,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 
 	for _, f := range files {
 		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileEnums, pn, err := processFile(sourceDir, path.Join(sourceDir, f.Name()))
+		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()))
 		if err != nil {
 			return err
 		}
@@ -103,7 +104,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		return errors.New("no package name found in any file")
 	}
 
-	err = os.WriteFile(destFile, []byte(fmtOutput(newChecksum, allEnums, pkgname)), 0o755)
+	err = os.WriteFile(destFile, []byte(fmtEnumOutput(newChecksum, allEnums, pkgname)), 0o755)
 	if err != nil {
 		return err
 	}
@@ -125,7 +126,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func processFile(basedir string, fn string) ([]EnumDef, string, error) {
+func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -149,7 +150,7 @@ func processFile(basedir string, fn string) ([]EnumDef, string, error) {
 			break
 		}
 
-		if match, ok := rexPackage.MatchFirst(line); i == 0 && ok {
+		if match, ok := rexEnumPackage.MatchFirst(line); i == 0 && ok {
 			pkgname = match.GroupByName("name").Value()
 			continue
 		}
@@ -172,7 +173,7 @@ func processFile(basedir string, fn string) ([]EnumDef, string, error) {
 			fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
 		}
 
-		if match, ok := rexValueDef.MatchFirst(line); ok {
+		if match, ok := rexEnumValueDef.MatchFirst(line); ok {
 			typename := match.GroupByName("type").Value()
 			def := EnumDefVal{
 				VarName:     match.GroupByName("name").Value(),
@@ -202,7 +203,7 @@ func processFile(basedir string, fn string) ([]EnumDef, string, error) {
 	return enums, pkgname, nil
 }
 
-func fmtOutput(cs string, enums []EnumDef, pkgname string) string {
+func fmtEnumOutput(cs string, enums []EnumDef, pkgname string) string {
 	str := "// Code generated by enum-generate.go DO NOT EDIT.\n"
 	str += "\n"
 	str += "package " + pkgname + "\n"
@@ -212,7 +213,7 @@ func fmtOutput(cs string, enums []EnumDef, pkgname string) string {
 	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/enums\"" + "\n"
 	str += "\n"
 
-	str += "const ChecksumGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
+	str += "const ChecksumEnumGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
 	str += "\n"
 
 	for _, enumdef := range enums {

bfcodegen/id-generate.go

@@ -0,0 +1,236 @@
+package bfcodegen
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type IDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+}
+
+var rexIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@id:type).*$`))
+
+var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+func GenerateIDSpecs(sourceDir string, destFile string) error {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[IDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[IDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]IDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\n")
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	err = os.WriteFile(destFile, []byte(fmtIDOutput(newChecksum, allIDs, pkgname)), 0o755)
+	if err != nil {
+		return err
+	}
+
+	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
+	if err != nil {
+		return err
+	}
+
+	if res.CommandTimedOut {
+		fmt.Println(res.StdCombined)
+		return errors.New("go fmt timed out")
+	}
+	if res.ExitCode != 0 {
+		fmt.Println(res.StdCombined)
+		return errors.New("go fmt did not succeed")
+	}
+
+	return nil
+}
+
+func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]IDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := IDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+			}
+			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtIDOutput(cs string, ids []IDDef, pkgname string) string {
+	str := "// Code generated by id-generate.go DO NOT EDIT.\n"
+	str += "\n"
+	str += "package " + pkgname + "\n"
+	str += "\n"
+
+	str += "import \"go.mongodb.org/mongo-driver/bson\"" + "\n"
+	str += "import \"go.mongodb.org/mongo-driver/bson/bsontype\"" + "\n"
+	str += "import \"go.mongodb.org/mongo-driver/bson/primitive\"" + "\n"
+	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/exerr\"" + "\n"
+	str += "\n"
+
+	str += "const ChecksumIDGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
+	str += "\n"
+
+	anyDef := langext.ArrFirstOrNil(ids, func(def IDDef) bool { return def.Name == "AnyID" || def.Name == "AnyId" })
+
+	for _, iddef := range ids {
+
+		str += "// ================================ " + iddef.Name + " (" + iddef.FileRelative + ") ================================" + "\n"
+		str += "" + "\n"
+
+		str += "func (i " + iddef.Name + ") MarshalBSONValue() (bsontype.Type, []byte, error) {" + "\n"
+		str += "	if objId, err := primitive.ObjectIDFromHex(string(i)); err == nil {" + "\n"
+		str += "		return bson.MarshalValue(objId)" + "\n"
+		str += "	} else {" + "\n"
+		str += "		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, \"Failed to marshal " + iddef.Name + "(\"+i.String()+\") to ObjectId\").Str(\"value\", string(i)).Type(\"type\", i).Build()" + "\n"
+		str += "	}" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (i " + iddef.Name + ") String() string {" + "\n"
+		str += "	return string(i)" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (i " + iddef.Name + ") ObjID() (primitive.ObjectID, error) {" + "\n"
+		str += "	return primitive.ObjectIDFromHex(string(i))" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (i " + iddef.Name + ") Valid() bool {" + "\n"
+		str += "	_, err := primitive.ObjectIDFromHex(string(i))" + "\n"
+		str += "	return err == nil" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		if anyDef != nil {
+			str += "func (i " + iddef.Name + ") AsAny() " + anyDef.Name + " {" + "\n"
+			str += "	return " + anyDef.Name + "(i)" + "\n"
+			str += "}" + "\n"
+
+			str += "" + "\n"
+		}
+
+		str += "func New" + iddef.Name + "() " + iddef.Name + " {" + "\n"
+		str += "	return " + iddef.Name + "(primitive.NewObjectID().Hex())" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+	}
+
+	return str
+}

cmdext/cmdrunner.go

@@ -133,9 +133,6 @@ func run(opt CommandRunner) (CommandResult, error) {
 
 	case <-stderrFailChan:
 		_ = cmd.Process.Kill()
-		for _, lstr := range opt.listener {
-			lstr.Timeout()
-		}
 
 		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, 32*time.Millisecond); ok {
 			// most of the time the cmd.Process.Kill() should also have finished the pipereader
@@ -160,7 +157,8 @@ func run(opt CommandRunner) (CommandResult, error) {
 		}
 
 	case outobj := <-outputChan:
-		if exiterr, ok := outobj.err.(*exec.ExitError); ok {
+		var exiterr *exec.ExitError
+		if errors.As(outobj.err, &exiterr) {
 			excode := exiterr.ExitCode()
 			for _, lstr := range opt.listener {
 				lstr.Finished(excode)

cmdext/pipereader.go

@@ -32,8 +32,8 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	stdout := ""
 	go func() {
 		buf := make([]byte, 128)
-		for true {
-			n, out := pr.stdout.Read(buf)
+		for {
+			n, err := pr.stdout.Read(buf)
 			if n > 0 {
 				txt := string(buf[:n])
 				stdout += txt
@@ -42,11 +42,11 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 					lstr.ReadRawStdout(buf[:n])
 				}
 			}
-			if out == io.EOF {
+			if err == io.EOF {
 				break
 			}
-			if out != nil {
-				errch <- out
+			if err != nil {
+				errch <- err
 				break
 			}
 		}
@@ -61,7 +61,7 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	stderr := ""
 	go func() {
 		buf := make([]byte, 128)
-		for true {
+		for {
 			n, err := pr.stderr.Read(buf)
 
 			if n > 0 {

exerr/data.go

@@ -55,6 +55,8 @@ var (
 	TypeBindFailFormData = NewType("BINDFAIL_FORMDATA", langext.Ptr(400))
 	TypeBindFailHeader   = NewType("BINDFAIL_HEADER", langext.Ptr(400))
 
+	TypeMarshalEntityID = NewType("MARSHAL_ENTITY_ID", langext.Ptr(400))
+
 	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
 	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))
 

exerr/exerr.go

@@ -164,7 +164,7 @@ func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
 }
 
 func (ee *ExErr) ShortLog(evt *zerolog.Event) {
-	ee.Meta.Apply(evt).Msg(ee.FormatLog(LogPrintShort))
+	ee.Meta.Apply(evt, langext.Ptr(240)).Msg(ee.FormatLog(LogPrintShort))
 }
 
 // RecursiveMessage returns the message to show

exerr/meta.go

@@ -217,23 +217,35 @@ func (v MetaValue) ShortString(lim int) string {
 	return "(err)"
 }
 
-func (v MetaValue) Apply(key string, evt *zerolog.Event) *zerolog.Event {
+func (v MetaValue) Apply(key string, evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	switch v.DataType {
 	case MDTString:
-		return evt.Str(key, v.Value.(string))
+		if limitLen == nil {
+			return evt.Str(key, v.Value.(string))
+		} else {
+			return evt.Str(key, langext.StrLimit(v.Value.(string), *limitLen, "..."))
+		}
 	case MDTID:
 		return evt.Str(key, v.Value.(IDWrap).Value)
 	case MDTAny:
 		if v.Value.(AnyWrap).IsError {
 			return evt.Str(key, "(err)")
 		} else {
-			return evt.Str(key, v.Value.(AnyWrap).Json)
+			if limitLen == nil {
+				return evt.Str(key, v.Value.(AnyWrap).Json)
+			} else {
+				return evt.Str(key, langext.StrLimit(v.Value.(AnyWrap).Json, *limitLen, "..."))
+			}
 		}
 	case MDTStringPtr:
 		if langext.IsNil(v.Value) {
 			return evt.Str(key, "<<null>>")
 		}
-		return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		if limitLen == nil {
+			return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		} else {
+			return evt.Str(key, langext.StrLimit(langext.CoalesceString(v.Value.(*string), "<<null>>"), *limitLen, "..."))
+		}
 	case MDTInt:
 		return evt.Int(key, v.Value.(int))
 	case MDTInt8:
@@ -702,9 +714,9 @@ func (mm MetaMap) Any() bool {
 	return len(mm) > 0
 }
 
-func (mm MetaMap) Apply(evt *zerolog.Event) *zerolog.Event {
+func (mm MetaMap) Apply(evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	for key, val := range mm {
-		evt = val.Apply(key, evt)
+		evt = val.Apply(key, evt, limitLen)
 	}
 	return evt
 }

ginext/response.go

@@ -27,7 +27,11 @@ func (j jsonHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
-	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true})
+	var f *string
+	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
+		f = &jsonfilter
+	}
+	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f})
 }
 
 func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {

go.mod

@@ -6,26 +6,26 @@ require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/rs/xid v1.5.0
-	github.com/rs/zerolog v1.30.0
+	github.com/rs/zerolog v1.31.0
 	go.mongodb.org/mongo-driver v1.12.1
-	golang.org/x/crypto v0.12.0
+	golang.org/x/crypto v0.13.0
 	golang.org/x/sys v0.12.0
 	golang.org/x/term v0.12.0
 )
 
 require (
-	github.com/bytedance/sonic v1.10.0 // indirect
+	github.com/bytedance/sonic v1.10.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.15.3 // indirect
+	github.com/go-playground/validator/v10 v10.15.5 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -42,7 +42,7 @@ require (
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	golang.org/x/arch v0.5.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

go.sum

@@ -6,6 +6,8 @@ github.com/bytedance/sonic v1.10.0-rc3 h1:uNSnscRapXTwUgTyOF0GVljYD08p9X/Lbr9Mwe
 github.com/bytedance/sonic v1.10.0-rc3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk=
 github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc=
+github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
@@ -34,6 +36,10 @@ github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2r
 github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-playground/validator/v10 v10.15.3 h1:S+sSpunYjNPDuXkWbK+x+bA7iXiW296KG4dL3X7xUZo=
 github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs=
+github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
+github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
@@ -54,6 +60,8 @@ github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQ
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -99,6 +107,8 @@ github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -151,6 +161,8 @@ golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -163,6 +175,8 @@ golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
 golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.257"
+const GoextVersion = "0.0.278"
 
-const GoextVersionTimestamp = "2023-09-05T15:01:55+0200"
+const GoextVersionTimestamp = "2023-10-05T12:54:07+0200"

gojson/encode.go

@@ -156,7 +156,6 @@ import (
 // an error.
 func Marshal(v any) ([]byte, error) {
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
 
 	err := e.marshal(v, encOpts{escapeHTML: true})
 	if err != nil {
@@ -164,6 +163,8 @@ func Marshal(v any) ([]byte, error) {
 	}
 	buf := append([]byte(nil), e.Bytes()...)
 
+	encodeStatePool.Put(e)
+
 	return buf, nil
 }
 
@@ -174,9 +175,9 @@ type IndentOpt struct {
 
 // MarshalSafeCollections is like Marshal except it will marshal nil maps and
 // slices as '{}' and '[]' respectfully instead of 'null'
-func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt) ([]byte, error) {
+func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt, filter *string) ([]byte, error) {
 	e := &encodeState{}
-	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps})
+	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps, filter: filter})
 	if err != nil {
 		return nil, err
 	}
@@ -393,6 +394,9 @@ type encOpts struct {
 	nilSafeSlices bool
 	// nilSafeMaps marshals a nil maps '{}' instead of 'null'
 	nilSafeMaps bool
+	// filter matches jsonfilter tag of struct
+	// marshals if no jsonfilter is set or otherwise if jsonfilter has the filter value
+	filter *string
 }
 
 type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
@@ -777,6 +781,8 @@ FieldLoop:
 
 		if f.omitEmpty && isEmptyValue(fv) {
 			continue
+		} else if opts.filter != nil && len(f.jsonfilter) > 0 && !f.jsonfilter.Contains(*opts.filter) {
+			continue
 		}
 		e.WriteByte(next)
 		next = ','
@@ -1220,15 +1226,28 @@ type field struct {
 	nameNonEsc  string // `"` + name + `":`
 	nameEscHTML string // `"` + HTMLEscape(name) + `":`
 
-	tag       bool
-	index     []int
-	typ       reflect.Type
-	omitEmpty bool
-	quoted    bool
+	tag        bool
+	index      []int
+	typ        reflect.Type
+	omitEmpty  bool
+	jsonfilter jsonfilter
+	quoted     bool
 
 	encoder encoderFunc
 }
 
+// jsonfilter stores the value of the jsonfilter struct tag
+type jsonfilter []string
+
+func (j jsonfilter) Contains(t string) bool {
+	for _, tag := range j {
+		if t == tag {
+			return true
+		}
+	}
+	return false
+}
+
 // byIndex sorts field by index sequence.
 type byIndex []field
 
@@ -1304,6 +1323,13 @@ func typeFields(t reflect.Type) structFields {
 				if !isValidTag(name) {
 					name = ""
 				}
+
+				var jsonfilter []string
+				jsonfilterTag := sf.Tag.Get("jsonfilter")
+				if jsonfilterTag != "" && jsonfilterTag != "-" {
+					jsonfilter = strings.Split(jsonfilterTag, ",")
+				}
+
 				index := make([]int, len(f.index)+1)
 				copy(index, f.index)
 				index[len(f.index)] = i
@@ -1334,12 +1360,13 @@ func typeFields(t reflect.Type) structFields {
 						name = sf.Name
 					}
 					field := field{
-						name:      name,
-						tag:       tagged,
-						index:     index,
-						typ:       ft,
-						omitEmpty: opts.Contains("omitempty"),
-						quoted:    quoted,
+						name:       name,
+						tag:        tagged,
+						index:      index,
+						typ:        ft,
+						omitEmpty:  opts.Contains("omitempty"),
+						jsonfilter: jsonfilter,
+						quoted:     quoted,
 					}
 					field.nameBytes = []byte(field.name)
 					field.equalFold = foldFunc(field.nameBytes)

gojson/encode_test.go

@@ -1253,6 +1253,10 @@ func TestMarshalSafeCollections(t *testing.T) {
 		nilMapStruct struct {
 			NilMap map[string]interface{} `json:"nil_map"`
 		}
+		testWithFilter struct {
+			Test1 string `json:"test1" jsonfilter:"FILTERONE"`
+			Test2 string `json:"test2" jsonfilter:"FILTERTWO"`
+		}
 	)
 
 	tests := []struct {
@@ -1271,10 +1275,12 @@ func TestMarshalSafeCollections(t *testing.T) {
 		{map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}"},
 		{pNilMap, "null"},
 		{nilMapStruct{}, "{\"nil_map\":{}}"},
+		{testWithFilter{}, "{\"test1\":\"\"}"},
 	}
 
+	filter := "FILTERONE"
 	for i, tt := range tests {
-		b, err := MarshalSafeCollections(tt.in, true, true, nil)
+		b, err := MarshalSafeCollections(tt.in, true, true, nil, &filter)
 		if err != nil {
 			t.Errorf("test %d, unexpected failure: %v", i, err)
 		}

gojson/fold.go

@@ -97,7 +97,10 @@ func equalFoldRight(s, t []byte) bool {
 		t = t[size:]
 
 	}
-	return len(t) == 0
+	if len(t) > 0 {
+		return false
+	}
+	return true
 }
 
 // asciiEqualFold is a specialization of bytes.EqualFold for use when

gojson/fold_test.go

@@ -52,7 +52,9 @@ func TestFold(t *testing.T) {
 }
 
 func TestFoldAgainstUnicode(t *testing.T) {
-	var buf1, buf2 []byte
+	const bufSize = 5
+	buf1 := make([]byte, 0, bufSize)
+	buf2 := make([]byte, 0, bufSize)
 	var runes []rune
 	for i := 0x20; i <= 0x7f; i++ {
 		runes = append(runes, rune(i))
@@ -94,8 +96,12 @@ func TestFoldAgainstUnicode(t *testing.T) {
 				continue
 			}
 			for _, r2 := range runes {
-				buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x')
-				buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x')
+				buf1 := append(buf1[:0], 'x')
+				buf2 := append(buf2[:0], 'x')
+				buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
+				buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
+				buf1 = append(buf1, 'x')
+				buf2 = append(buf2, 'x')
 				want := bytes.EqualFold(buf1, buf2)
 				if got := ff.fold(buf1, buf2); got != want {
 					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)

gojson/gionic.go

@@ -17,6 +17,7 @@ type GoJsonRender struct {
 	NilSafeSlices bool
 	NilSafeMaps   bool
 	Indent        *IndentOpt
+	Filter        *string
 }
 
 func (r GoJsonRender) Render(w http.ResponseWriter) error {
@@ -25,7 +26,7 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
 		header["Content-Type"] = []string{"application/json; charset=utf-8"}
 	}
 
-	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent)
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
 	if err != nil {
 		panic(err)
 	}

gojson/number_test.go

@@ -116,3 +116,18 @@ func TestNumberIsValid(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkNumberIsValid(b *testing.B) {
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		isValidNumber(s)
+	}
+}
+
+func BenchmarkNumberIsValidRegexp(b *testing.B) {
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		jsonNumberRegexp.MatchString(s)
+	}
+}

gojson/scanner.go

@@ -594,7 +594,7 @@ func (s *scanner) error(c byte, context string) int {
 	return scanError
 }
 
-// quoteChar formats c as a quoted character literal.
+// quoteChar formats c as a quoted character literal
 func quoteChar(c byte) string {
 	// special cases - different from quoted strings
 	if c == '\'' {

gojson/stream.go

@@ -179,11 +179,9 @@ func nonSpace(b []byte) bool {
 
 // An Encoder writes JSON values to an output stream.
 type Encoder struct {
-	w             io.Writer
-	err           error
-	escapeHTML    bool
-	nilSafeSlices bool
-	nilSafeMaps   bool
+	w          io.Writer
+	err        error
+	escapeHTML bool
 
 	indentBuf    *bytes.Buffer
 	indentPrefix string
@@ -204,11 +202,8 @@ func (enc *Encoder) Encode(v any) error {
 	if enc.err != nil {
 		return enc.err
 	}
-
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
-
-	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
+	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
 	if err != nil {
 		return err
 	}
@@ -236,6 +231,7 @@ func (enc *Encoder) Encode(v any) error {
 	if _, err = enc.w.Write(b); err != nil {
 		enc.err = err
 	}
+	encodeStatePool.Put(e)
 	return err
 }
 
@@ -247,13 +243,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) {
 	enc.indentValue = indent
 }
 
-// SetNilSafeCollection specifies whether to represent nil slices and maps as
-// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
-func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
-	enc.nilSafeSlices = nilSafeSlices
-	enc.nilSafeMaps = nilSafeMaps
-}
-
 // SetEscapeHTML specifies whether problematic HTML characters
 // should be escaped inside JSON quoted strings.
 // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e

gojson/stream_test.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
-	"runtime/debug"
 	"strings"
 	"testing"
 )
@@ -42,7 +41,7 @@ false
 
 func TestEncoder(t *testing.T) {
 	for i := 0; i <= len(streamTest); i++ {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		// Check that enc.SetIndent("", "") turns off indentation.
 		enc.SetIndent(">", ".")
@@ -60,43 +59,6 @@ func TestEncoder(t *testing.T) {
 	}
 }
 
-func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
-	// Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test.
-	percent := debug.SetGCPercent(-1)
-	defer debug.SetGCPercent(percent)
-
-	// Trigger an error in Marshal with cyclic data.
-	type Dummy struct {
-		Name string
-		Next *Dummy
-	}
-	dummy := Dummy{Name: "Dummy"}
-	dummy.Next = &dummy
-
-	var buf bytes.Buffer
-	enc := NewEncoder(&buf)
-	if err := enc.Encode(dummy); err == nil {
-		t.Errorf("Encode(dummy) == nil; want error")
-	}
-
-	type Data struct {
-		A string
-		I int
-	}
-	data := Data{A: "a", I: 1}
-	if err := enc.Encode(data); err != nil {
-		t.Errorf("Marshal(%v) = %v", data, err)
-	}
-
-	var data2 Data
-	if err := Unmarshal(buf.Bytes(), &data2); err != nil {
-		t.Errorf("Unmarshal(%v) = %v", data2, err)
-	}
-	if data2 != data {
-		t.Errorf("expect: %v, but get: %v", data, data2)
-	}
-}
-
 var streamEncodedIndent = `0.1
 "hello"
 null
@@ -115,7 +77,7 @@ false
 `
 
 func TestEncoderIndent(t *testing.T) {
-	var buf strings.Builder
+	var buf bytes.Buffer
 	enc := NewEncoder(&buf)
 	enc.SetIndent(">", ".")
 	for _, v := range streamTest {
@@ -185,7 +147,7 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
 			`{"bar":"\"<html>foobar</html>\""}`,
 		},
 	} {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		if err := enc.Encode(tt.v); err != nil {
 			t.Errorf("Encode(%s): %s", tt.name, err)
@@ -347,6 +309,21 @@ func TestBlocking(t *testing.T) {
 	}
 }
 
+func BenchmarkEncoderEncode(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X, Y string
+	}
+	v := &T{"foo", "bar"}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			if err := NewEncoder(io.Discard).Encode(v); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
 type tokenStreamCase struct {
 	json      string
 	expTokens []any
@@ -495,45 +472,3 @@ func TestHTTPDecoding(t *testing.T) {
 		t.Errorf("err = %v; want io.EOF", err)
 	}
 }
-
-func TestEncoderSetNilSafeCollection(t *testing.T) {
-	var (
-		nilSlice  []interface{}
-		pNilSlice *[]interface{}
-		nilMap    map[string]interface{}
-		pNilMap   *map[string]interface{}
-	)
-	for _, tt := range []struct {
-		name        string
-		v           interface{}
-		want        string
-		rescuedWant string
-	}{
-		{"nilSlice", nilSlice, "null", "[]"},
-		{"nonNilSlice", []interface{}{}, "[]", "[]"},
-		{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
-		{"pNilSlice", pNilSlice, "null", "null"},
-		{"nilMap", nilMap, "null", "{}"},
-		{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
-		{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
-		{"pNilMap", pNilMap, "null", "null"},
-	} {
-		var buf bytes.Buffer
-		enc := NewEncoder(&buf)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.want {
-			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
-		}
-		buf.Reset()
-		enc.SetNilSafeCollection(true, true)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
-			t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
-				tt.name, got, tt.want)
-		}
-	}
-}

langext/panic.go

@@ -1,7 +1,10 @@
 package langext
 
+import "runtime/debug"
+
 type PanicWrappedErr struct {
 	panic any
+	Stack string
 }
 
 func (p PanicWrappedErr) Error() string {
@@ -15,7 +18,7 @@ func (p PanicWrappedErr) ReoveredObj() any {
 func RunPanicSafe(fn func()) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -27,7 +30,7 @@ func RunPanicSafe(fn func()) (err error) {
 func RunPanicSafeR1(fn func() error) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -38,7 +41,7 @@ func RunPanicSafeR2[T1 any](fn func() (T1, error)) (r1 T1, err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
 			r1 = *new(T1)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -50,7 +53,7 @@ func RunPanicSafeR3[T1 any, T2 any](fn func() (T1, T2, error)) (r1 T1, r2 T2, er
 		if rec := recover(); rec != nil {
 			r1 = *new(T1)
 			r2 = *new(T2)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -63,7 +66,7 @@ func RunPanicSafeR4[T1 any, T2 any, T3 any](fn func() (T1, T2, T3, error)) (r1 T
 			r1 = *new(T1)
 			r2 = *new(T2)
 			r3 = *new(T3)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 

syncext/atomic.go

@@ -27,10 +27,12 @@ func (a *AtomicBool) Get() bool {
 	return a.v
 }
 
-func (a *AtomicBool) Set(value bool) {
+func (a *AtomicBool) Set(value bool) bool {
 	a.lock.Lock()
 	defer a.lock.Unlock()
 
+	oldValue := a.v
+
 	a.v = value
 
 	for k, v := range a.listener {
@@ -42,6 +44,8 @@ func (a *AtomicBool) Set(value bool) {
 			delete(a.listener, k)
 		}
 	}
+
+	return oldValue
 }
 
 func (a *AtomicBool) Wait(waitFor bool) {

timeext/calendarweek.go

@@ -0,0 +1,28 @@
+package timeext
+
+import "time"
+
+func WeekStart(year, week int) time.Time {
+
+	// https://stackoverflow.com/a/52303730/1761622
+
+	// Start from the middle of the year:
+	t := time.Date(year, 7, 1, 0, 0, 0, 0, time.UTC)
+
+	// Roll back to Monday:
+	if wd := t.Weekday(); wd == time.Sunday {
+		t = t.AddDate(0, 0, -6)
+	} else {
+		t = t.AddDate(0, 0, -int(wd)+1)
+	}
+
+	// Difference in weeks:
+	_, w := t.ISOWeek()
+	t = t.AddDate(0, 0, (week-w)*7)
+
+	return t
+}
+
+func WeekEnd(year, week int) time.Time {
+	return WeekStart(year, week).AddDate(0, 0, 7).Add(time.Duration(-1))
+}

timeext/calendarweek_test.go

@@ -0,0 +1,25 @@
+package timeext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+	"time"
+)
+
+func TestWeekStart(t *testing.T) {
+
+	tst.AssertEqual(t, WeekStart(2018, 1).Format(time.RFC3339Nano), "2018-01-01T00:00:00Z")
+	tst.AssertEqual(t, WeekStart(2018, 2).Format(time.RFC3339Nano), "2018-01-08T00:00:00Z")
+	tst.AssertEqual(t, WeekStart(2019, 1).Format(time.RFC3339Nano), "2018-12-31T00:00:00Z")
+	tst.AssertEqual(t, WeekStart(2019, 2).Format(time.RFC3339Nano), "2019-01-07T00:00:00Z")
+
+}
+
+func TestWeekEnd(t *testing.T) {
+
+	tst.AssertEqual(t, WeekEnd(2018, 1).Format(time.RFC3339Nano), "2018-01-07T23:59:59.999999999Z")
+	tst.AssertEqual(t, WeekEnd(2018, 2).Format(time.RFC3339Nano), "2018-01-14T23:59:59.999999999Z")
+	tst.AssertEqual(t, WeekEnd(2019, 1).Format(time.RFC3339Nano), "2019-01-06T23:59:59.999999999Z")
+	tst.AssertEqual(t, WeekEnd(2019, 2).Format(time.RFC3339Nano), "2019-01-13T23:59:59.999999999Z")
+
+}

tst/assertions.go

@@ -2,23 +2,59 @@ package tst
 
 import (
 	"encoding/hex"
+	"reflect"
 	"runtime/debug"
 	"testing"
 )
 
 func AssertEqual[T comparable](t *testing.T, actual T, expected T) {
+	t.Helper()
 	if actual != expected {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
 
 func AssertNotEqual[T comparable](t *testing.T, actual T, expected T) {
+	t.Helper()
 	if actual == expected {
 		t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
 
+func AssertDeepEqual[T any](t *testing.T, actual T, expected T) {
+	t.Helper()
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}
+
+func AssertSetDeepEqual[T any](t *testing.T, actual []T, expected []T) {
+	t.Helper()
+	if len(actual) != len(expected) {
+		t.Errorf("values differ in length: Actual (n=%d): '%v', Expected (n=%d): '%v'", len(actual), actual, len(expected), expected)
+	}
+
+	for _, a := range expected {
+		found := false
+		for _, b := range actual {
+			found = found || reflect.DeepEqual(a, b)
+		}
+		if !found {
+			t.Errorf("values differ: Element '%v' not found. Actual: '%v', Expected: '%v'", a, actual, expected)
+			return
+		}
+	}
+}
+
+func AssertNotDeepEqual[T any](t *testing.T, actual T, expected T) {
+	t.Helper()
+	if reflect.DeepEqual(actual, expected) {
+		t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}
+
 func AssertDeRefEqual[T comparable](t *testing.T, actual *T, expected T) {
+	t.Helper()
 	if actual == nil {
 		t.Errorf("values differ: Actual: NIL, Expected: '%v'", expected)
 	}
@@ -28,6 +64,7 @@ func AssertDeRefEqual[T comparable](t *testing.T, actual *T, expected T) {
 }
 
 func AssertPtrEqual[T comparable](t *testing.T, actual *T, expected *T) {
+	t.Helper()
 	if actual == nil && expected == nil {
 		return
 	}
@@ -47,6 +84,7 @@ func AssertPtrEqual[T comparable](t *testing.T, actual *T, expected *T) {
 }
 
 func AssertHexEqual(t *testing.T, expected string, actual []byte) {
+	t.Helper()
 	actualStr := hex.EncodeToString(actual)
 	if actualStr != expected {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actualStr, expected)
@@ -54,18 +92,21 @@ func AssertHexEqual(t *testing.T, expected string, actual []byte) {
 }
 
 func AssertTrue(t *testing.T, value bool) {
+	t.Helper()
 	if !value {
 		t.Error("value should be true\n" + string(debug.Stack()))
 	}
 }
 
 func AssertFalse(t *testing.T, value bool) {
+	t.Helper()
 	if value {
 		t.Error("value should be false\n" + string(debug.Stack()))
 	}
 }
 
 func AssertNoErr(t *testing.T, anerr error) {
+	t.Helper()
 	if anerr != nil {
 		t.Error("Function returned an error: " + anerr.Error() + "\n" + string(debug.Stack()))
 	}

wmo/queryList.go

@@ -21,19 +21,7 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 
 	if filter != nil {
 		pipeline = filter.FilterQuery()
-		fpf1, fpd1, fpf2, fpd2 := filter.Pagination()
-		if fpf1 != "" {
-			pf1 = fpf1
-		}
-		if fpd1 != "" {
-			pd1 = fpd1
-		}
-		if fpf2 != "" {
-			pf2 = fpf2
-		}
-		if fpd2 != "" {
-			pd2 = fpd2
-		}
+		pf1, pd1, pf2, pd2 = filter.Pagination()
 	}
 
 	sortPrimary := pf1
@@ -128,3 +116,17 @@ func (c *Coll[TData]) Count(ctx context.Context, filter ct.Filter) (int64, error
 
 	return 0, nil
 }
+
+func (c *Coll[TData]) ListWithCount(ctx context.Context, filter ct.Filter, pageSize *int, inTok ct.CursorToken) ([]TData, ct.CursorToken, int64, error) {
+	// NOTE: Possible optimization: Cache count in CursorToken, then fetch count only on first page.
+	count, err := c.Count(ctx, filter)
+	if err != nil {
+		return nil, ct.CursorToken{}, 0, err
+	}
+
+	data, token, err := c.List(ctx, filter, pageSize, inTok)
+	if err != nil {
+		return nil, ct.CursorToken{}, 0, err
+	}
+	return data, token, count, nil
+}

wmo/queryUpdate.go

@@ -73,7 +73,7 @@ func (c *Coll[TData]) ReplaceOne(ctx context.Context, filterQuery bson.M, value
 }
 
 func (c *Coll[TData]) FindOneAndReplace(ctx context.Context, filterQuery bson.M, value TData) (TData, error) {
-	mongoRes := c.coll.FindOneAndUpdate(ctx, filterQuery, bson.M{"$set": value}, options.FindOneAndUpdate().SetReturnDocument(options.After))
+	mongoRes := c.coll.FindOneAndReplace(ctx, filterQuery, value, options.FindOneAndReplace().SetReturnDocument(options.After))
 	if err := mongoRes.Err(); err != nil {
 		return *new(TData), exerr.Wrap(err, "mongo-query[find-one-and-update] failed").
 			Str("collection", c.Name()).

wmo/reflection.go

@@ -25,7 +25,7 @@ func (c *Coll[TData]) EnsureInitializedReflection(v TData) {
 
 	m := make(map[string]fullTypeRef)
 
-	c.initFields("", rval, m, make([]int, 0))
+	c.initFields("", rval.Type(), m, make([]int, 0), make([]reflect.Type, 0))
 
 	c.implDataTypeMap[rval.Type()] = m
 }
@@ -50,20 +50,16 @@ func (c *Coll[TData]) init() {
 		c.implDataTypeMap = make(map[reflect.Type]map[string]fullTypeRef)
 
 		v := reflect.ValueOf(example)
-		c.initFields("", v, c.dataTypeMap, make([]int, 0))
+		c.initFields("", v.Type(), c.dataTypeMap, make([]int, 0), make([]reflect.Type, 0))
 
 	}
 
 }
 
-func (c *Coll[TData]) initFields(prefix string, rval reflect.Value, m map[string]fullTypeRef, idxarr []int) {
-
-	rtyp := rval.Type()
-
+func (c *Coll[TData]) initFields(prefix string, rtyp reflect.Type, m map[string]fullTypeRef, idxarr []int, typesInPath []reflect.Type) {
 	for i := 0; i < rtyp.NumField(); i++ {
 
 		rsfield := rtyp.Field(i)
-		rvfield := rval.Field(i)
 
 		if !rsfield.IsExported() {
 			continue
@@ -91,21 +87,21 @@ func (c *Coll[TData]) initFields(prefix string, rval reflect.Value, m map[string
 		newIdxArr := langext.ArrCopy(idxarr)
 		newIdxArr = append(newIdxArr, i)
 
-		if langext.InArray("inline", bsontags) && rvfield.Kind() == reflect.Struct {
+		if langext.InArray("inline", bsontags) && rsfield.Type.Kind() == reflect.Struct {
 
 			// pass-through field
-			c.initFields(prefix, rvfield, m, newIdxArr)
+			c.initFields(prefix, rsfield.Type, m, newIdxArr, typesInPath)
 
 		} else {
 
-			if rvfield.Type().Kind() == reflect.Pointer {
+			if rsfield.Type.Kind() == reflect.Pointer {
 
 				m[fullKey] = fullTypeRef{
 					IsPointer:      true,
-					RealType:       rvfield.Type(),
-					Kind:           rvfield.Type().Elem().Kind(),
-					Type:           rvfield.Type().Elem(),
-					UnderlyingType: reflectext.Underlying(rvfield.Type().Elem()),
+					RealType:       rsfield.Type,
+					Kind:           rsfield.Type.Elem().Kind(),
+					Type:           rsfield.Type.Elem(),
+					UnderlyingType: reflectext.Underlying(rsfield.Type.Elem()),
 					Name:           rsfield.Name,
 					Index:          newIdxArr,
 				}
@@ -114,20 +110,37 @@ func (c *Coll[TData]) initFields(prefix string, rval reflect.Value, m map[string
 
 				m[fullKey] = fullTypeRef{
 					IsPointer:      false,
-					RealType:       rvfield.Type(),
-					Kind:           rvfield.Type().Kind(),
-					Type:           rvfield.Type(),
-					UnderlyingType: reflectext.Underlying(rvfield.Type()),
+					RealType:       rsfield.Type,
+					Kind:           rsfield.Type.Kind(),
+					Type:           rsfield.Type,
+					UnderlyingType: reflectext.Underlying(rsfield.Type),
 					Name:           rsfield.Name,
 					Index:          newIdxArr,
 				}
 
 			}
 
-			if rvfield.Kind() == reflect.Struct {
-				c.initFields(fullKey+".", rvfield, m, newIdxArr)
+			if rsfield.Type.Kind() == reflect.Struct {
+				c.initFields(fullKey+".", rsfield.Type, m, newIdxArr, typesInPath)
 			}
 
+			if rsfield.Type.Kind() == reflect.Pointer && rsfield.Type.Elem().Kind() == reflect.Struct {
+				innerType := rsfield.Type.Elem()
+
+				// check if there is recursion
+				recursion := false
+				for _, typ := range typesInPath {
+					recursion = recursion || (typ == innerType)
+				}
+
+				if !recursion {
+					// Store all seen types before that deref a pointer to prevent endless recursion
+					newTypesInPath := make([]reflect.Type, len(typesInPath))
+					copy(newTypesInPath, typesInPath)
+					newTypesInPath = append(newTypesInPath, rtyp)
+					c.initFields(fullKey+".", innerType, m, newIdxArr, newTypesInPath)
+				}
+			}
 		}
 
 	}

wmo/reflection_test.go

@@ -23,6 +23,9 @@ func TestReflectionGetFieldType(t *testing.T) {
 		Sub   struct {
 			A string `bson:"a"`
 		} `bson:"sub"`
+		SubPtr *struct {
+			A string `bson:"a"`
+		} `bson:"subPtr"`
 		Str   string                  `bson:"str"`
 		Ptr   *int                    `bson:"ptr"`
 		MDate rfctime.RFC3339NanoTime `bson:"mdate"`
@@ -43,6 +46,11 @@ func TestReflectionGetFieldType(t *testing.T) {
 		}{
 			A: "2",
 		},
+		SubPtr: &struct {
+			A string `bson:"a"`
+		}{
+			A: "4",
+		},
 		Str:   "3",
 		Ptr:   langext.Ptr(4),
 		MDate: t1,
@@ -82,6 +90,12 @@ func TestReflectionGetFieldType(t *testing.T) {
 	tst.AssertEqual(t, gft("sub.a").IsPointer, false)
 	tst.AssertEqual(t, gfv("sub.a").(string), "2")
 
+	tst.AssertEqual(t, gft("subPtr.a").Kind.String(), "string")
+	tst.AssertEqual(t, gft("subPtr.a").Type.String(), "string")
+	tst.AssertEqual(t, gft("subPtr.a").Name, "A")
+	tst.AssertEqual(t, gft("subPtr.a").IsPointer, false)
+	tst.AssertEqual(t, gfv("subPtr.a").(string), "4")
+
 	tst.AssertEqual(t, gft("str").Kind.String(), "string")
 	tst.AssertEqual(t, gft("str").Type.String(), "string")
 	tst.AssertEqual(t, gft("str").Name, "Str")
@@ -99,16 +113,25 @@ func TestReflectionGetTokenValueAsMongoType(t *testing.T) {
 
 	type IDType string
 
+	type RecurseiveType struct {
+		Other int             `bson:"other"`
+		Inner *RecurseiveType `bson:"inner"`
+	}
+
 	type TestData struct {
 		ID    IDType    `bson:"_id"`
 		CDate time.Time `bson:"cdate"`
 		Sub   struct {
 			A string `bson:"a"`
 		} `bson:"sub"`
+		SubPtr *struct {
+			A string `bson:"a"`
+		} `bson:"subPtr"`
 		Str   string                  `bson:"str"`
 		Ptr   *int                    `bson:"ptr"`
 		Num   int                     `bson:"num"`
 		MDate rfctime.RFC3339NanoTime `bson:"mdate"`
+		Rec   RecurseiveType          `bson:"rec"`
 	}
 
 	coll := W[TestData](&mongo.Collection{})
@@ -130,6 +153,9 @@ func TestReflectionGetTokenValueAsMongoType(t *testing.T) {
 	}
 
 	tst.AssertEqual(t, gtvasmt("hello", "str").(string), "hello")
+	tst.AssertEqual(t, gtvasmt("hello", "sub.a").(string), "hello")
+	tst.AssertEqual(t, gtvasmt("hello", "subPtr.a").(string), "hello")
+	tst.AssertEqual(t, gtvasmt("4", "rec.other").(int), 4)
 	tst.AssertEqual(t, gtvasmt("4", "num").(int), 4)
 	tst.AssertEqual(t, gtvasmt("asdf", "_id").(IDType), "asdf")
 	tst.AssertEqual(t, gtvasmt("", "ptr").(*int), nil)