v0.0.319 fix error when findoneÃ+pipeline fails

.gitea/workflows/Dockerfile_tests

@@ -1,9 +0,0 @@
-FROM golang:latest
-
-RUN apt install -y make curl python3 && go install gotest.tools/gotestsum@latest
-
-COPY . /source
-
-WORKDIR /source
-
-CMD ["make", "test"]

.gitea/workflows/tests.yml

@@ -6,7 +6,12 @@
 name: Build Docker and Deploy
 run-name: Build & Deploy ${{ gitea.ref }} on ${{ gitea.actor }}
 
-on: [push]
+on:
+  push:
+    branches:
+      - '*'
+      - '**'
+
 
 jobs:
   run_tests:
@@ -17,14 +22,34 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Build test docker
-        id: build_docker
-        run: echo "DOCKER_IMG_ID=$(docker build -q . -f .gitea/workflows/Dockerfile_tests || echo __err_build__)" >> $GITHUB_OUTPUT
+      - name: Setup go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: '${{ gitea.workspace }}/go.mod'
+
+      - name: Setup packages
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: curl python3
+          version: 1.0
+
+      - name: go version
+        run: go version
 
       - name: Run tests
-        run: docker run --rm "${{ steps.build_docker.outputs.DOCKER_IMG_ID }}"
+        run: cd "${{ gitea.workspace }}" && make test
 
-      - name: Cleanup
-        if: always()
-        run: docker image rm "${{ steps.build_docker.outputs.DOCKER_IMG_ID }}"
+      - name: Send failure mail
+        if: failure()
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.fastmail.com
+          server_port: 465
+          secure: true
+          username: ${{secrets.MAIL_USERNAME}}
+          password: ${{secrets.MAIL_PASSWORD}}
+          subject: Pipeline on '${{ gitea.repository }}' failed
+          to: ${{ steps.commiter_info.outputs.MAIL }}
+          from: Gitea Actions <gitea_actions@blackforestbytes.de>
+          body: "Go to https://gogs.blackforestbytes.com/${{ gitea.repository }}/actions"
 

.idea/golinter.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoLinterSettings">
+    <option name="checkGoLinterExe" value="false" />
+  </component>
+</project>

Makefile

@@ -7,5 +7,11 @@ test:
 	which gotestsum || go install gotest.tools/gotestsum@latest
 	gotestsum --format "testname" -- -tags="timetzdata sqlite_fts5 sqlite_foreign_keys" "./..."
 
+test-in-docker:
+	tag="goext_temp_test_image:$(shell uuidgen | tr -d '-')";        \
+	docker build --tag $$tag . -f .gitea/workflows/Dockerfile_tests; \
+	docker run --rm $$tag;                                           \
+	docker rmi $$tag
+
 version:
 	_data/version.sh

README.md

@@ -11,17 +11,19 @@ Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
 ### Packages:
 
 | Name        | Maintainer | Description                                                                                                   |
-|--------------|------------|---------------------------------------------------------------------------------------------------------------|
+|-------------|------------|---------------------------------------------------------------------------------------------------------------|
 | langext     | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
 | mathext     | Mike       | Utility/Helper functions for math                                                                             |
 | cryptext    | Mike       | Utility/Helper functions for encryption                                                                       |
 | syncext     | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
 | dataext     | Mike       | Various useful data structures                                                                                |
 | zipext      | Mike       | Utility for zip/gzip/tar etc                                                                                  |
-| reflectext   | Mike       | Utility for golagn reflection                                                                                 |
+| reflectext  | Mike       | Utility for golang reflection                                                                                 |
+| fsext       | Mike       | Utility for filesytem access                                                                                  |
 |             |            |                                                                                                               |
 | mongoext    | Mike       | Utility/Helper functions for mongodb                                                                          |
 | cursortoken | Mike       | MongoDB cursortoken implementation                                                                            |
+| pagination  | Mike       | Pagination implementation                                                                                     |
 |             |            |                                                                                                               |
 | totpext     | Mike       | Implementation of TOTP (2-Factor-Auth)                                                                        |
 | termext     | Mike       | Utilities for terminals (mostly color output)                                                                 |

bfcodegen/csid-generate.go

@@ -0,0 +1,182 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"go/format"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type CSIDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+	Prefix       string
+}
+
+var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
+
+var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed csid-generate.template
+var templateCSIDGenerateText string
+
+func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexCSIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[IDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[IDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\n")
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	fdata, err := format.Source([]byte(fmtCSIDOutput(newChecksum, allIDs, pkgname)))
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(destFile, fdata, 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexCSIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexCSIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := CSIDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+				Prefix:       match.GroupByName("prefix").Value(),
+			}
+			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtCSIDOutput(cs string, ids []CSIDDef, pkgname string) string {
+	templ := template.Must(template.New("csid-generate").Parse(templateCSIDGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/csid-generate.template

@@ -0,0 +1,190 @@
+// Code generated by csid-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "crypto/rand"
+import "crypto/sha256"
+import "fmt"
+import "github.com/go-playground/validator/v10"
+import "github.com/rs/zerolog/log"
+import "gogs.mikescher.com/BlackForestBytes/goext/exerr"
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+import "gogs.mikescher.com/BlackForestBytes/goext/rext"
+import "math/big"
+import "reflect"
+import "regexp"
+import "strings"
+
+const ChecksumCharsetIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+const idlen = 24
+
+const checklen = 1
+
+const idCharset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+const idCharsetLen = len(idCharset)
+
+var charSetReverseMap = generateCharsetMap()
+
+const ({{range .IDs}}
+	prefix{{.Name}} = "{{.Prefix}}" {{end}}
+)
+
+var ({{range .IDs}}
+	regex{{.Name}} = generateRegex(prefix{{.Name}}) {{end}}
+)
+
+func generateRegex(prefix string) rext.Regex {
+	return rext.W(regexp.MustCompile(fmt.Sprintf("^%s[%s]{%d}[%s]{%d}$", prefix, idCharset, idlen-len(prefix)-checklen, idCharset, checklen)))
+}
+
+func generateCharsetMap() []int {
+	result := make([]int, 128)
+	for i := 0; i < len(result); i++ {
+		result[i] = -1
+	}
+	for idx, chr := range idCharset {
+		result[int(chr)] = idx
+	}
+	return result
+}
+
+func generateID(prefix string) string {
+	k := ""
+	csMax := big.NewInt(int64(idCharsetLen))
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		v, err := rand.Int(rand.Reader, csMax)
+		if err != nil {
+			panic(err)
+		}
+		v64 := v.Int64()
+		k += string(idCharset[v64])
+		checksum = (checksum + int(v64)) % (idCharsetLen)
+	}
+	checkstr := string(idCharset[checksum%idCharsetLen])
+	return prefix + k + checkstr
+}
+
+func generateIDFromSeed(prefix string, seed string) string {
+	h := sha256.New()
+
+	iddata := ""
+	for len(iddata) < idlen-len(prefix)-checklen {
+		h.Write([]byte(seed))
+		bs := h.Sum(nil)
+		iddata += langext.NewAnyBaseConverter(idCharset).Encode(bs)
+	}
+
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		ichr := int(iddata[i])
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	return prefix + iddata[:(idlen-len(prefix)-checklen)] + checkstr
+}
+
+func validateID(prefix string, value string) error {
+	if len(value) != idlen {
+		return exerr.New(exerr.TypeInvalidCSID, "id has the wrong length").Str("value", value).Build()
+	}
+
+	if !strings.HasPrefix(value, prefix) {
+		return exerr.New(exerr.TypeInvalidCSID, "id is missing the correct prefix").Str("value", value).Str("prefix", prefix).Build()
+	}
+
+	checksum := 0
+	for i := len(prefix); i < len(value)-checklen; i++ {
+		ichr := int(value[i])
+		if ichr < 0 || ichr >= len(charSetReverseMap) || charSetReverseMap[ichr] == -1 {
+			return exerr.New(exerr.TypeInvalidCSID, "id contains invalid characters").Str("value", value).Build()
+		}
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	if !strings.HasSuffix(value, checkstr) {
+		return exerr.New(exerr.TypeInvalidCSID, "id checkstring is invalid").Str("value", value).Str("checkstr", checkstr).Build()
+	}
+
+	return nil
+}
+
+func getRawData(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[len(prefix) : idlen-checklen]
+}
+
+func getCheckString(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[idlen-checklen:]
+}
+
+func ValidateEntityID(vfl validator.FieldLevel) bool {
+	if !vfl.Field().CanInterface() {
+		log.Error().Msgf("Failed to validate EntityID (cannot interface ?!?)")
+		return false
+	}
+
+	ifvalue := vfl.Field().Interface()
+
+	if value1, ok := ifvalue.(EntityID); ok {
+
+		if vfl.Field().Type().Kind() == reflect.Pointer && langext.IsNil(value1) {
+			return true
+		}
+
+		if err := value1.Valid(); err != nil {
+			log.Debug().Msgf("Failed to validate EntityID '%s' (%s)", value1.String(), err.Error())
+			return false
+		} else {
+			return true
+		}
+
+	} else {
+		log.Error().Msgf("Failed to validate EntityID (wrong type: %T)", ifvalue)
+		return false
+	}
+}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(generateID(prefix{{.Name}}))
+}
+
+func (id {{.Name}}) Valid() error {
+	return validateID(prefix{{.Name}}, string(id))
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) Prefix() string {
+	return prefix{{.Name}}
+}
+
+func (id {{.Name}}) Raw() string {
+	return getRawData(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) CheckString() string {
+	return getCheckString(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) Regex() rext.Regex {
+	return regex{{.Name}}
+}
+
+{{end}}

bfcodegen/csid-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example.tgz
+var CSIDExampleModels []byte
+
+func TestGenerateCSIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, CSIDExampleModels, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/csid_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/enum-generate.go

@@ -1,10 +1,12 @@
 package bfcodegen
 
 import (
+	"bytes"
+	_ "embed"
 	"errors"
 	"fmt"
+	"go/format"
 	"gogs.mikescher.com/BlackForestBytes/goext"
-	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
 	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rext"
@@ -14,7 +16,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
+	"text/template"
 )
 
 type EnumDefVal struct {
@@ -35,10 +37,13 @@ var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]
 
 var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
 
-var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:]+"|[0-9]+))\s*(//(?P<descr>.*))?.*$`))
+var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:\s]+"|[0-9]+))\s*(//(?P<descr>.*))?.*$`))
 
 var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
 
+//go:embed enum-generate.template
+var templateEnumGenerateText string
+
 func GenerateEnumSpecs(sourceDir string, destFile string) error {
 
 	files, err := os.ReadDir(sourceDir)
@@ -104,25 +109,16 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		return errors.New("no package name found in any file")
 	}
 
-	err = os.WriteFile(destFile, []byte(fmtEnumOutput(newChecksum, allEnums, pkgname)), 0o755)
+	fdata, err := format.Source([]byte(fmtEnumOutput(newChecksum, allEnums, pkgname)))
 	if err != nil {
 		return err
 	}
 
-	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
+	err = os.WriteFile(destFile, fdata, 0o755)
 	if err != nil {
 		return err
 	}
 
-	if res.CommandTimedOut {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt timed out")
-	}
-	if res.ExitCode != 0 {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt did not succeed")
-	}
-
 	return nil
 }
 
@@ -204,133 +200,32 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 }
 
 func fmtEnumOutput(cs string, enums []EnumDef, pkgname string) string {
-	str := "// Code generated by enum-generate.go DO NOT EDIT.\n"
-	str += "\n"
-	str += "package " + pkgname + "\n"
-	str += "\n"
 
-	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/langext\"" + "\n"
-	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/enums\"" + "\n"
-	str += "\n"
+	templ := template.New("enum-generate")
 
-	str += "const ChecksumEnumGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
-	str += "\n"
+	templ = templ.Funcs(template.FuncMap{
+		"boolToStr": func(b bool) string { return langext.Conditional(b, "true", "false") },
+		"deref":     func(v *string) string { return *v },
+		"trimSpace": func(str string) string { return strings.TrimSpace(str) },
+		"hasStr":    func(v EnumDef) bool { return v.Type == "string" },
+		"hasDescr": func(v EnumDef) bool {
+			return langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Description != nil })
+		},
+	})
 
-	for _, enumdef := range enums {
+	templ = template.Must(templ.Parse(templateEnumGenerateText))
 
-		hasDescr := langext.ArrAll(enumdef.Values, func(val EnumDefVal) bool { return val.Description != nil })
-		hasStr := enumdef.Type == "string"
+	buffer := bytes.Buffer{}
 
-		str += "// ================================ " + enumdef.EnumTypeName + " ================================" + "\n"
-		str += "//" + "\n"
-		str += "// File:       " + enumdef.FileRelative + "\n"
-		str += "// StringEnum: " + langext.Conditional(hasStr, "true", "false") + "\n"
-		str += "// DescrEnum:  " + langext.Conditional(hasDescr, "true", "false") + "\n"
-		str += "//" + "\n"
-		str += "" + "\n"
-
-		str += "var __" + enumdef.EnumTypeName + "Values = []" + enumdef.EnumTypeName + "{" + "\n"
-		for _, v := range enumdef.Values {
-			str += "    " + v.VarName + "," + "\n"
-		}
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		if hasDescr {
-			str += "var __" + enumdef.EnumTypeName + "Descriptions = map[" + enumdef.EnumTypeName + "]string{" + "\n"
-			for _, v := range enumdef.Values {
-				str += "    " + v.VarName + ": \"" + strings.TrimSpace(*v.Description) + "\"," + "\n"
-			}
-			str += "}" + "\n"
-			str += "" + "\n"
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"Enums":        enums,
+	})
+	if err != nil {
+		panic(err)
 	}
 
-		str += "var __" + enumdef.EnumTypeName + "Varnames = map[" + enumdef.EnumTypeName + "]string{" + "\n"
-		for _, v := range enumdef.Values {
-			str += "    " + v.VarName + ": \"" + v.VarName + "\"," + "\n"
-		}
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") Valid() bool {" + "\n"
-		str += "    return langext.InArray(e, __" + enumdef.EnumTypeName + "Values)" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") Values() []" + enumdef.EnumTypeName + " {" + "\n"
-		str += "    return __" + enumdef.EnumTypeName + "Values" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") ValuesAny() []any {" + "\n"
-		str += "    return langext.ArrCastToAny(__" + enumdef.EnumTypeName + "Values)" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") ValuesMeta() []enums.EnumMetaValue {" + "\n"
-		str += "    return " + enumdef.EnumTypeName + "ValuesMeta()"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		if hasStr {
-			str += "func (e " + enumdef.EnumTypeName + ") String() string {" + "\n"
-			str += "    return string(e)" + "\n"
-			str += "}" + "\n"
-			str += "" + "\n"
-		}
-
-		if hasDescr {
-			str += "func (e " + enumdef.EnumTypeName + ") Description() string {" + "\n"
-			str += "    if d, ok := __" + enumdef.EnumTypeName + "Descriptions[e]; ok {" + "\n"
-			str += "        return d" + "\n"
-			str += "    }" + "\n"
-			str += "    return \"\"" + "\n"
-			str += "}" + "\n"
-			str += "" + "\n"
-		}
-
-		str += "func (e " + enumdef.EnumTypeName + ") VarName() string {" + "\n"
-		str += "    if d, ok := __" + enumdef.EnumTypeName + "Varnames[e]; ok {" + "\n"
-		str += "        return d" + "\n"
-		str += "    }" + "\n"
-		str += "    return \"\"" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") Meta() enums.EnumMetaValue {" + "\n"
-		if hasDescr {
-			str += "    return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())}"
-		} else {
-			str += "    return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil}"
-		}
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func Parse" + enumdef.EnumTypeName + "(vv string) (" + enumdef.EnumTypeName + ", bool) {" + "\n"
-		str += "    for _, ev := range __" + enumdef.EnumTypeName + "Values {" + "\n"
-		str += "        if string(ev) == vv {" + "\n"
-		str += "            return ev, true" + "\n"
-		str += "        }" + "\n"
-		str += "    }" + "\n"
-		str += "    return \"\", false" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func " + enumdef.EnumTypeName + "Values() []" + enumdef.EnumTypeName + " {" + "\n"
-		str += "    return __" + enumdef.EnumTypeName + "Values" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func " + enumdef.EnumTypeName + "ValuesMeta() []enums.EnumMetaValue {" + "\n"
-		str += "    return []enums.EnumMetaValue{" + "\n"
-		for _, v := range enumdef.Values {
-			str += "        " + v.VarName + ".Meta(),\n"
-		}
-		str += "    }" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-	}
-
-	return str
+	return buffer.String()
 }

bfcodegen/enum-generate.template

@@ -0,0 +1,97 @@
+// Code generated by enum-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+import "gogs.mikescher.com/BlackForestBytes/goext/enums"
+
+const ChecksumEnumGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{range .Enums}}
+
+{{ $hasStr   := ( . | hasStr   ) }}
+{{ $hasDescr := ( . | hasDescr ) }}
+
+// ================================ {{.EnumTypeName}} ================================
+//
+// File:       {{.FileRelative}}
+// StringEnum: {{$hasStr   | boolToStr}}
+// DescrEnum:  {{$hasDescr | boolToStr}}
+//
+
+var __{{.EnumTypeName}}Values = []{{.EnumTypeName}}{ {{range .Values}}
+	{{.VarName}}, {{end}}
+}
+
+{{if $hasDescr}}
+var __{{.EnumTypeName}}Descriptions = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: "{{.Description | deref | trimSpace}}", {{end}}
+}
+{{end}}
+
+var __{{.EnumTypeName}}Varnames = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: "{{.VarName}}", {{end}}
+}
+
+func (e {{.EnumTypeName}}) Valid() bool {
+	return langext.InArray(e, __{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func (e {{.EnumTypeName}}) ValuesAny() []any {
+	return langext.ArrCastToAny(__{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) ValuesMeta() []enums.EnumMetaValue {
+	return {{.EnumTypeName}}ValuesMeta()
+}
+
+{{if $hasStr}}
+func (e {{.EnumTypeName}}) String() string {
+	return string(e)
+}
+{{end}}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) Description() string {
+	if d, ok := __{{.EnumTypeName}}Descriptions[e]; ok {
+		return d
+	}
+	return ""
+}
+{{end}}
+
+func (e {{.EnumTypeName}}) VarName() string {
+	if d, ok := __{{.EnumTypeName}}Varnames[e]; ok {
+		return d
+	}
+	return ""
+}
+
+func (e {{.EnumTypeName}}) Meta() enums.EnumMetaValue {
+    {{if $hasDescr}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())} {{else}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil} {{end}}
+}
+
+func Parse{{.EnumTypeName}}(vv string) ({{.EnumTypeName}}, bool) {
+	for _, ev := range __{{.EnumTypeName}}Values {
+		if string(ev) == vv {
+			return ev, true
+		}
+	}
+	return "", false
+}
+
+func {{.EnumTypeName}}Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func {{.EnumTypeName}}ValuesMeta() []enums.EnumMetaValue {
+	return []enums.EnumMetaValue{ {{range .Values}}
+            {{.VarName}}.Meta(), {{end}}
+	}
+}
+
+{{end}}

bfcodegen/enum-generate_test.go

@@ -2,6 +2,7 @@ package bfcodegen
 
 import (
 	_ "embed"
+	"fmt"
 	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/tst"
@@ -12,7 +13,7 @@ import (
 )
 
 //go:embed _test_example.tgz
-var ExampleModels []byte
+var EnumExampleModels []byte
 
 func TestGenerateEnumSpecs(t *testing.T) {
 
@@ -20,7 +21,7 @@ func TestGenerateEnumSpecs(t *testing.T) {
 
 	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
 
-	err := os.WriteFile(tmpFile, ExampleModels, 0o777)
+	err := os.WriteFile(tmpFile, EnumExampleModels, 0o777)
 	tst.AssertNoErr(t, err)
 
 	t.Cleanup(func() { _ = os.Remove(tmpFile) })
@@ -39,4 +40,13 @@ func TestGenerateEnumSpecs(t *testing.T) {
 	err = GenerateEnumSpecs(tmpDir, tmpDir+"/enums_gen.go")
 	tst.AssertNoErr(t, err)
 
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/enums_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
 }

bfcodegen/id-generate.go

@@ -1,10 +1,12 @@
 package bfcodegen
 
 import (
+	"bytes"
+	_ "embed"
 	"errors"
 	"fmt"
+	"go/format"
 	"gogs.mikescher.com/BlackForestBytes/goext"
-	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
 	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rext"
@@ -14,7 +16,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
+	"text/template"
 )
 
 type IDDef struct {
@@ -29,6 +31,9 @@ var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+s
 
 var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
 
+//go:embed id-generate.template
+var templateIDGenerateText string
+
 func GenerateIDSpecs(sourceDir string, destFile string) error {
 
 	files, err := os.ReadDir(sourceDir)
@@ -94,25 +99,16 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 		return errors.New("no package name found in any file")
 	}
 
-	err = os.WriteFile(destFile, []byte(fmtIDOutput(newChecksum, allIDs, pkgname)), 0o755)
+	fdata, err := format.Source([]byte(fmtIDOutput(newChecksum, allIDs, pkgname)))
 	if err != nil {
 		return err
 	}
 
-	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
+	err = os.WriteFile(destFile, fdata, 0o755)
 	if err != nil {
 		return err
 	}
 
-	if res.CommandTimedOut {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt timed out")
-	}
-	if res.ExitCode != 0 {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt did not succeed")
-	}
-
 	return nil
 }
 
@@ -166,71 +162,22 @@ func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
 }
 
 func fmtIDOutput(cs string, ids []IDDef, pkgname string) string {
-	str := "// Code generated by id-generate.go DO NOT EDIT.\n"
-	str += "\n"
-	str += "package " + pkgname + "\n"
-	str += "\n"
+	templ := template.Must(template.New("id-generate").Parse(templateIDGenerateText))
 
-	str += "import \"go.mongodb.org/mongo-driver/bson\"" + "\n"
-	str += "import \"go.mongodb.org/mongo-driver/bson/bsontype\"" + "\n"
-	str += "import \"go.mongodb.org/mongo-driver/bson/primitive\"" + "\n"
-	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/exerr\"" + "\n"
-	str += "\n"
-
-	str += "const ChecksumIDGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
-	str += "\n"
+	buffer := bytes.Buffer{}
 
 	anyDef := langext.ArrFirstOrNil(ids, func(def IDDef) bool { return def.Name == "AnyID" || def.Name == "AnyId" })
 
-	for _, iddef := range ids {
-
-		str += "// ================================ " + iddef.Name + " (" + iddef.FileRelative + ") ================================" + "\n"
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") MarshalBSONValue() (bsontype.Type, []byte, error) {" + "\n"
-		str += "	if objId, err := primitive.ObjectIDFromHex(string(i)); err == nil {" + "\n"
-		str += "		return bson.MarshalValue(objId)" + "\n"
-		str += "	} else {" + "\n"
-		str += "		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, \"Failed to marshal " + iddef.Name + "(\"+i.String()+\") to ObjectId\").Str(\"value\", string(i)).Type(\"type\", i).Build()" + "\n"
-		str += "	}" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") String() string {" + "\n"
-		str += "	return string(i)" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") ObjID() (primitive.ObjectID, error) {" + "\n"
-		str += "	return primitive.ObjectIDFromHex(string(i))" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") Valid() bool {" + "\n"
-		str += "	_, err := primitive.ObjectIDFromHex(string(i))" + "\n"
-		str += "	return err == nil" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		if anyDef != nil {
-			str += "func (i " + iddef.Name + ") AsAny() " + anyDef.Name + " {" + "\n"
-			str += "	return " + anyDef.Name + "(i)" + "\n"
-			str += "}" + "\n"
-
-			str += "" + "\n"
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+		"AnyDef":       anyDef,
+	})
+	if err != nil {
+		panic(err)
 	}
 
-		str += "func New" + iddef.Name + "() " + iddef.Name + " {" + "\n"
-		str += "	return " + iddef.Name + "(primitive.NewObjectID().Hex())" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-	}
-
-	return str
+	return buffer.String()
 }

bfcodegen/id-generate.template

@@ -0,0 +1,47 @@
+// Code generated by id-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "go.mongodb.org/mongo-driver/bson"
+import "go.mongodb.org/mongo-driver/bson/bsontype"
+import "go.mongodb.org/mongo-driver/bson/primitive"
+import "gogs.mikescher.com/BlackForestBytes/goext/exerr"
+
+const ChecksumIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func (i {{.Name}}) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	if objId, err := primitive.ObjectIDFromHex(string(i)); err == nil {
+		return bson.MarshalValue(objId)
+	} else {
+		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, "Failed to marshal {{.Name}}("+i.String()+") to ObjectId").Str("value", string(i)).Type("type", i).Build()
+	}
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) ObjID() (primitive.ObjectID, error) {
+	return primitive.ObjectIDFromHex(string(i))
+}
+
+func (i {{.Name}}) Valid() bool {
+	_, err := primitive.ObjectIDFromHex(string(i))
+	return err == nil
+}
+
+{{if ne $.AnyDef nil}}
+func (i {{.Name}}) AsAny() {{$.AnyDef.Name}} {
+	return {{$.AnyDef.Name}}(i)
+}
+{{end}}
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(primitive.NewObjectID().Hex())
+}
+
+{{end}}

bfcodegen/id-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example.tgz
+var IDExampleModels []byte
+
+func TestGenerateIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, IDExampleModels, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/id_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

cmdext/cmdrunner.go

@@ -133,9 +133,6 @@ func run(opt CommandRunner) (CommandResult, error) {
 
 	case <-stderrFailChan:
 		_ = cmd.Process.Kill()
-		for _, lstr := range opt.listener {
-			lstr.Timeout()
-		}
 
 		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, 32*time.Millisecond); ok {
 			// most of the time the cmd.Process.Kill() should also have finished the pipereader
@@ -160,7 +157,8 @@ func run(opt CommandRunner) (CommandResult, error) {
 		}
 
 	case outobj := <-outputChan:
-		if exiterr, ok := outobj.err.(*exec.ExitError); ok {
+		var exiterr *exec.ExitError
+		if errors.As(outobj.err, &exiterr) {
 			excode := exiterr.ExitCode()
 			for _, lstr := range opt.listener {
 				lstr.Finished(excode)

cmdext/pipereader.go

@@ -32,8 +32,8 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	stdout := ""
 	go func() {
 		buf := make([]byte, 128)
-		for true {
-			n, out := pr.stdout.Read(buf)
+		for {
+			n, err := pr.stdout.Read(buf)
 			if n > 0 {
 				txt := string(buf[:n])
 				stdout += txt
@@ -42,11 +42,11 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 					lstr.ReadRawStdout(buf[:n])
 				}
 			}
-			if out == io.EOF {
+			if err == io.EOF {
 				break
 			}
-			if out != nil {
-				errch <- out
+			if err != nil {
+				errch <- err
 				break
 			}
 		}
@@ -61,7 +61,7 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	stderr := ""
 	go func() {
 		buf := make([]byte, 128)
-		for true {
+		for {
 			n, err := pr.stderr.Read(buf)
 
 			if n > 0 {

confext/confParser.go

@@ -41,13 +41,13 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			continue
 		}
 
-		if rvfield.Kind() == reflect.Struct {
-
 		envkey, found := rsfield.Tag.Lookup("env")
 		if !found || envkey == "-" {
 			continue
 		}
 
+		if rvfield.Kind() == reflect.Struct && rvfield.Type() != reflect.TypeOf(time.UnixMilli(0)) {
+
 			subPrefix := prefix
 			if envkey != "" {
 				subPrefix = subPrefix + envkey + delim
@@ -57,10 +57,7 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			if err != nil {
 				return err
 			}
-		}
 
-		envkey := rsfield.Tag.Get("env")
-		if envkey == "" || envkey == "-" {
 			continue
 		}
 

cryptext/passHash.go

@@ -66,7 +66,6 @@ func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bo
 		return int(version), nil, payload, false, nil, true
 	}
 
-	//
 	if version == 2 {
 		if len(split) != 3 {
 			return -1, nil, nil, false, nil, false

cursortoken/filter.go

@@ -4,6 +4,10 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
+type RawFilter interface {
+	FilterQuery() mongo.Pipeline
+}
+
 type Filter interface {
 	FilterQuery() mongo.Pipeline
 	Pagination() (string, SortDirection, string, SortDirection)

exerr/builder.go

@@ -275,7 +275,7 @@ func (b *Builder) Any(key string, val any) *Builder {
 }
 
 func (b *Builder) Stringer(key string, val fmt.Stringer) *Builder {
-	if val == nil {
+	if langext.IsNil(val) {
 		return b.addMeta(key, MDTString, "(!nil)")
 	} else {
 		return b.addMeta(key, MDTString, val.String())

exerr/data.go

@@ -46,6 +46,7 @@ var (
 	TypeCursorTokenDecode = NewType("CURSOR_TOKEN_DECODE", langext.Ptr(500))
 	TypeMongoFilter       = NewType("MONGO_FILTER", langext.Ptr(500))
 	TypeMongoReflection   = NewType("MONGO_REFLECTION", langext.Ptr(500))
+	TypeMongoInvalidOpt   = NewType("MONGO_INVALIDOPT", langext.Ptr(500))
 
 	TypeWrap = NewType("Wrap", nil)
 
@@ -56,6 +57,7 @@ var (
 	TypeBindFailHeader   = NewType("BINDFAIL_HEADER", langext.Ptr(400))
 
 	TypeMarshalEntityID = NewType("MARSHAL_ENTITY_ID", langext.Ptr(400))
+	TypeInvalidCSID     = NewType("INVALID_CSID", langext.Ptr(400))
 
 	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
 	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))

exerr/exerr.go

@@ -30,7 +30,7 @@ type ExErr struct {
 }
 
 func (ee *ExErr) Error() string {
-	return ee.Message
+	return ee.RecursiveMessage()
 }
 
 // Unwrap must be implemented so that some error.XXX methods work
@@ -164,19 +164,37 @@ func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
 }
 
 func (ee *ExErr) ShortLog(evt *zerolog.Event) {
-	ee.Meta.Apply(evt).Msg(ee.FormatLog(LogPrintShort))
+	ee.Meta.Apply(evt, langext.Ptr(240)).Msg(ee.FormatLog(LogPrintShort))
 }
 
 // RecursiveMessage returns the message to show
 // = first error (top-down) that is not wrapping/foreign/empty
+// = lowest level error (that is not empty)
+// = fallback to self.message
 func (ee *ExErr) RecursiveMessage() string {
+
+	// ==== [1] ==== first error (top-down) that is not wrapping/foreign/empty
+
 	for curr := ee; curr != nil; curr = curr.OriginalError {
 		if curr.Message != "" && curr.Category != CatWrap && curr.Category != CatForeign {
 			return curr.Message
 		}
 	}
 
-	// fallback to self
+	// ==== [2] ==== lowest level error (that is not empty)
+
+	deepestMsg := ""
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Message != "" {
+			deepestMsg = curr.Message
+		}
+	}
+	if deepestMsg != "" {
+		return deepestMsg
+	}
+
+	// ==== [3] ==== fallback to self.message
+
 	return ee.Message
 }
 

exerr/meta.go

@@ -217,23 +217,35 @@ func (v MetaValue) ShortString(lim int) string {
 	return "(err)"
 }
 
-func (v MetaValue) Apply(key string, evt *zerolog.Event) *zerolog.Event {
+func (v MetaValue) Apply(key string, evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	switch v.DataType {
 	case MDTString:
+		if limitLen == nil {
 			return evt.Str(key, v.Value.(string))
+		} else {
+			return evt.Str(key, langext.StrLimit(v.Value.(string), *limitLen, "..."))
+		}
 	case MDTID:
 		return evt.Str(key, v.Value.(IDWrap).Value)
 	case MDTAny:
 		if v.Value.(AnyWrap).IsError {
 			return evt.Str(key, "(err)")
 		} else {
+			if limitLen == nil {
 				return evt.Str(key, v.Value.(AnyWrap).Json)
+			} else {
+				return evt.Str(key, langext.StrLimit(v.Value.(AnyWrap).Json, *limitLen, "..."))
+			}
 		}
 	case MDTStringPtr:
 		if langext.IsNil(v.Value) {
 			return evt.Str(key, "<<null>>")
 		}
+		if limitLen == nil {
 			return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		} else {
+			return evt.Str(key, langext.StrLimit(langext.CoalesceString(v.Value.(*string), "<<null>>"), *limitLen, "..."))
+		}
 	case MDTInt:
 		return evt.Int(key, v.Value.(int))
 	case MDTInt8:
@@ -702,9 +714,9 @@ func (mm MetaMap) Any() bool {
 	return len(mm) > 0
 }
 
-func (mm MetaMap) Apply(evt *zerolog.Event) *zerolog.Event {
+func (mm MetaMap) Apply(evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	for key, val := range mm {
-		evt = val.Apply(key, evt)
+		evt = val.Apply(key, evt, limitLen)
 	}
 	return evt
 }

fsext/exists.go

@@ -0,0 +1,36 @@
+package fsext
+
+import "os"
+
+func PathExists(fp string) (bool, error) {
+	_, err := os.Stat(fp)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func FileExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return !stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func DirectoryExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}

ginext/engine.go

@@ -59,7 +59,7 @@ func NewEngine(allowCors bool, ginDebug bool, bufferBody bool, timeout time.Dura
 	// do not debug-print routes
 	gin.DebugPrintRouteFunc = func(_, _, _ string, _ int) {}
 
-	if ginDebug {
+	if !ginDebug {
 		gin.SetMode(gin.ReleaseMode)
 
 		ginlogger := gin.Logger()

ginext/preContext.go

@@ -1,12 +1,15 @@
 package ginext
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin/binding"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"io"
 	"runtime/debug"
 	"time"
 )
@@ -17,6 +20,7 @@ type PreContext struct {
 	uri     any
 	query   any
 	body    any
+	rawbody *[]byte
 	form    any
 	header  any
 	timeout *time.Duration
@@ -37,6 +41,11 @@ func (pctx *PreContext) Body(body any) *PreContext {
 	return pctx
 }
 
+func (pctx *PreContext) RawBody(rawbody *[]byte) *PreContext {
+	pctx.rawbody = rawbody
+	return pctx
+}
+
 func (pctx *PreContext) Form(form any) *PreContext {
 	pctx.form = form
 	return pctx
@@ -90,6 +99,23 @@ func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
 		}
 	}
 
+	if pctx.rawbody != nil {
+		if brc, ok := pctx.ginCtx.Request.Body.(dataext.BufferedReadCloser); ok {
+			v, err := brc.BufferedAll()
+			if err != nil {
+				return nil, nil, langext.Ptr(Error(err))
+			}
+			*pctx.rawbody = v
+		} else {
+			buf := &bytes.Buffer{}
+			_, err := io.Copy(buf, pctx.ginCtx.Request.Body)
+			if err != nil {
+				return nil, nil, langext.Ptr(Error(err))
+			}
+			*pctx.rawbody = buf.Bytes()
+		}
+	}
+
 	if pctx.form != nil {
 		if pctx.ginCtx.ContentType() == "multipart/form-data" {
 			if err := pctx.ginCtx.ShouldBindWith(pctx.form, binding.Form); err != nil {

ginext/response.go

@@ -27,7 +27,11 @@ func (j jsonHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
-	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true})
+	var f *string
+	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
+		f = &jsonfilter
+	}
+	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f})
 }
 
 func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {

ginext/routes.go

@@ -58,6 +58,14 @@ func (w *GinRoutesWrapper) Use(middleware ...gin.HandlerFunc) *GinRoutesWrapper
 	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
 }
 
+func (w *GinRoutesWrapper) WithJSONFilter(filter string) *GinRoutesWrapper {
+	defHandler := langext.ArrCopy(w.defaultHandler)
+	defHandler = append(defHandler, func(g *gin.Context) {
+		g.Set("goext.jsonfilter", filter)
+	})
+	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+}
+
 func (w *GinRoutesWrapper) GET(relativePath string) *GinRouteBuilder {
 	return w._route(http.MethodGet, relativePath)
 }
@@ -109,6 +117,13 @@ func (w *GinRouteBuilder) Use(middleware ...gin.HandlerFunc) *GinRouteBuilder {
 	return w
 }
 
+func (w *GinRouteBuilder) WithJSONFilter(filter string) *GinRouteBuilder {
+	w.handlers = append(w.handlers, func(g *gin.Context) {
+		g.Set("goext.jsonfilter", filter)
+	})
+	return w
+}
+
 func (w *GinRouteBuilder) Handle(handler WHandlerFunc) {
 
 	if w.routes.wrapper.bufferBody {

go.mod

@@ -1,50 +1,49 @@
 module gogs.mikescher.com/BlackForestBytes/goext
 
-go 1.19
+go 1.21
 
 require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/rs/xid v1.5.0
-	github.com/rs/zerolog v1.30.0
-	go.mongodb.org/mongo-driver v1.12.1
-	golang.org/x/crypto v0.13.0
-	golang.org/x/sys v0.12.0
-	golang.org/x/term v0.12.0
+	github.com/rs/zerolog v1.31.0
+	go.mongodb.org/mongo-driver v1.13.0
+	golang.org/x/crypto v0.15.0
+	golang.org/x/sys v0.14.0
+	golang.org/x/term v0.14.0
 )
 
 require (
-	github.com/bytedance/sonic v1.10.1 // indirect
+	github.com/bytedance/sonic v1.10.2 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
-	github.com/chenzhuoyu/iasm v0.9.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.15.4 // indirect
+	github.com/go-playground/validator/v10 v10.16.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	golang.org/x/arch v0.5.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/arch v0.6.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,243 +1,198 @@
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
-github.com/bytedance/sonic v1.10.0-rc2 h1:oDfRZ+4m6AYCOC0GFeOCeYqvBmucy1isvouS2K0cPzo=
-github.com/bytedance/sonic v1.10.0-rc2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/bytedance/sonic v1.10.0-rc3 h1:uNSnscRapXTwUgTyOF0GVljYD08p9X/Lbr9MweSV3V0=
-github.com/bytedance/sonic v1.10.0-rc3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk=
-github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc=
-github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
+github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
 github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo=
 github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
+github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=
-github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw=
-github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM=
-github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.3 h1:S+sSpunYjNPDuXkWbK+x+bA7iXiW296KG4dL3X7xUZo=
-github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs=
-github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
+github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
+github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
 github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
+github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
-github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
-github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
-github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
-github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
-github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
-github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
-github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
-github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
-github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
-github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
-go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
-go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
-go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
 go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
 go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY=
+go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc=
-golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y=
 golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc=
+golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
-golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
-golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
+golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.267"
+const GoextVersion = "0.0.319"
 
-const GoextVersionTimestamp = "2023-09-21T14:15:02+0200"
+const GoextVersionTimestamp = "2023-11-13T16:45:00+0100"

gojson/encode.go

@@ -156,7 +156,6 @@ import (
 // an error.
 func Marshal(v any) ([]byte, error) {
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
 
 	err := e.marshal(v, encOpts{escapeHTML: true})
 	if err != nil {
@@ -164,6 +163,8 @@ func Marshal(v any) ([]byte, error) {
 	}
 	buf := append([]byte(nil), e.Bytes()...)
 
+	encodeStatePool.Put(e)
+
 	return buf, nil
 }
 
@@ -174,9 +175,9 @@ type IndentOpt struct {
 
 // MarshalSafeCollections is like Marshal except it will marshal nil maps and
 // slices as '{}' and '[]' respectfully instead of 'null'
-func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt) ([]byte, error) {
+func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt, filter *string) ([]byte, error) {
 	e := &encodeState{}
-	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps})
+	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps, filter: filter})
 	if err != nil {
 		return nil, err
 	}
@@ -393,6 +394,9 @@ type encOpts struct {
 	nilSafeSlices bool
 	// nilSafeMaps marshals a nil maps '{}' instead of 'null'
 	nilSafeMaps bool
+	// filter matches jsonfilter tag of struct
+	// marshals if no jsonfilter is set or otherwise if jsonfilter has the filter value
+	filter *string
 }
 
 type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
@@ -777,6 +781,8 @@ FieldLoop:
 
 		if f.omitEmpty && isEmptyValue(fv) {
 			continue
+		} else if opts.filter != nil && len(f.jsonfilter) > 0 && !f.jsonfilter.Contains(*opts.filter) {
+			continue
 		}
 		e.WriteByte(next)
 		next = ','
@@ -1224,11 +1230,24 @@ type field struct {
 	index      []int
 	typ        reflect.Type
 	omitEmpty  bool
+	jsonfilter jsonfilter
 	quoted     bool
 
 	encoder encoderFunc
 }
 
+// jsonfilter stores the value of the jsonfilter struct tag
+type jsonfilter []string
+
+func (j jsonfilter) Contains(t string) bool {
+	for _, tag := range j {
+		if t == tag {
+			return true
+		}
+	}
+	return false
+}
+
 // byIndex sorts field by index sequence.
 type byIndex []field
 
@@ -1304,6 +1323,13 @@ func typeFields(t reflect.Type) structFields {
 				if !isValidTag(name) {
 					name = ""
 				}
+
+				var jsonfilter []string
+				jsonfilterTag := sf.Tag.Get("jsonfilter")
+				if jsonfilterTag != "" && jsonfilterTag != "-" {
+					jsonfilter = strings.Split(jsonfilterTag, ",")
+				}
+
 				index := make([]int, len(f.index)+1)
 				copy(index, f.index)
 				index[len(f.index)] = i
@@ -1339,6 +1365,7 @@ func typeFields(t reflect.Type) structFields {
 						index:      index,
 						typ:        ft,
 						omitEmpty:  opts.Contains("omitempty"),
+						jsonfilter: jsonfilter,
 						quoted:     quoted,
 					}
 					field.nameBytes = []byte(field.name)

gojson/encode_test.go

@@ -1253,6 +1253,10 @@ func TestMarshalSafeCollections(t *testing.T) {
 		nilMapStruct struct {
 			NilMap map[string]interface{} `json:"nil_map"`
 		}
+		testWithFilter struct {
+			Test1 string `json:"test1" jsonfilter:"FILTERONE"`
+			Test2 string `json:"test2" jsonfilter:"FILTERTWO"`
+		}
 	)
 
 	tests := []struct {
@@ -1271,10 +1275,12 @@ func TestMarshalSafeCollections(t *testing.T) {
 		{map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}"},
 		{pNilMap, "null"},
 		{nilMapStruct{}, "{\"nil_map\":{}}"},
+		{testWithFilter{}, "{\"test1\":\"\"}"},
 	}
 
+	filter := "FILTERONE"
 	for i, tt := range tests {
-		b, err := MarshalSafeCollections(tt.in, true, true, nil)
+		b, err := MarshalSafeCollections(tt.in, true, true, nil, &filter)
 		if err != nil {
 			t.Errorf("test %d, unexpected failure: %v", i, err)
 		}

gojson/fold.go

@@ -97,7 +97,10 @@ func equalFoldRight(s, t []byte) bool {
 		t = t[size:]
 
 	}
-	return len(t) == 0
+	if len(t) > 0 {
+		return false
+	}
+	return true
 }
 
 // asciiEqualFold is a specialization of bytes.EqualFold for use when

gojson/fold_test.go

@@ -52,7 +52,9 @@ func TestFold(t *testing.T) {
 }
 
 func TestFoldAgainstUnicode(t *testing.T) {
-	var buf1, buf2 []byte
+	const bufSize = 5
+	buf1 := make([]byte, 0, bufSize)
+	buf2 := make([]byte, 0, bufSize)
 	var runes []rune
 	for i := 0x20; i <= 0x7f; i++ {
 		runes = append(runes, rune(i))
@@ -94,8 +96,12 @@ func TestFoldAgainstUnicode(t *testing.T) {
 				continue
 			}
 			for _, r2 := range runes {
-				buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x')
-				buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x')
+				buf1 := append(buf1[:0], 'x')
+				buf2 := append(buf2[:0], 'x')
+				buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
+				buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
+				buf1 = append(buf1, 'x')
+				buf2 = append(buf2, 'x')
 				want := bytes.EqualFold(buf1, buf2)
 				if got := ff.fold(buf1, buf2); got != want {
 					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)

gojson/gionic.go

@@ -17,6 +17,7 @@ type GoJsonRender struct {
 	NilSafeSlices bool
 	NilSafeMaps   bool
 	Indent        *IndentOpt
+	Filter        *string
 }
 
 func (r GoJsonRender) Render(w http.ResponseWriter) error {
@@ -25,7 +26,7 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
 		header["Content-Type"] = []string{"application/json; charset=utf-8"}
 	}
 
-	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent)
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
 	if err != nil {
 		panic(err)
 	}

gojson/number_test.go

@@ -116,3 +116,18 @@ func TestNumberIsValid(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkNumberIsValid(b *testing.B) {
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		isValidNumber(s)
+	}
+}
+
+func BenchmarkNumberIsValidRegexp(b *testing.B) {
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		jsonNumberRegexp.MatchString(s)
+	}
+}

gojson/scanner.go

@@ -594,7 +594,7 @@ func (s *scanner) error(c byte, context string) int {
 	return scanError
 }
 
-// quoteChar formats c as a quoted character literal.
+// quoteChar formats c as a quoted character literal
 func quoteChar(c byte) string {
 	// special cases - different from quoted strings
 	if c == '\'' {

gojson/stream.go

@@ -182,8 +182,6 @@ type Encoder struct {
 	w          io.Writer
 	err        error
 	escapeHTML bool
-	nilSafeSlices bool
-	nilSafeMaps   bool
 
 	indentBuf    *bytes.Buffer
 	indentPrefix string
@@ -204,11 +202,8 @@ func (enc *Encoder) Encode(v any) error {
 	if enc.err != nil {
 		return enc.err
 	}
-
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
-
-	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
+	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
 	if err != nil {
 		return err
 	}
@@ -236,6 +231,7 @@ func (enc *Encoder) Encode(v any) error {
 	if _, err = enc.w.Write(b); err != nil {
 		enc.err = err
 	}
+	encodeStatePool.Put(e)
 	return err
 }
 
@@ -247,13 +243,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) {
 	enc.indentValue = indent
 }
 
-// SetNilSafeCollection specifies whether to represent nil slices and maps as
-// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
-func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
-	enc.nilSafeSlices = nilSafeSlices
-	enc.nilSafeMaps = nilSafeMaps
-}
-
 // SetEscapeHTML specifies whether problematic HTML characters
 // should be escaped inside JSON quoted strings.
 // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e

gojson/stream_test.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
-	"runtime/debug"
 	"strings"
 	"testing"
 )
@@ -42,7 +41,7 @@ false
 
 func TestEncoder(t *testing.T) {
 	for i := 0; i <= len(streamTest); i++ {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		// Check that enc.SetIndent("", "") turns off indentation.
 		enc.SetIndent(">", ".")
@@ -60,43 +59,6 @@ func TestEncoder(t *testing.T) {
 	}
 }
 
-func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
-	// Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test.
-	percent := debug.SetGCPercent(-1)
-	defer debug.SetGCPercent(percent)
-
-	// Trigger an error in Marshal with cyclic data.
-	type Dummy struct {
-		Name string
-		Next *Dummy
-	}
-	dummy := Dummy{Name: "Dummy"}
-	dummy.Next = &dummy
-
-	var buf bytes.Buffer
-	enc := NewEncoder(&buf)
-	if err := enc.Encode(dummy); err == nil {
-		t.Errorf("Encode(dummy) == nil; want error")
-	}
-
-	type Data struct {
-		A string
-		I int
-	}
-	data := Data{A: "a", I: 1}
-	if err := enc.Encode(data); err != nil {
-		t.Errorf("Marshal(%v) = %v", data, err)
-	}
-
-	var data2 Data
-	if err := Unmarshal(buf.Bytes(), &data2); err != nil {
-		t.Errorf("Unmarshal(%v) = %v", data2, err)
-	}
-	if data2 != data {
-		t.Errorf("expect: %v, but get: %v", data, data2)
-	}
-}
-
 var streamEncodedIndent = `0.1
 "hello"
 null
@@ -115,7 +77,7 @@ false
 `
 
 func TestEncoderIndent(t *testing.T) {
-	var buf strings.Builder
+	var buf bytes.Buffer
 	enc := NewEncoder(&buf)
 	enc.SetIndent(">", ".")
 	for _, v := range streamTest {
@@ -185,7 +147,7 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
 			`{"bar":"\"<html>foobar</html>\""}`,
 		},
 	} {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		if err := enc.Encode(tt.v); err != nil {
 			t.Errorf("Encode(%s): %s", tt.name, err)
@@ -347,6 +309,21 @@ func TestBlocking(t *testing.T) {
 	}
 }
 
+func BenchmarkEncoderEncode(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X, Y string
+	}
+	v := &T{"foo", "bar"}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			if err := NewEncoder(io.Discard).Encode(v); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
 type tokenStreamCase struct {
 	json      string
 	expTokens []any
@@ -495,45 +472,3 @@ func TestHTTPDecoding(t *testing.T) {
 		t.Errorf("err = %v; want io.EOF", err)
 	}
 }
-
-func TestEncoderSetNilSafeCollection(t *testing.T) {
-	var (
-		nilSlice  []interface{}
-		pNilSlice *[]interface{}
-		nilMap    map[string]interface{}
-		pNilMap   *map[string]interface{}
-	)
-	for _, tt := range []struct {
-		name        string
-		v           interface{}
-		want        string
-		rescuedWant string
-	}{
-		{"nilSlice", nilSlice, "null", "[]"},
-		{"nonNilSlice", []interface{}{}, "[]", "[]"},
-		{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
-		{"pNilSlice", pNilSlice, "null", "null"},
-		{"nilMap", nilMap, "null", "{}"},
-		{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
-		{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
-		{"pNilMap", pNilMap, "null", "null"},
-	} {
-		var buf bytes.Buffer
-		enc := NewEncoder(&buf)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.want {
-			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
-		}
-		buf.Reset()
-		enc.SetNilSafeCollection(true, true)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
-			t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
-				tt.name, got, tt.want)
-		}
-	}
-}

langext/baseAny.go

@@ -0,0 +1,178 @@
+package langext
+
+import (
+	"crypto/rand"
+	"errors"
+	"math"
+	"math/big"
+)
+
+type AnyBaseConverter struct {
+	base    uint64
+	charset []rune
+}
+
+func NewAnyBaseConverter(cs string) AnyBaseConverter {
+	rcs := []rune(cs)
+	return AnyBaseConverter{
+		base:    uint64(len(rcs)),
+		charset: rcs,
+	}
+}
+
+func (bc AnyBaseConverter) Rand(rlen int) string {
+	biBase := big.NewInt(int64(bc.base))
+
+	randMax := big.NewInt(math.MaxInt64)
+
+	r := ""
+
+	for i := 0; i < rlen; i++ {
+		v, err := rand.Int(rand.Reader, randMax)
+		if err != nil {
+			panic(err)
+		}
+
+		r += string(bc.charset[v.Mod(v, biBase).Int64()])
+	}
+
+	return r
+}
+
+func (bc AnyBaseConverter) EncodeUInt64(num uint64) string {
+	if num == 0 {
+		return "0"
+	}
+
+	b := ""
+
+	// loop as long the num is bigger than zero
+	for num > 0 {
+		r := num % bc.base
+
+		num -= r
+		num /= base62Base
+
+		b += string(bc.charset[int(r)])
+	}
+
+	return b
+}
+
+func (bc AnyBaseConverter) DecodeUInt64(str string) (uint64, error) {
+	if str == "" {
+		return 0, errors.New("empty string")
+	}
+
+	result := uint64(0)
+
+	for _, v := range str {
+		result *= base62Base
+
+		pos := ArrFirstIndex(bc.charset, v)
+		if pos == -1 {
+			return 0, errors.New("invalid character: " + string(v))
+		}
+
+		result += uint64(pos)
+	}
+
+	return result, nil
+}
+
+func (bc AnyBaseConverter) Encode(src []byte) string {
+	value := new(big.Int)
+	value.SetBytes(src)
+	return bc.EncodeBigInt(value)
+}
+
+func (bc AnyBaseConverter) EncodeBigInt(src *big.Int) string {
+	value := new(big.Int)
+	value.Set(src)
+
+	isneg := value.Sign() < 0
+
+	answer := ""
+
+	if isneg {
+		value.Neg(value)
+	}
+
+	biBase := big.NewInt(int64(bc.base))
+
+	rem := new(big.Int)
+
+	for value.Sign() > 0 {
+		value.QuoRem(value, biBase, rem)
+		answer = string(bc.charset[rem.Int64()]) + answer
+	}
+
+	if isneg {
+		return "-" + answer
+	} else {
+		return answer
+	}
+}
+
+func (bc AnyBaseConverter) Decode(src string) ([]byte, error) {
+	value, err := bc.DecodeToBigInt(src)
+	if err != nil {
+		return nil, err
+	}
+	return value.Bytes(), nil
+}
+
+func (bc AnyBaseConverter) DecodeToBigInt(_src string) (*big.Int, error) {
+	result := new(big.Int)
+	result.SetInt64(0)
+
+	src := []rune(_src)
+
+	if len(src) == 0 {
+		return nil, errors.New("string is empty")
+	}
+	if bc.base < 2 {
+		return nil, errors.New("not enough digits")
+	}
+
+	i := 0
+
+	sign := new(big.Int)
+	sign.SetInt64(1)
+	if src[i] == '+' {
+		i++
+	} else if src[i] == '-' {
+		i++
+		sign.SetInt64(-1)
+	}
+
+	if i >= len(src) {
+		return nil, errors.New("no digits in input")
+	}
+
+	biBase := big.NewInt(int64(bc.base))
+
+	oldResult := new(big.Int)
+
+	for ; i < len(src); i++ {
+		n := ArrFirstIndex(bc.charset, src[i])
+		if n < 0 {
+			return nil, errors.New("invalid characters in input")
+		}
+
+		oldResult.Set(result)
+
+		result.Mul(result, biBase)
+		result.Add(result, big.NewInt(int64(n)))
+
+		if result.Cmp(oldResult) < 0 {
+			return nil, errors.New("overflow")
+		}
+	}
+
+	if sign.Cmp(big.NewInt(0)) < 0 {
+		result.Neg(result)
+	}
+
+	return result, nil
+}

langext/baseAny_test.go

@@ -0,0 +1,80 @@
+package langext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func _anyEncStr(bc AnyBaseConverter, v string) string {
+	vr := bc.Encode([]byte(v))
+	return vr
+}
+
+func _anyDecStr(bc AnyBaseConverter, v string) string {
+	vr, err := bc.Decode(v)
+	if err != nil {
+		panic(err)
+	}
+	return string(vr)
+}
+
+func TestAnyBase58DefaultEncoding(t *testing.T) {
+	tst.AssertEqual(t, _anyEncStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "Hello"), "9Ajdvzr")
+	tst.AssertEqual(t, _anyEncStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX")
+}
+
+func TestAnyBase58DefaultDecoding(t *testing.T) {
+	tst.AssertEqual(t, _anyDecStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "9Ajdvzr"), "Hello")
+	tst.AssertEqual(t, _anyDecStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestAnyBaseDecode(t *testing.T) {
+
+	const (
+		Binary  = "01"
+		Decimal = "0123456789"
+		Hex     = "0123456789ABCDEF"
+		DNA     = "ACGT"
+		Base32  = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+		Base58  = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+		Base62  = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+		Base64  = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+		Base256 = "🚀🪐☄🛰🌌🌑🌒🌓🌔🌕🌖🌗🌘🌍🌏🌎🐉☀💻🖥💾💿😂❤😍🤣😊🙏💕😭😘👍😅👏😁🔥🥰💔💖💙😢🤔😆🙄💪😉☺👌🤗💜😔😎😇🌹🤦🎉💞✌✨🤷😱😌🌸🙌😋💗💚😏💛🙂💓🤩😄😀🖤😃💯🙈👇🎶😒🤭❣😜💋👀😪😑💥🙋😞😩😡🤪👊🥳😥🤤👉💃😳✋😚😝😴🌟😬🙃🍀🌷😻😓⭐✅🥺🌈😈🤘💦✔😣🏃💐☹🎊💘😠☝😕🌺🎂🌻😐🖕💝🙊😹🗣💫💀👑🎵🤞😛🔴😤🌼😫⚽🤙☕🏆🤫👈😮🙆🍻🍃🐶💁😲🌿🧡🎁⚡🌞🎈❌✊👋😰🤨😶🤝🚶💰🍓💢🤟🙁🚨💨🤬✈🎀🍺🤓😙💟🌱😖👶🥴▶➡❓💎💸⬇😨🌚🦋😷🕺⚠🙅😟😵👎🤲🤠🤧📌🔵💅🧐🐾🍒😗🤑🌊🤯🐷☎💧😯💆👆🎤🙇🍑❄🌴💣🐸💌📍🥀🤢👅💡💩👐📸👻🤐🤮🎼🥵🚩🍎🍊👼💍📣🥂"
+	)
+
+	type TestDef struct {
+		FromCS  string
+		FromVal string
+		ToCS    string
+		ToVal   string
+	}
+
+	defs := []TestDef{
+		{Binary, "10100101011100000101010", Decimal, "5421098"},
+		{Decimal, "5421098", DNA, "CCAGGTGAAGGG"},
+		{Decimal, "5421098", DNA, "CCAGGTGAAGGG"},
+		{Decimal, "80085", Base256, "🪐💞🔵"},
+		{Hex, "48656C6C6C20576F526C5421", Base64, "SGVsbGwgV29SbFQh"},
+		{Base64, "SGVsbGw/gV29SbF+Qh", Base32, "CIMVWGY3B7QFO32SNRPZBB"},
+		{Base64, "SGVsbGw/gV29SbF+Qh", Base58, "2fUsGKQUcgQcwSqpvy6"},
+		{Base64, "SGVsbGw/gV29SbF+Qh", Base62, "V34nvybdQ3m3RHk9Sr"},
+	}
+
+	for _, def := range defs {
+
+		d1 := NewAnyBaseConverter(def.FromCS)
+		d2 := NewAnyBaseConverter(def.ToCS)
+
+		v1 := tst.Must(d1.Decode(def.FromVal))(t)
+		v2 := tst.Must(d2.Decode(def.ToVal))(t)
+
+		tst.AssertArrayEqual(t, v1, v2)
+
+		str2 := d2.Encode(v1)
+		tst.AssertEqual(t, str2, def.ToVal)
+
+		str1 := d1.Encode(v2)
+		tst.AssertEqual(t, str1, def.FromVal)
+
+	}
+}

langext/panic.go

@@ -1,7 +1,10 @@
 package langext
 
+import "runtime/debug"
+
 type PanicWrappedErr struct {
 	panic any
+	Stack string
 }
 
 func (p PanicWrappedErr) Error() string {
@@ -15,7 +18,7 @@ func (p PanicWrappedErr) ReoveredObj() any {
 func RunPanicSafe(fn func()) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -27,7 +30,7 @@ func RunPanicSafe(fn func()) (err error) {
 func RunPanicSafeR1(fn func() error) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -38,7 +41,7 @@ func RunPanicSafeR2[T1 any](fn func() (T1, error)) (r1 T1, err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
 			r1 = *new(T1)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -50,7 +53,7 @@ func RunPanicSafeR3[T1 any, T2 any](fn func() (T1, T2, error)) (r1 T1, r2 T2, er
 		if rec := recover(); rec != nil {
 			r1 = *new(T1)
 			r2 = *new(T2)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -63,7 +66,7 @@ func RunPanicSafeR4[T1 any, T2 any, T3 any](fn func() (T1, T2, T3, error)) (r1 T
 			r1 = *new(T1)
 			r2 = *new(T2)
 			r3 = *new(T3)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 

mongoext/registry.go

@@ -3,6 +3,9 @@ package mongoext
 import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
 	"reflect"
 )
@@ -16,13 +19,29 @@ func CreateGoExtBsonRegistry() *bsoncodec.Registry {
 	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.RFC3339NanoTime{}), rfctime.RFC3339NanoTime{})
 	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.RFC3339NanoTime{}), rfctime.RFC3339NanoTime{})
 
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.UnixTime{}), rfctime.UnixTime{})
+	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.UnixTime{}), rfctime.UnixTime{})
+
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.UnixMilliTime{}), rfctime.UnixMilliTime{})
+	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.UnixMilliTime{}), rfctime.UnixMilliTime{})
+
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.UnixNanoTime{}), rfctime.UnixNanoTime{})
+	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.UnixNanoTime{}), rfctime.UnixNanoTime{})
+
 	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.Date{}), rfctime.Date{})
 	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.Date{}), rfctime.Date{})
 
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.SecondsF64(0)), rfctime.SecondsF64(0))
+	rb.RegisterTypeDecoder(reflect.TypeOf(langext.Ptr(rfctime.SecondsF64(0))), rfctime.SecondsF64(0))
+
 	bsoncodec.DefaultValueEncoders{}.RegisterDefaultEncoders(rb)
 	bsoncodec.DefaultValueDecoders{}.RegisterDefaultDecoders(rb)
 
 	bson.PrimitiveCodecs{}.RegisterPrimitiveCodecs(rb)
 
+	// otherwise we get []primitve.E when unmarshalling into any
+	// which will result in {'key': .., 'value': ...}[] json when json-marshalling
+	rb.RegisterTypeMapEntry(bsontype.EmbeddedDocument, reflect.TypeOf(primitive.M{}))
+
 	return rb.Build()
 }

pagination/filter.go

@@ -0,0 +1,28 @@
+package pagination
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type Filter interface {
+	FilterQuery() mongo.Pipeline
+	Sort() bson.D
+}
+
+type dynamicFilter struct {
+	pipeline mongo.Pipeline
+	sort     bson.D
+}
+
+func (d dynamicFilter) FilterQuery() mongo.Pipeline {
+	return d.pipeline
+}
+
+func (d dynamicFilter) Sort() bson.D {
+	return d.sort
+}
+
+func CreateFilter(pipeline mongo.Pipeline, sort bson.D) Filter {
+	return dynamicFilter{pipeline: pipeline, sort: sort}
+}

pagination/pagination.go

@@ -0,0 +1,16 @@
+package pagination
+
+type Pagination struct {
+	Page             int `json:"page"`            // page (first page == 1)
+	Limit            int `json:"limit"`           // max-page-size
+	TotalPages       int `json:"totalPages"`      // total page-count
+	TotalItems       int `json:"totalItems"`      // total items-count
+	CurrentPageCount int `json:"currntPageCount"` // item-count in current page ( == len(data) )
+}
+
+func CalcPaginationTotalPages(totalItems int, limit int) int {
+	if totalItems == 0 {
+		return 0
+	}
+	return 1 + (totalItems-1)/limit
+}

rfctime/seconds.go

@@ -2,7 +2,14 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
 	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"reflect"
 	"time"
 )
 
@@ -54,6 +61,63 @@ func (d SecondsF64) MarshalJSON() ([]byte, error) {
 	return json.Marshal(secs)
 }
 
+func (d *SecondsF64) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*d = SecondsF64(0)
+		return nil
+	}
+	if bt != bson.TypeDouble {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into SecondsF64", bt))
+	}
+	var tt float64
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*d = SecondsF64(tt)
+	return nil
+}
+
+func (d SecondsF64) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(d.Seconds())
+}
+
+func (d SecondsF64) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = d.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&d))
+	} else {
+		val.Set(reflect.ValueOf(d))
+	}
+
+	return nil
+}
+
 func NewSecondsF64(t time.Duration) SecondsF64 {
 	return SecondsF64(t)
 }

rfctime/unix.go

@@ -2,6 +2,13 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -59,6 +66,63 @@ func (t *UnixTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *UnixTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*t = UnixTime{}
+		return nil
+	}
+	if bt != bson.TypeDateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into UnixTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = UnixTime(tt)
+	return nil
+}
+
+func (t UnixTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t UnixTime) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = t.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&t))
+	} else {
+		val.Set(reflect.ValueOf(t))
+	}
+
+	return nil
+}
+
 func (t UnixTime) Serialize() string {
 	return strconv.FormatInt(t.Time().Unix(), 10)
 }

rfctime/unixMilli.go

@@ -2,6 +2,13 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -59,6 +66,63 @@ func (t *UnixMilliTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *UnixMilliTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*t = UnixMilliTime{}
+		return nil
+	}
+	if bt != bson.TypeDateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into UnixMilliTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = UnixMilliTime(tt)
+	return nil
+}
+
+func (t UnixMilliTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t UnixMilliTime) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = t.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&t))
+	} else {
+		val.Set(reflect.ValueOf(t))
+	}
+
+	return nil
+}
+
 func (t UnixMilliTime) Serialize() string {
 	return strconv.FormatInt(t.Time().UnixMilli(), 10)
 }

rfctime/unixNano.go

@@ -2,6 +2,13 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -59,6 +66,63 @@ func (t *UnixNanoTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *UnixNanoTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*t = UnixNanoTime{}
+		return nil
+	}
+	if bt != bson.TypeDateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into UnixNanoTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = UnixNanoTime(tt)
+	return nil
+}
+
+func (t UnixNanoTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t UnixNanoTime) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = t.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&t))
+	} else {
+		val.Set(reflect.ValueOf(t))
+	}
+
+	return nil
+}
+
 func (t UnixNanoTime) Serialize() string {
 	return strconv.FormatInt(t.Time().UnixNano(), 10)
 }

syncext/atomic.go

@@ -1,109 +1,34 @@
 package syncext
 
 import (
-	"context"
-	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"sync"
-	"time"
 )
 
-type AtomicBool struct {
-	v        bool
-	listener map[string]chan bool
-	lock     sync.Mutex
+type Atomic[T any] struct {
+	v    T
+	lock sync.RWMutex
 }
 
-func NewAtomicBool(value bool) *AtomicBool {
-	return &AtomicBool{
+func NewAtomic[T any](value T) *Atomic[T] {
+	return &Atomic[T]{
 		v:    value,
-		listener: make(map[string]chan bool),
-		lock:     sync.Mutex{},
+		lock: sync.RWMutex{},
 	}
 }
 
-func (a *AtomicBool) Get() bool {
-	a.lock.Lock()
-	defer a.lock.Unlock()
+func (a *Atomic[T]) Get() T {
+	a.lock.RLock()
+	defer a.lock.RUnlock()
 	return a.v
 }
 
-func (a *AtomicBool) Set(value bool) {
+func (a *Atomic[T]) Set(value T) T {
 	a.lock.Lock()
 	defer a.lock.Unlock()
 
+	oldValue := a.v
+
 	a.v = value
 
-	for k, v := range a.listener {
-		select {
-		case v <- value:
-			// message sent
-		default:
-			// no receiver on channel
-			delete(a.listener, k)
-		}
-	}
-}
-
-func (a *AtomicBool) Wait(waitFor bool) {
-	_ = a.WaitWithContext(context.Background(), waitFor)
-}
-
-func (a *AtomicBool) WaitWithTimeout(timeout time.Duration, waitFor bool) error {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	return a.WaitWithContext(ctx, waitFor)
-}
-
-func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-
-	if a.Get() == waitFor {
-		return nil
-	}
-
-	uuid, _ := langext.NewHexUUID()
-
-	waitchan := make(chan bool)
-
-	a.lock.Lock()
-	a.listener[uuid] = waitchan
-	a.lock.Unlock()
-	defer func() {
-		a.lock.Lock()
-		delete(a.listener, uuid)
-		a.lock.Unlock()
-	}()
-
-	for {
-		if err := ctx.Err(); err != nil {
-			return err
-		}
-
-		timeOut := 1024 * time.Millisecond
-
-		if dl, ok := ctx.Deadline(); ok {
-			timeOutMax := dl.Sub(time.Now())
-			if timeOutMax <= 0 {
-				timeOut = 0
-			} else if 0 < timeOutMax && timeOutMax < timeOut {
-				timeOut = timeOutMax
-			}
-		}
-
-		if v, ok := ReadChannelWithTimeout(waitchan, timeOut); ok {
-			if v == waitFor {
-				return nil
-			}
-		} else {
-			if err := ctx.Err(); err != nil {
-				return err
-			}
-
-			if a.Get() == waitFor {
-				return nil
-			}
-		}
-	}
+	return oldValue
 }

syncext/bool.go

@@ -0,0 +1,113 @@
+package syncext
+
+import (
+	"context"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"sync"
+	"time"
+)
+
+type AtomicBool struct {
+	v        bool
+	listener map[string]chan bool
+	lock     sync.Mutex
+}
+
+func NewAtomicBool(value bool) *AtomicBool {
+	return &AtomicBool{
+		v:        value,
+		listener: make(map[string]chan bool),
+		lock:     sync.Mutex{},
+	}
+}
+
+func (a *AtomicBool) Get() bool {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+	return a.v
+}
+
+func (a *AtomicBool) Set(value bool) bool {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	oldValue := a.v
+
+	a.v = value
+
+	for k, v := range a.listener {
+		select {
+		case v <- value:
+			// message sent
+		default:
+			// no receiver on channel
+			delete(a.listener, k)
+		}
+	}
+
+	return oldValue
+}
+
+func (a *AtomicBool) Wait(waitFor bool) {
+	_ = a.WaitWithContext(context.Background(), waitFor)
+}
+
+func (a *AtomicBool) WaitWithTimeout(timeout time.Duration, waitFor bool) error {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	return a.WaitWithContext(ctx, waitFor)
+}
+
+func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	if a.Get() == waitFor {
+		return nil
+	}
+
+	uuid, _ := langext.NewHexUUID()
+
+	waitchan := make(chan bool)
+
+	a.lock.Lock()
+	a.listener[uuid] = waitchan
+	a.lock.Unlock()
+	defer func() {
+		a.lock.Lock()
+		delete(a.listener, uuid)
+		a.lock.Unlock()
+	}()
+
+	for {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+
+		timeOut := 1024 * time.Millisecond
+
+		if dl, ok := ctx.Deadline(); ok {
+			timeOutMax := dl.Sub(time.Now())
+			if timeOutMax <= 0 {
+				timeOut = 0
+			} else if 0 < timeOutMax && timeOutMax < timeOut {
+				timeOut = timeOutMax
+			}
+		}
+
+		if v, ok := ReadChannelWithTimeout(waitchan, timeOut); ok {
+			if v == waitFor {
+				return nil
+			}
+		} else {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+
+			if a.Get() == waitFor {
+				return nil
+			}
+		}
+	}
+}

timeext/calendarweek.go

@@ -0,0 +1,28 @@
+package timeext
+
+import "time"
+
+func WeekStart(year, week int) time.Time {
+
+	// https://stackoverflow.com/a/52303730/1761622
+
+	// Start from the middle of the year:
+	t := time.Date(year, 7, 1, 0, 0, 0, 0, time.UTC)
+
+	// Roll back to Monday:
+	if wd := t.Weekday(); wd == time.Sunday {
+		t = t.AddDate(0, 0, -6)
+	} else {
+		t = t.AddDate(0, 0, -int(wd)+1)
+	}
+
+	// Difference in weeks:
+	_, w := t.ISOWeek()
+	t = t.AddDate(0, 0, (week-w)*7)
+
+	return t
+}
+
+func WeekEnd(year, week int) time.Time {
+	return WeekStart(year, week).AddDate(0, 0, 7).Add(time.Duration(-1))
+}

timeext/calendarweek_test.go

@@ -0,0 +1,25 @@
+package timeext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+	"time"
+)
+
+func TestWeekStart(t *testing.T) {
+
+	tst.AssertEqual(t, WeekStart(2018, 1).Format(time.RFC3339Nano), "2018-01-01T00:00:00Z")
+	tst.AssertEqual(t, WeekStart(2018, 2).Format(time.RFC3339Nano), "2018-01-08T00:00:00Z")
+	tst.AssertEqual(t, WeekStart(2019, 1).Format(time.RFC3339Nano), "2018-12-31T00:00:00Z")
+	tst.AssertEqual(t, WeekStart(2019, 2).Format(time.RFC3339Nano), "2019-01-07T00:00:00Z")
+
+}
+
+func TestWeekEnd(t *testing.T) {
+
+	tst.AssertEqual(t, WeekEnd(2018, 1).Format(time.RFC3339Nano), "2018-01-07T23:59:59.999999999Z")
+	tst.AssertEqual(t, WeekEnd(2018, 2).Format(time.RFC3339Nano), "2018-01-14T23:59:59.999999999Z")
+	tst.AssertEqual(t, WeekEnd(2019, 1).Format(time.RFC3339Nano), "2019-01-06T23:59:59.999999999Z")
+	tst.AssertEqual(t, WeekEnd(2019, 2).Format(time.RFC3339Nano), "2019-01-13T23:59:59.999999999Z")
+
+}

tst/assertions.go

@@ -14,6 +14,20 @@ func AssertEqual[T comparable](t *testing.T, actual T, expected T) {
 	}
 }
 
+func AssertArrayEqual[T comparable](t *testing.T, actual []T, expected []T) {
+	t.Helper()
+	if len(actual) != len(expected) {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v' (len %d <> %d)", actual, expected, len(actual), len(expected))
+		return
+	}
+	for i := 0; i < len(actual); i++ {
+		if actual[i] != expected[i] {
+			t.Errorf("values differ: Actual: '%v', Expected: '%v' (at index %d)", actual, expected, i)
+			return
+		}
+	}
+}
+
 func AssertNotEqual[T comparable](t *testing.T, actual T, expected T) {
 	t.Helper()
 	if actual == expected {
@@ -23,14 +37,32 @@ func AssertNotEqual[T comparable](t *testing.T, actual T, expected T) {
 
 func AssertDeepEqual[T any](t *testing.T, actual T, expected T) {
 	t.Helper()
-	if reflect.DeepEqual(actual, expected) {
+	if !reflect.DeepEqual(actual, expected) {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
 
+func AssertSetDeepEqual[T any](t *testing.T, actual []T, expected []T) {
+	t.Helper()
+	if len(actual) != len(expected) {
+		t.Errorf("values differ in length: Actual (n=%d): '%v', Expected (n=%d): '%v'", len(actual), actual, len(expected), expected)
+	}
+
+	for _, a := range expected {
+		found := false
+		for _, b := range actual {
+			found = found || reflect.DeepEqual(a, b)
+		}
+		if !found {
+			t.Errorf("values differ: Element '%v' not found. Actual: '%v', Expected: '%v'", a, actual, expected)
+			return
+		}
+	}
+}
+
 func AssertNotDeepEqual[T any](t *testing.T, actual T, expected T) {
 	t.Helper()
-	if !reflect.DeepEqual(actual, expected) {
+	if reflect.DeepEqual(actual, expected) {
 		t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }

tst/must.go

@@ -0,0 +1,21 @@
+package tst
+
+import (
+	"runtime/debug"
+	"testing"
+)
+
+// Must can b used to AssertNoErr of an (T, err) function
+//
+// Usage:
+//
+// input := "123.8"
+// value := tst.Must(strconv.Atoi(input))(t)
+func Must[T any](v T, anerr error) func(t *testing.T) T {
+	return func(t *testing.T) T {
+		if anerr != nil {
+			t.Error("Function returned an error: " + anerr.Error() + "\n" + string(debug.Stack()))
+		}
+		return v
+	}
+}

wmo/collection.go

@@ -44,6 +44,8 @@ type Coll[TData any] struct {
 	implDataTypeMap     map[reflect.Type]map[string]fullTypeRef                  // dynamic list of fields of TData implementations (only if TData is an interface)
 	customDecoder       *func(ctx context.Context, dec Decodable) (TData, error) // custom decoding function (useful if TData is an interface)
 	isInterfaceDataType bool                                                     // true if TData is an interface (not a struct)
+	unmarshalHooks      []func(d TData) TData                                    // called for every object after unmarshalling
+	extraModPipeline    mongo.Pipeline                                           // appended to pipelines after filter/limit/skip/sort, used for $lookup, $set, $unset, $project, etc
 }
 
 func (c *Coll[TData]) Collection() *mongo.Collection {
@@ -54,14 +56,6 @@ func (c *Coll[TData]) Name() string {
 	return c.coll.Name()
 }
 
-func (c *Coll[TData]) WithDecodeFunc(cdf func(ctx context.Context, dec Decodable) (TData, error), example TData) *Coll[TData] {
-
-	c.EnsureInitializedReflection(example)
-
-	c.customDecoder = langext.Ptr(cdf)
-	return c
-}
-
 func (c *Coll[TData]) Indexes() mongo.IndexView {
 	return c.coll.Indexes()
 }
@@ -74,6 +68,26 @@ func (c *Coll[TData]) Drop(ctx context.Context) error {
 	return nil
 }
 
+func (c *Coll[TData]) WithDecodeFunc(cdf func(ctx context.Context, dec Decodable) (TData, error), example TData) *Coll[TData] {
+
+	c.EnsureInitializedReflection(example)
+
+	c.customDecoder = langext.Ptr(cdf)
+	return c
+}
+
+func (c *Coll[TData]) WithUnmarshalHook(fn func(d TData) TData) *Coll[TData] {
+	c.unmarshalHooks = append(c.unmarshalHooks, fn)
+
+	return c
+}
+
+func (c *Coll[TData]) WithModifyingPipeline(p mongo.Pipeline) *Coll[TData] {
+	c.extraModPipeline = append(c.extraModPipeline, p...)
+
+	return c
+}
+
 func (c *Coll[TData]) createToken(fieldPrimary string, dirPrimary ct.SortDirection, fieldSecondary *string, dirSecondary *ct.SortDirection, lastEntity TData, pageSize *int) (ct.CursorToken, error) {
 
 	valuePrimary, err := c.getFieldValueAsTokenString(lastEntity, fieldPrimary)

wmo/decoding.go

@@ -2,37 +2,39 @@ package wmo
 
 import (
 	"context"
+	"go.mongodb.org/mongo-driver/bson"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 )
 
 func (c *Coll[TData]) decodeSingle(ctx context.Context, dec Decodable) (TData, error) {
-	if c.customDecoder != nil {
 
-		res, err := (*c.customDecoder)(ctx, dec)
+	var res TData
+	var err error
+
+	if c.customDecoder != nil {
+		res, err = (*c.customDecoder)(ctx, dec)
 		if err != nil {
 			return *new(TData), exerr.Wrap(err, "failed to decode single entity with custom-decoder").Type("decoder", *c.customDecoder).Build()
 		}
-		return res, nil
-
 	} else {
-
-		var res TData
-
-		err := dec.Decode(&res)
+		err = dec.Decode(&res)
 		if err != nil {
 			return *new(TData), exerr.Wrap(err, "failed to decode single entity").Type("target-type", res).Build()
 		}
+	}
+
+	for _, hook := range c.unmarshalHooks {
+		res = hook(res)
+	}
 
 	return res, nil
-
-	}
 }
 
 func (c *Coll[TData]) decodeAll(ctx context.Context, cursor Cursorable) ([]TData, error) {
-	if c.customDecoder != nil {
 
 	res := make([]TData, 0, cursor.RemainingBatchLength())
 
+	if c.customDecoder != nil {
 		for cursor.Next(ctx) {
 			entry, err := (*c.customDecoder)(ctx, cursor)
 			if err != nil {
@@ -40,20 +42,48 @@ func (c *Coll[TData]) decodeAll(ctx context.Context, cursor Cursorable) ([]TData
 			}
 			res = append(res, entry)
 		}
-
-		return res, nil
-
 	} else {
-
-		res := make([]TData, 0, cursor.RemainingBatchLength())
-
 		err := cursor.All(ctx, &res)
 		if err != nil {
 			return nil, exerr.Wrap(err, "failed to batch-decode entity").Type("target-type", res).Build()
 		}
+	}
+
+	for i := 0; i < len(res); i++ {
+		for _, hook := range c.unmarshalHooks {
+			res[i] = hook(res[i])
+		}
+	}
 
 	return res, nil
 
 }
 
+func (c *Coll[TData]) decodeSingleOrRequery(ctx context.Context, dec Decodable) (TData, error) {
+	if c.extraModPipeline == nil {
+
+		// simple case, we can just decode the result and return it
+		return c.decodeSingle(ctx, dec)
+
+	} else {
+
+		// annyoing case, we have a extraModPipeline and need to re-query the document such that the extraModPipeline is applied...
+
+		type genDoc struct {
+			ID any `bson:"_id"`
+		}
+		var res genDoc
+		err := dec.Decode(&res)
+		if err != nil {
+			return *new(TData), exerr.Wrap(err, "failed to ID-decode entity").Build()
+		}
+
+		v, err := c.findOneInternal(ctx, bson.M{"_id": res.ID}, false)
+		if err != nil {
+			return *new(TData), exerr.Wrap(err, "failed to re-query entity").Any("_id", res.ID).Build()
+		}
+
+		return *v, nil
+
+	}
 }

wmo/pagination.go

@@ -1,87 +0,0 @@
-package wmo
-
-import (
-	"go.mongodb.org/mongo-driver/bson"
-	ct "gogs.mikescher.com/BlackForestBytes/goext/cursortoken"
-	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
-)
-
-func CreatePagination[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, error) {
-
-	cond := bson.A{}
-	sort := bson.D{}
-
-	valuePrimary, err := coll.getTokenValueAsMongoType(token.ValuePrimary, fieldPrimary)
-	if err != nil {
-		return nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
-	}
-
-	if sortPrimary == ct.SortASC {
-		// We sort ASC on <field> - so we want all entries newer ($gt) than the $primary
-		cond = append(cond, bson.M{fieldPrimary: bson.M{"$gt": valuePrimary}})
-		sort = append(sort, bson.E{Key: fieldPrimary, Value: +1})
-	} else if sortPrimary == ct.SortDESC {
-		// We sort DESC on <field> - so we want all entries older ($lt) than the $primary
-		cond = append(cond, bson.M{fieldPrimary: bson.M{"$lt": valuePrimary}})
-		sort = append(sort, bson.E{Key: fieldPrimary, Value: -1})
-	}
-
-	if fieldSecondary != nil && sortSecondary != nil && *fieldSecondary != fieldPrimary {
-
-		valueSecondary, err := coll.getTokenValueAsMongoType(token.ValueSecondary, *fieldSecondary)
-		if err != nil {
-			return nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
-		}
-
-		if *sortSecondary == ct.SortASC {
-
-			// the conflict-resolution condition, for entries with the _same_ <field> as the $primary we take the ones with a greater $secondary (= newer)
-			cond = append(cond, bson.M{"$and": bson.A{
-				bson.M{fieldPrimary: valuePrimary},
-				bson.M{*fieldSecondary: bson.M{"$gt": valueSecondary}},
-			}})
-
-			sort = append(sort, bson.E{Key: fieldPrimary, Value: +1})
-
-		} else if *sortSecondary == ct.SortDESC {
-
-			// the conflict-resolution condition, for entries with the _same_ <field> as the $primary we take the ones with a smaller $secondary (= older)
-			cond = append(cond, bson.M{"$and": bson.A{
-				bson.M{fieldPrimary: valuePrimary},
-				bson.M{*fieldSecondary: bson.M{"$lt": valueSecondary}},
-			}})
-
-			sort = append(sort, bson.E{Key: fieldPrimary, Value: -1})
-
-		}
-	}
-
-	pipeline := make([]bson.D, 0, 3)
-
-	if token.Mode == ct.CTMStart {
-
-		// no gt/lt condition
-
-	} else if token.Mode == ct.CTMNormal {
-
-		pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$or": cond}}})
-
-	} else if token.Mode == ct.CTMEnd {
-
-		// false
-		pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$expr": bson.M{"$eq": bson.A{"1", "0"}}}}})
-
-	} else {
-
-		return nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build()
-
-	}
-
-	pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}})
-
-	if pageSize != nil {
-		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}})
-	}
-
-	return pipeline, nil
-}

wmo/queryAggregate.go

@@ -5,9 +5,13 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 )
 
 func (c *Coll[TData]) Aggregate(ctx context.Context, pipeline mongo.Pipeline, opts ...*options.AggregateOptions) ([]TData, error) {
+
+	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+
 	cursor, err := c.coll.Aggregate(ctx, pipeline, opts...)
 	if err != nil {
 		return nil, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Any("options", opts).Str("collection", c.Name()).Build()
@@ -22,6 +26,9 @@ func (c *Coll[TData]) Aggregate(ctx context.Context, pipeline mongo.Pipeline, op
 }
 
 func (c *Coll[TData]) AggregateOneOpt(ctx context.Context, pipeline mongo.Pipeline, opts ...*options.AggregateOptions) (*TData, error) {
+
+	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+
 	cursor, err := c.coll.Aggregate(ctx, pipeline, opts...)
 	if err != nil {
 		return nil, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Any("options", opts).Str("collection", c.Name()).Build()
@@ -39,6 +46,9 @@ func (c *Coll[TData]) AggregateOneOpt(ctx context.Context, pipeline mongo.Pipeli
 }
 
 func (c *Coll[TData]) AggregateOne(ctx context.Context, pipeline mongo.Pipeline, opts ...*options.AggregateOptions) (TData, error) {
+
+	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+
 	cursor, err := c.coll.Aggregate(ctx, pipeline, opts...)
 	if err != nil {
 		return *new(TData), exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Any("options", opts).Str("collection", c.Name()).Build()

wmo/queryFind.go

@@ -2,69 +2,56 @@ package wmo
 
 import (
 	"context"
-	"errors"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 )
 
-func (c *Coll[TData]) FindOne(ctx context.Context, filter bson.M) (TData, error) {
-	mongoRes := c.coll.FindOne(ctx, filter)
-	if err := mongoRes.Err(); err != nil {
-		return *new(TData), exerr.Wrap(err, "mongo-query[find-one] failed").
-			Str("collection", c.Name()).
-			Any("filter", filter).
-			Build()
-	}
-
-	return c.decodeSingle(ctx, mongoRes)
-}
-
-func (c *Coll[TData]) FindOneOpt(ctx context.Context, filter bson.M) (*TData, error) {
-	mongoRes := c.coll.FindOne(ctx, filter)
-
-	res, err := c.decodeSingle(ctx, mongoRes)
-	if errors.Is(err, mongo.ErrNoDocuments) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, exerr.Wrap(err, "mongo-query[find-one-opt] failed").Any("filter", filter).Str("collection", c.Name()).Build()
-	}
-
-	return &res, nil
-}
-
-func (c *Coll[TData]) FindOneByID(ctx context.Context, id EntityID) (TData, error) {
-	mongoRes := c.coll.FindOne(ctx, bson.M{"_id": id})
-	if err := mongoRes.Err(); err != nil {
-		return *new(TData), exerr.Wrap(err, "mongo-query[find-one-by-id] failed").
-			Str("collection", c.Name()).
-			Id("id", id).
-			Build()
-	}
-
-	return c.decodeSingle(ctx, mongoRes)
-}
-
-func (c *Coll[TData]) FindOneOptByID(ctx context.Context, id EntityID) (*TData, error) {
-	mongoRes := c.coll.FindOne(ctx, bson.M{"_id": id})
-
-	res, err := c.decodeSingle(ctx, mongoRes)
-	if errors.Is(err, mongo.ErrNoDocuments) {
-		return nil, nil
-	}
-	if err != nil {
-		return nil, exerr.Wrap(err, "mongo-query[find-one-opt-by-id] failed").Id("id", id).Str("collection", c.Name()).Build()
-	}
-
-	return &res, nil
-}
-
 func (c *Coll[TData]) Find(ctx context.Context, filter bson.M, opts ...*options.FindOptions) ([]TData, error) {
-	cursor, err := c.coll.Find(ctx, filter, opts...)
+
+	pipeline := mongo.Pipeline{}
+	pipeline = append(pipeline, bson.D{{Key: "$match", Value: filter}})
+
+	for _, opt := range opts {
+		if opt != nil && opt.Sort != nil {
+			pipeline = append(pipeline, bson.D{{Key: "$sort", Value: opt.Sort}})
+		}
+	}
+
+	for _, opt := range opts {
+		if opt != nil && opt.Skip != nil {
+			pipeline = append(pipeline, bson.D{{Key: "$skip", Value: *opt.Skip}})
+		}
+	}
+
+	for _, opt := range opts {
+		if opt != nil && opt.Limit != nil {
+			pipeline = append(pipeline, bson.D{{Key: "$limit", Value: *opt.Limit}})
+		}
+	}
+
+	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+
+	for _, opt := range opts {
+		if opt != nil && opt.Projection != nil {
+			pipeline = append(pipeline, bson.D{{Key: "$project", Value: opt.Projection}})
+		}
+	}
+
+	convOpts := make([]*options.AggregateOptions, 0, len(opts))
+	for _, v := range opts {
+		vConv, err := convertFindOpt(v)
 		if err != nil {
-		return nil, exerr.Wrap(err, "mongo-query[find-one-opt] failed").Any("filter", filter).Any("opts", opts).Str("collection", c.Name()).Build()
+			return nil, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
+		}
+		convOpts = append(convOpts, vConv)
+	}
+
+	cursor, err := c.coll.Aggregate(ctx, pipeline, convOpts...)
+	if err != nil {
+		return nil, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
 	}
 
 	res, err := c.decodeAll(ctx, cursor)
@@ -74,3 +61,66 @@ func (c *Coll[TData]) Find(ctx context.Context, filter bson.M, opts ...*options.
 
 	return res, nil
 }
+
+// converts FindOptions to AggregateOptions
+func convertFindOpt(v *options.FindOptions) (*options.AggregateOptions, error) {
+	if v == nil {
+		return nil, nil
+	}
+
+	r := options.Aggregate()
+
+	if v.AllowDiskUse != nil {
+		r.SetAllowDiskUse(*v.AllowDiskUse)
+	}
+	if v.AllowPartialResults != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'AllowPartialResults' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.BatchSize != nil {
+		r.SetBatchSize(*v.BatchSize)
+	}
+	if v.Collation != nil {
+		r.SetCollation(v.Collation)
+	}
+	if v.Comment != nil {
+		r.SetComment(*v.Comment)
+	}
+	if v.CursorType != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'CursorType' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.Hint != nil {
+		r.SetHint(v.Hint)
+	}
+	if v.Max != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'Max' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.MaxAwaitTime != nil {
+		r.SetMaxAwaitTime(*v.MaxAwaitTime)
+	}
+	if v.MaxTime != nil {
+		r.SetMaxTime(*v.MaxTime)
+	}
+	if v.Min != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'Min' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.NoCursorTimeout != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'NoCursorTimeout' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.OplogReplay != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'OplogReplay' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.ReturnKey != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'ReturnKey' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.ShowRecordID != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'ShowRecordID' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.Snapshot != nil {
+		return nil, exerr.New(exerr.TypeMongoInvalidOpt, "Invalid option 'Snapshot' (cannot convert to AggregateOptions)").Build()
+	}
+	if v.Let != nil {
+		r.SetLet(v.Let)
+	}
+
+	return r, nil
+}

wmo/queryFindOne.go

@@ -0,0 +1,93 @@
+package wmo
+
+import (
+	"context"
+	"errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+)
+
+func (c *Coll[TData]) FindOne(ctx context.Context, filter bson.M) (TData, error) {
+	r, err := c.findOneInternal(ctx, filter, false)
+	if err != nil {
+		return *new(TData), exerr.Wrap(err, "mongo-query[find-one] failed").Str("collection", c.Name()).Build()
+	}
+
+	return *r, nil
+}
+
+func (c *Coll[TData]) FindOneOpt(ctx context.Context, filter bson.M) (*TData, error) {
+	r, err := c.findOneInternal(ctx, filter, true)
+	if err != nil {
+		return nil, exerr.Wrap(err, "mongo-query[find-one-opt] failed").Str("collection", c.Name()).Build()
+	}
+
+	return r, nil
+}
+
+func (c *Coll[TData]) FindOneByID(ctx context.Context, id EntityID) (TData, error) {
+	r, err := c.findOneInternal(ctx, bson.M{"_id": id}, false)
+	if err != nil {
+		return *new(TData), exerr.Wrap(err, "mongo-query[find-one-by-id] failed").Id("id", id).Str("collection", c.Name()).Build()
+	}
+
+	return *r, nil
+}
+
+func (c *Coll[TData]) FindOneOptByID(ctx context.Context, id EntityID) (*TData, error) {
+	r, err := c.findOneInternal(ctx, bson.M{"_id": id}, true)
+	if err != nil {
+		return nil, exerr.Wrap(err, "mongo-query[find-one-opt-by-id] failed").Id("id", id).Str("collection", c.Name()).Build()
+	}
+
+	return r, nil
+}
+
+func (c *Coll[TData]) findOneInternal(ctx context.Context, filter bson.M, allowNull bool) (*TData, error) {
+
+	if len(c.extraModPipeline) == 0 {
+
+		// simple case, use mongo FindOne
+
+		mongoRes := c.coll.FindOne(ctx, filter)
+
+		res, err := c.decodeSingle(ctx, mongoRes)
+		if allowNull && errors.Is(err, mongo.ErrNoDocuments) {
+			return nil, nil
+		}
+		if err != nil {
+			return nil, exerr.Wrap(err, "mongo-query[find-one] failed").Any("filter", filter).Str("collection", c.Name()).Build()
+		}
+
+		return &res, nil
+
+	} else {
+
+		// complex case, we one ore more additional pipeline stages, convert to aggregation
+
+		pipeline := mongo.Pipeline{}
+		pipeline = append(pipeline, bson.D{{Key: "$match", Value: filter}})
+		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: 1}})
+
+		pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+
+		cursor, err := c.coll.Aggregate(ctx, pipeline)
+		if err != nil {
+			return nil, exerr.Wrap(err, "mongo-aggregation [find-one] failed").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
+		}
+
+		if cursor.Next(ctx) {
+			v, err := c.decodeSingle(ctx, cursor)
+			if err != nil {
+				return nil, exerr.Wrap(err, "mongo-aggregation [find-one] failed to decode results").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
+			}
+			return &v, nil
+		} else if allowNull {
+			return nil, nil
+		} else {
+			return nil, exerr.Wrap(mongo.ErrNoDocuments, "mongo-aggregation [find-one] returned no documents").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
+		}
+	}
+}

wmo/queryInsert.go

@@ -14,9 +14,25 @@ func (c *Coll[TData]) InsertOne(ctx context.Context, valueIn TData) (TData, erro
 		return *new(TData), exerr.Wrap(err, "mongo-query[insert-one] failed").Str("collection", c.Name()).Build()
 	}
 
-	mongoRes := c.coll.FindOne(ctx, bson.M{"_id": insRes.InsertedID})
+	r, err := c.findOneInternal(ctx, bson.M{"_id": insRes.InsertedID}, false)
+	if err != nil {
+		return *new(TData), exerr.Wrap(err, "mongo-query[insert-one] failed").Str("collection", c.Name()).Build()
+	}
+	return *r, nil
+}
 
-	return c.decodeSingle(ctx, mongoRes)
+// InsertOneUnchecked behaves the same as InsertOne, but allows arbitrary data to be inserted (valueIn is any instead of TData)
+func (c *Coll[TData]) InsertOneUnchecked(ctx context.Context, valueIn any) (TData, error) {
+	insRes, err := c.coll.InsertOne(ctx, valueIn)
+	if err != nil {
+		return *new(TData), exerr.Wrap(err, "mongo-query[insert-one] failed").Str("collection", c.Name()).Build()
+	}
+
+	r, err := c.findOneInternal(ctx, bson.M{"_id": insRes.InsertedID}, false)
+	if err != nil {
+		return *new(TData), exerr.Wrap(err, "mongo-query[insert-one] failed").Str("collection", c.Name()).Build()
+	}
+	return *r, nil
 }
 
 func (c *Coll[TData]) InsertMany(ctx context.Context, valueIn []TData) (*mongo.InsertManyResult, error) {
@@ -27,3 +43,13 @@ func (c *Coll[TData]) InsertMany(ctx context.Context, valueIn []TData) (*mongo.I
 
 	return insRes, nil
 }
+
+// InsertManyUnchecked behaves the same as InsertOne, but allows arbitrary data to be inserted (valueIn is []any instead of []TData)
+func (c *Coll[TData]) InsertManyUnchecked(ctx context.Context, valueIn []any) (*mongo.InsertManyResult, error) {
+	insRes, err := c.coll.InsertMany(ctx, langext.ArrayToInterface(valueIn))
+	if err != nil {
+		return nil, exerr.Wrap(err, "mongo-query[insert-many] failed").Int("len(valueIn)", len(valueIn)).Str("collection", c.Name()).Build()
+	}
+
+	return insRes, nil
+}

wmo/queryList.go

@@ -34,7 +34,7 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		sortDirSecondary = nil
 	}
 
-	paginationPipeline, err := CreatePagination(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize)
+	paginationPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize)
 	if err != nil {
 		return nil, ct.CursorToken{}, exerr.
 			Wrap(err, "failed to create pagination").
@@ -50,6 +50,7 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 	}
 
 	pipeline = append(pipeline, paginationPipeline...)
+	pipeline = append(pipeline, c.extraModPipeline...)
 
 	cursor, err := c.coll.Aggregate(ctx, pipeline)
 	if err != nil {
@@ -91,11 +92,11 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 	return entities, nextToken, nil
 }
 
+func (c *Coll[TData]) Count(ctx context.Context, filter ct.RawFilter) (int64, error) {
 	type countRes struct {
 		Count int64 `bson:"c"`
 	}
 
-func (c *Coll[TData]) Count(ctx context.Context, filter ct.Filter) (int64, error) {
 	pipeline := filter.FilterQuery()
 
 	pipeline = append(pipeline, bson.D{{Key: "$count", Value: "c"}})
@@ -130,3 +131,83 @@ func (c *Coll[TData]) ListWithCount(ctx context.Context, filter ct.Filter, pageS
 	}
 	return data, token, count, nil
 }
+
+func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, error) {
+
+	cond := bson.A{}
+	sort := bson.D{}
+
+	valuePrimary, err := coll.getTokenValueAsMongoType(token.ValuePrimary, fieldPrimary)
+	if err != nil {
+		return nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
+	}
+
+	if sortPrimary == ct.SortASC {
+		// We sort ASC on <field> - so we want all entries newer ($gt) than the $primary
+		cond = append(cond, bson.M{fieldPrimary: bson.M{"$gt": valuePrimary}})
+		sort = append(sort, bson.E{Key: fieldPrimary, Value: +1})
+	} else if sortPrimary == ct.SortDESC {
+		// We sort DESC on <field> - so we want all entries older ($lt) than the $primary
+		cond = append(cond, bson.M{fieldPrimary: bson.M{"$lt": valuePrimary}})
+		sort = append(sort, bson.E{Key: fieldPrimary, Value: -1})
+	}
+
+	if fieldSecondary != nil && sortSecondary != nil && *fieldSecondary != fieldPrimary {
+
+		valueSecondary, err := coll.getTokenValueAsMongoType(token.ValueSecondary, *fieldSecondary)
+		if err != nil {
+			return nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
+		}
+
+		if *sortSecondary == ct.SortASC {
+
+			// the conflict-resolution condition, for entries with the _same_ <field> as the $primary we take the ones with a greater $secondary (= newer)
+			cond = append(cond, bson.M{"$and": bson.A{
+				bson.M{fieldPrimary: valuePrimary},
+				bson.M{*fieldSecondary: bson.M{"$gt": valueSecondary}},
+			}})
+
+			sort = append(sort, bson.E{Key: fieldPrimary, Value: +1})
+
+		} else if *sortSecondary == ct.SortDESC {
+
+			// the conflict-resolution condition, for entries with the _same_ <field> as the $primary we take the ones with a smaller $secondary (= older)
+			cond = append(cond, bson.M{"$and": bson.A{
+				bson.M{fieldPrimary: valuePrimary},
+				bson.M{*fieldSecondary: bson.M{"$lt": valueSecondary}},
+			}})
+
+			sort = append(sort, bson.E{Key: fieldPrimary, Value: -1})
+
+		}
+	}
+
+	pipeline := make([]bson.D, 0, 3)
+
+	if token.Mode == ct.CTMStart {
+
+		// no gt/lt condition
+
+	} else if token.Mode == ct.CTMNormal {
+
+		pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$or": cond}}})
+
+	} else if token.Mode == ct.CTMEnd {
+
+		// false
+		pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$expr": bson.M{"$eq": bson.A{"1", "0"}}}}})
+
+	} else {
+
+		return nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build()
+
+	}
+
+	pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}})
+
+	if pageSize != nil {
+		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}})
+	}
+
+	return pipeline, nil
+}

wmo/queryPaginate.go

@@ -0,0 +1,82 @@
+package wmo
+
+import (
+	"context"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	pag "gogs.mikescher.com/BlackForestBytes/goext/pagination"
+)
+
+func (c *Coll[TData]) Paginate(ctx context.Context, filter pag.Filter, page int, limit *int) ([]TData, pag.Pagination, error) {
+	type totalCountResult struct {
+		Count int `bson:"count"`
+	}
+
+	if page < 0 {
+		page = 1
+	}
+
+	pipelineSort := mongo.Pipeline{}
+	pipelineFilter := mongo.Pipeline{}
+	sort := bson.D{}
+
+	if filter != nil {
+		pipelineFilter = filter.FilterQuery()
+		sort = filter.Sort()
+	}
+
+	if len(sort) != 0 {
+		pipelineSort = append(pipelineSort, bson.D{{Key: "$sort", Value: sort}})
+	}
+
+	pipelinePaginate := mongo.Pipeline{}
+	if limit != nil {
+		pipelinePaginate = append(pipelinePaginate, bson.D{{Key: "$skip", Value: *limit * (page - 1)}})
+		pipelinePaginate = append(pipelinePaginate, bson.D{{Key: "$limit", Value: *limit}})
+	} else {
+		page = 1
+	}
+
+	pipelineCount := mongo.Pipeline{}
+	pipelineCount = append(pipelineCount, bson.D{{Key: "$count", Value: "count"}})
+
+	pipelineList := langext.ArrConcat(pipelineFilter, pipelineSort, pipelinePaginate, c.extraModPipeline)
+	pipelineTotalCount := langext.ArrConcat(pipelineFilter, pipelineCount)
+
+	cursorList, err := c.coll.Aggregate(ctx, pipelineList)
+	if err != nil {
+		return nil, pag.Pagination{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipelineList).Str("collection", c.Name()).Build()
+	}
+
+	entities, err := c.decodeAll(ctx, cursorList)
+	if err != nil {
+		return nil, pag.Pagination{}, exerr.Wrap(err, "failed to all-decode entities").Build()
+	}
+
+	cursorTotalCount, err := c.coll.Aggregate(ctx, pipelineTotalCount)
+	if err != nil {
+		return nil, pag.Pagination{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipelineTotalCount).Str("collection", c.Name()).Build()
+	}
+
+	var tcRes totalCountResult
+	if cursorTotalCount.Next(ctx) {
+		err = cursorTotalCount.Decode(&tcRes)
+		if err != nil {
+			return nil, pag.Pagination{}, exerr.Wrap(err, "failed to decode mongo-aggregation $count result").Any("pipeline", pipelineTotalCount).Str("collection", c.Name()).Build()
+		}
+	} else {
+		tcRes.Count = 0 // no entries in DB
+	}
+
+	paginationObj := pag.Pagination{
+		Page:             page,
+		Limit:            langext.Coalesce(limit, tcRes.Count),
+		TotalPages:       pag.CalcPaginationTotalPages(tcRes.Count, langext.Coalesce(limit, tcRes.Count)),
+		TotalItems:       tcRes.Count,
+		CurrentPageCount: len(entities),
+	}
+
+	return entities, paginationObj, nil
+}

wmo/queryUpdate.go

@@ -18,7 +18,7 @@ func (c *Coll[TData]) FindOneAndUpdate(ctx context.Context, filterQuery bson.M,
 			Build()
 	}
 
-	return c.decodeSingle(ctx, mongoRes)
+	return c.decodeSingleOrRequery(ctx, mongoRes)
 }
 
 func (c *Coll[TData]) UpdateOne(ctx context.Context, filterQuery bson.M, updateQuery bson.M) error {
@@ -73,7 +73,7 @@ func (c *Coll[TData]) ReplaceOne(ctx context.Context, filterQuery bson.M, value
 }
 
 func (c *Coll[TData]) FindOneAndReplace(ctx context.Context, filterQuery bson.M, value TData) (TData, error) {
-	mongoRes := c.coll.FindOneAndUpdate(ctx, filterQuery, bson.M{"$set": value}, options.FindOneAndUpdate().SetReturnDocument(options.After))
+	mongoRes := c.coll.FindOneAndReplace(ctx, filterQuery, value, options.FindOneAndReplace().SetReturnDocument(options.After))
 	if err := mongoRes.Err(); err != nil {
 		return *new(TData), exerr.Wrap(err, "mongo-query[find-one-and-update] failed").
 			Str("collection", c.Name()).
@@ -81,5 +81,5 @@ func (c *Coll[TData]) FindOneAndReplace(ctx context.Context, filterQuery bson.M,
 			Build()
 	}
 
-	return c.decodeSingle(ctx, mongoRes)
+	return c.decodeSingleOrRequery(ctx, mongoRes)
 }