v0.0.402 add PackageName() and TypeName() to enums_codegen

.gitea/workflows/Dockerfile_tests

@@ -1,9 +0,0 @@
-FROM golang:latest
-
-RUN apt install -y make curl python3 && go install gotest.tools/gotestsum@latest
-
-COPY . /source
-
-WORKDIR /source
-
-CMD ["make", "test"]

.gitea/workflows/tests.yml

@@ -6,25 +6,50 @@
 name: Build Docker and Deploy
 run-name: Build & Deploy ${{ gitea.ref }} on ${{ gitea.actor }}
 
-on: [push]
+on:
+  push:
+    branches:
+      - '*'
+      - '**'
+
 
 jobs:
   run_tests:
-    name: Run goext test-suite
+    name: Run goext test-suite  
     runs-on: bfb-cicd-latest
     steps:
 
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Build test docker
-        id: build_docker
-        run: echo "DOCKER_IMG_ID=$(docker build -q . -f .gitea/workflows/Dockerfile_tests || echo __err_build__)" >> $GITHUB_OUTPUT
+      - name: Setup go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: '${{ gitea.workspace }}/go.mod'
+
+      - name: Setup packages
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: curl python3
+          version: 1.0
+
+      - name: go version
+        run: go version
 
       - name: Run tests
-        run: docker run --rm "${{ steps.build_docker.outputs.DOCKER_IMG_ID }}"
+        run: cd "${{ gitea.workspace }}" && make test
 
-      - name: Cleanup
-        if: always()
-        run: docker image rm "${{ steps.build_docker.outputs.DOCKER_IMG_ID }}"
+      - name: Send failure mail
+        if: failure()
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.fastmail.com
+          server_port: 465
+          secure: true
+          username: ${{secrets.MAIL_USERNAME}}
+          password: ${{secrets.MAIL_PASSWORD}}
+          subject: Pipeline on '${{ gitea.repository }}' failed
+          to: ${{ steps.commiter_info.outputs.MAIL }}
+          from: Gitea Actions <gitea_actions@blackforestbytes.de>
+          body: "Go to https://gogs.blackforestbytes.com/${{ gitea.repository }}/actions"
 

.idea/.gitignore

@@ -6,3 +6,5 @@
 # Datasource local storage ignored files
 /dataSources/
 /dataSources.local.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

.idea/golinter.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoLinterSettings">
+    <option name="checkGoLinterExe" value="false" />
+  </component>
+</project>

.idea/sqldialects.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/sq/sq_test.go" dialect="SQLite" />
+  </component>
+</project>

Makefile

@@ -7,5 +7,11 @@ test:
 	which gotestsum || go install gotest.tools/gotestsum@latest
 	gotestsum --format "testname" -- -tags="timetzdata sqlite_fts5 sqlite_foreign_keys" "./..."
 
+test-in-docker:
+	tag="goext_temp_test_image:$(shell uuidgen | tr -d '-')";        \
+	docker build --tag $$tag . -f .gitea/workflows/Dockerfile_tests; \
+	docker run --rm $$tag;                                           \
+	docker rmi $$tag
+
 version:
 	_data/version.sh

README.md

@@ -10,32 +10,36 @@ Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
 
 ### Packages:
 
-| Name         | Maintainer | Description                                                                                                   |
-|--------------|------------|---------------------------------------------------------------------------------------------------------------|
-| langext      | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
-| mathext      | Mike       | Utility/Helper functions for math                                                                             |
-| cryptext     | Mike       | Utility/Helper functions for encryption                                                                       |
-| syncext      | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
-| dataext      | Mike       | Various useful data structures                                                                                |
-| zipext       | Mike       | Utility for zip/gzip/tar etc                                                                                  |
-| reflectext   | Mike       | Utility for golagn reflection                                                                                 |
-|              |            |                                                                                                               |
-| mongoext     | Mike       | Utility/Helper functions for mongodb                                                                          |
-| cursortoken  | Mike       | MongoDB cursortoken implementation                                                                            |
-|              |            |                                                                                                               |
-| totpext      | Mike       | Implementation of TOTP (2-Factor-Auth)                                                                        |
-| termext      | Mike       | Utilities for terminals (mostly color output)                                                                 |
-| confext      | Mike       | Parses environment configuration into structs                                                                 |
-| cmdext       | Mike       | Runner for external commands/processes                                                                        |
-|              |            |                                                                                                               |
-| sq           | Mike       | Utility functions for sql based databases                                                                     |
-| tst          | Mike       | Utility functions for unit tests                                                                              |
-|              |            |                                                                                                               |
-| rfctime      | Mike       | Classes for time seriallization, with different marshallign method for mongo and json                         |
-| gojson       | Mike       | Same interface for marshalling/unmarshalling as go/json, except with proper serialization of null arrays/maps |
-|              |            |                                                                                                               |
-| bfcodegen    | Mike       | Various codegen tools (run via go generate)                                                                   |
-|              |            |                                                                                                               |
-| rext         | Mike       | Regex Wrapper, wraps regexp with a better interface                                                           |
-| wmo          | Mike       | Mongo Wrapper, wraps mongodb with a better interface                                                          |
-|              |            |                                                                                                               |
+| Name        | Maintainer | Description                                                                                                   |
+|-------------|------------|---------------------------------------------------------------------------------------------------------------|
+| langext     | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
+| mathext     | Mike       | Utility/Helper functions for math                                                                             |
+| cryptext    | Mike       | Utility/Helper functions for encryption                                                                       |
+| syncext     | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
+| dataext     | Mike       | Various useful data structures                                                                                |
+| zipext      | Mike       | Utility for zip/gzip/tar etc                                                                                  |
+| reflectext  | Mike       | Utility for golang reflection                                                                                 |
+| fsext       | Mike       | Utility for filesytem access                                                                                  |
+|             |            |                                                                                                               |
+| mongoext    | Mike       | Utility/Helper functions for mongodb                                                                          |
+| cursortoken | Mike       | MongoDB cursortoken implementation                                                                            |
+| pagination  | Mike       | Pagination implementation                                                                                     |
+|             |            |                                                                                                               |
+| totpext     | Mike       | Implementation of TOTP (2-Factor-Auth)                                                                        |
+| termext     | Mike       | Utilities for terminals (mostly color output)                                                                 |
+| confext     | Mike       | Parses environment configuration into structs                                                                 |
+| cmdext      | Mike       | Runner for external commands/processes                                                                        |
+|             |            |                                                                                                               |
+| sq          | Mike       | Utility functions for sql based databases (primarily sqlite)                                                  |
+| tst         | Mike       | Utility functions for unit tests                                                                              |
+|             |            |                                                                                                               |
+| rfctime     | Mike       | Classes for time seriallization, with different marshallign method for mongo and json                         |
+| gojson      | Mike       | Same interface for marshalling/unmarshalling as go/json, except with proper serialization of null arrays/maps |
+|             |            |                                                                                                               |
+| bfcodegen   | Mike       | Various codegen tools (run via go generate)                                                                   |
+|             |            |                                                                                                               |
+| rext        | Mike       | Regex Wrapper, wraps regexp with a better interface                                                           |
+| wmo         | Mike       | Mongo Wrapper, wraps mongodb with a better interface                                                          |
+|             |            |                                                                                                               |
+| scn         | Mike       | SimpleCloudNotifier                                                                                           |
+|             |            |                                                                                                               |

TODO.md

@@ -2,6 +2,8 @@
 
  - cronext
 
- - rfctime.DateOnly
  - rfctime.HMSTimeOnly
- - rfctime.NanoTimeOnly
+ - rfctime.NanoTimeOnly
+
+ - remove sqlx dependency from sq  (unmaintained, and mostly superseeded by our own stuff?)
+ - Move DBLogger and DBPreprocessor to sq

bfcodegen/csid-generate.go

@@ -0,0 +1,197 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"go/format"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type CSIDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+	Prefix       string
+}
+
+type CSIDGenOptions struct {
+	DebugOutput *bool
+}
+
+var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
+
+var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed csid-generate.template
+var templateCSIDGenerateText string
+
+func GenerateCharsetIDSpecs(sourceDir string, destFile string, opt CSIDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexCSIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[CSIDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[CSIDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
+		if err != nil {
+			return err
+		}
+
+		if debugOutput {
+			fmt.Printf("\n")
+		}
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	fdata, err := format.Source([]byte(fmtCSIDOutput(newChecksum, allIDs, pkgname)))
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(destFile, fdata, 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processCSIDFile(basedir string, fn string, debugOutput bool) ([]CSIDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexCSIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexCSIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := CSIDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+				Prefix:       match.GroupByName("prefix").Value(),
+			}
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtCSIDOutput(cs string, ids []CSIDDef, pkgname string) string {
+	templ := template.Must(template.New("csid-generate").Parse(templateCSIDGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/csid-generate.template

@@ -0,0 +1,190 @@
+// Code generated by csid-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "crypto/rand"
+import "crypto/sha256"
+import "fmt"
+import "github.com/go-playground/validator/v10"
+import "github.com/rs/zerolog/log"
+import "gogs.mikescher.com/BlackForestBytes/goext/exerr"
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+import "gogs.mikescher.com/BlackForestBytes/goext/rext"
+import "math/big"
+import "reflect"
+import "regexp"
+import "strings"
+
+const ChecksumCharsetIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+const idlen = 24
+
+const checklen = 1
+
+const idCharset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+const idCharsetLen = len(idCharset)
+
+var charSetReverseMap = generateCharsetMap()
+
+const ({{range .IDs}}
+	prefix{{.Name}} = "{{.Prefix}}" {{end}}
+)
+
+var ({{range .IDs}}
+	regex{{.Name}} = generateRegex(prefix{{.Name}}) {{end}}
+)
+
+func generateRegex(prefix string) rext.Regex {
+	return rext.W(regexp.MustCompile(fmt.Sprintf("^%s[%s]{%d}[%s]{%d}$", prefix, idCharset, idlen-len(prefix)-checklen, idCharset, checklen)))
+}
+
+func generateCharsetMap() []int {
+	result := make([]int, 128)
+	for i := 0; i < len(result); i++ {
+		result[i] = -1
+	}
+	for idx, chr := range idCharset {
+		result[int(chr)] = idx
+	}
+	return result
+}
+
+func generateID(prefix string) string {
+	k := ""
+	csMax := big.NewInt(int64(idCharsetLen))
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		v, err := rand.Int(rand.Reader, csMax)
+		if err != nil {
+			panic(err)
+		}
+		v64 := v.Int64()
+		k += string(idCharset[v64])
+		checksum = (checksum + int(v64)) % (idCharsetLen)
+	}
+	checkstr := string(idCharset[checksum%idCharsetLen])
+	return prefix + k + checkstr
+}
+
+func generateIDFromSeed(prefix string, seed string) string {
+	h := sha256.New()
+
+	iddata := ""
+	for len(iddata) < idlen-len(prefix)-checklen {
+		h.Write([]byte(seed))
+		bs := h.Sum(nil)
+		iddata += langext.NewAnyBaseConverter(idCharset).Encode(bs)
+	}
+
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		ichr := int(iddata[i])
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	return prefix + iddata[:(idlen-len(prefix)-checklen)] + checkstr
+}
+
+func validateID(prefix string, value string) error {
+	if len(value) != idlen {
+		return exerr.New(exerr.TypeInvalidCSID, "id has the wrong length").Str("value", value).Build()
+	}
+
+	if !strings.HasPrefix(value, prefix) {
+		return exerr.New(exerr.TypeInvalidCSID, "id is missing the correct prefix").Str("value", value).Str("prefix", prefix).Build()
+	}
+
+	checksum := 0
+	for i := len(prefix); i < len(value)-checklen; i++ {
+		ichr := int(value[i])
+		if ichr < 0 || ichr >= len(charSetReverseMap) || charSetReverseMap[ichr] == -1 {
+			return exerr.New(exerr.TypeInvalidCSID, "id contains invalid characters").Str("value", value).Build()
+		}
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	if !strings.HasSuffix(value, checkstr) {
+		return exerr.New(exerr.TypeInvalidCSID, "id checkstring is invalid").Str("value", value).Str("checkstr", checkstr).Build()
+	}
+
+	return nil
+}
+
+func getRawData(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[len(prefix) : idlen-checklen]
+}
+
+func getCheckString(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[idlen-checklen:]
+}
+
+func ValidateEntityID(vfl validator.FieldLevel) bool {
+	if !vfl.Field().CanInterface() {
+		log.Error().Msgf("Failed to validate EntityID (cannot interface ?!?)")
+		return false
+	}
+
+	ifvalue := vfl.Field().Interface()
+
+	if value1, ok := ifvalue.(EntityID); ok {
+
+		if vfl.Field().Type().Kind() == reflect.Pointer && langext.IsNil(value1) {
+			return true
+		}
+
+		if err := value1.Valid(); err != nil {
+			log.Debug().Msgf("Failed to validate EntityID '%s' (%s)", value1.String(), err.Error())
+			return false
+		} else {
+			return true
+		}
+
+	} else {
+		log.Error().Msgf("Failed to validate EntityID (wrong type: %T)", ifvalue)
+		return false
+	}
+}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(generateID(prefix{{.Name}}))
+}
+
+func (id {{.Name}}) Valid() error {
+	return validateID(prefix{{.Name}}, string(id))
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) Prefix() string {
+	return prefix{{.Name}}
+}
+
+func (id {{.Name}}) Raw() string {
+	return getRawData(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) CheckString() string {
+	return getCheckString(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) Regex() rext.Regex {
+	return regex{{.Name}}
+}
+
+{{end}}

bfcodegen/csid-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var CSIDExampleModels1 []byte
+
+func TestGenerateCSIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, CSIDExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/csid_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/enum-generate.go

@@ -1,10 +1,13 @@
 package bfcodegen
 
 import (
+	"bytes"
+	_ "embed"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"go/format"
 	"gogs.mikescher.com/BlackForestBytes/goext"
-	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
 	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rext"
@@ -12,15 +15,18 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"strings"
-	"time"
+	"text/template"
 )
 
 type EnumDefVal struct {
 	VarName     string
 	Value       string
 	Description *string
+	Data        *map[string]any
+	RawComment  *string
 }
 
 type EnumDef struct {
@@ -31,20 +37,23 @@ type EnumDef struct {
 	Values       []EnumDefVal
 }
 
+type EnumGenOptions struct {
+	DebugOutput *bool
+	GoFormat    *bool
+}
+
 var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
 
-var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:]+"|[0-9]+))\s*(//(?P<descr>.*))?.*$`))
+var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:\s\-.]+"|[0-9]+))\s*(//(?P<comm>.*))?.*$`))
 
 var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
 
-func GenerateEnumSpecs(sourceDir string, destFile string) error {
+//go:embed enum-generate.template
+var templateEnumGenerateText string
 
-	files, err := os.ReadDir(sourceDir)
-	if err != nil {
-		return err
-	}
+func GenerateEnumSpecs(sourceDir string, destFile string, opt EnumGenOptions) error {
 
 	oldChecksum := "N/A"
 	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
@@ -57,6 +66,30 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		}
 	}
 
+	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, langext.Coalesce(opt.GoFormat, true), langext.Coalesce(opt.DebugOutput, false))
+	if err != nil {
+		return err
+	}
+
+	if !changed {
+		return nil
+	}
+
+	err = os.WriteFile(destFile, []byte(gocode), 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool, debugOutput bool) (string, string, bool, error) {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return "", "", false, err
+	}
+
 	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
 	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
 	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
@@ -66,7 +99,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	for _, f := range files {
 		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
 		if err != nil {
-			return err
+			return "", "", false, err
 		}
 		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
 	}
@@ -77,7 +110,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		fmt.Printf("[EnumGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
 	} else {
 		fmt.Printf("[EnumGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
-		return nil
+		return "", oldChecksum, false, nil
 	}
 
 	allEnums := make([]EnumDef, 0)
@@ -85,13 +118,18 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()))
-		if err != nil {
-			return err
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
 		}
 
-		fmt.Printf("\n")
+		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
+		if err != nil {
+			return "", "", false, err
+		}
+
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allEnums = append(allEnums, fileEnums...)
 
@@ -101,32 +139,24 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	}
 
 	if pkgname == "" {
-		return errors.New("no package name found in any file")
+		return "", "", false, errors.New("no package name found in any file")
 	}
 
-	err = os.WriteFile(destFile, []byte(fmtEnumOutput(newChecksum, allEnums, pkgname)), 0o755)
+	rdata := fmtEnumOutput(newChecksum, allEnums, pkgname)
+
+	if !gofmt {
+		return rdata, newChecksum, true, nil
+	}
+
+	fdata, err := format.Source([]byte(rdata))
 	if err != nil {
-		return err
+		return "", "", false, err
 	}
 
-	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
-	if err != nil {
-		return err
-	}
-
-	if res.CommandTimedOut {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt timed out")
-	}
-	if res.ExitCode != 0 {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt did not succeed")
-	}
-
-	return nil
+	return string(fdata), newChecksum, true, nil
 }
 
-func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
+func processEnumFile(basedir string, fn string, debugOutput bool) ([]EnumDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -170,15 +200,42 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 				Values:       make([]EnumDefVal, 0),
 			}
 			enums = append(enums, def)
-			fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+
+			if debugOutput {
+				fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+			}
 		}
 
 		if match, ok := rexEnumValueDef.MatchFirst(line); ok {
 			typename := match.GroupByName("type").Value()
+
+			comment := match.GroupByNameOrEmpty("comm").ValueOrNil()
+			var descr *string = nil
+			var data *map[string]any = nil
+			if comment != nil {
+				comment = langext.Ptr(strings.TrimSpace(*comment))
+				if strings.HasPrefix(*comment, "{") {
+					if v, ok := tryParseDataComment(*comment); ok {
+						data = &v
+						if anyDataDescr, ok := v["description"]; ok {
+							if dataDescr, ok := anyDataDescr.(string); ok {
+								descr = &dataDescr
+							}
+						}
+					} else {
+						descr = comment
+					}
+				} else {
+					descr = comment
+				}
+			}
+
 			def := EnumDefVal{
 				VarName:     match.GroupByName("name").Value(),
 				Value:       match.GroupByName("value").Value(),
-				Description: match.GroupByNameOrEmpty("descr").ValueOrNil(),
+				RawComment:  comment,
+				Description: descr,
+				Data:        data,
 			}
 
 			found := false
@@ -186,16 +243,21 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 				if v.EnumTypeName == typename {
 					enums[i].Values = append(enums[i].Values, def)
 					found = true
-					if def.Description != nil {
-						fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
-					} else {
-						fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+
+					if debugOutput {
+						if def.Description != nil {
+							fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
+						} else {
+							fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+						}
 					}
 					break
 				}
 			}
 			if !found {
-				fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				if debugOutput {
+					fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				}
 			}
 		}
 	}
@@ -203,134 +265,109 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 	return enums, pkgname, nil
 }
 
-func fmtEnumOutput(cs string, enums []EnumDef, pkgname string) string {
-	str := "// Code generated by enum-generate.go DO NOT EDIT.\n"
-	str += "\n"
-	str += "package " + pkgname + "\n"
-	str += "\n"
+func tryParseDataComment(s string) (map[string]any, bool) {
 
-	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/langext\"" + "\n"
-	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/enums\"" + "\n"
-	str += "\n"
-
-	str += "const ChecksumEnumGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
-	str += "\n"
-
-	for _, enumdef := range enums {
-
-		hasDescr := langext.ArrAll(enumdef.Values, func(val EnumDefVal) bool { return val.Description != nil })
-		hasStr := enumdef.Type == "string"
-
-		str += "// ================================ " + enumdef.EnumTypeName + " ================================" + "\n"
-		str += "//" + "\n"
-		str += "// File:       " + enumdef.FileRelative + "\n"
-		str += "// StringEnum: " + langext.Conditional(hasStr, "true", "false") + "\n"
-		str += "// DescrEnum:  " + langext.Conditional(hasDescr, "true", "false") + "\n"
-		str += "//" + "\n"
-		str += "" + "\n"
-
-		str += "var __" + enumdef.EnumTypeName + "Values = []" + enumdef.EnumTypeName + "{" + "\n"
-		for _, v := range enumdef.Values {
-			str += "    " + v.VarName + "," + "\n"
-		}
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		if hasDescr {
-			str += "var __" + enumdef.EnumTypeName + "Descriptions = map[" + enumdef.EnumTypeName + "]string{" + "\n"
-			for _, v := range enumdef.Values {
-				str += "    " + v.VarName + ": \"" + strings.TrimSpace(*v.Description) + "\"," + "\n"
-			}
-			str += "}" + "\n"
-			str += "" + "\n"
-		}
-
-		str += "var __" + enumdef.EnumTypeName + "Varnames = map[" + enumdef.EnumTypeName + "]string{" + "\n"
-		for _, v := range enumdef.Values {
-			str += "    " + v.VarName + ": \"" + v.VarName + "\"," + "\n"
-		}
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") Valid() bool {" + "\n"
-		str += "    return langext.InArray(e, __" + enumdef.EnumTypeName + "Values)" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") Values() []" + enumdef.EnumTypeName + " {" + "\n"
-		str += "    return __" + enumdef.EnumTypeName + "Values" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") ValuesAny() []any {" + "\n"
-		str += "    return langext.ArrCastToAny(__" + enumdef.EnumTypeName + "Values)" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") ValuesMeta() []enums.EnumMetaValue {" + "\n"
-		str += "    return " + enumdef.EnumTypeName + "ValuesMeta()"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		if hasStr {
-			str += "func (e " + enumdef.EnumTypeName + ") String() string {" + "\n"
-			str += "    return string(e)" + "\n"
-			str += "}" + "\n"
-			str += "" + "\n"
-		}
-
-		if hasDescr {
-			str += "func (e " + enumdef.EnumTypeName + ") Description() string {" + "\n"
-			str += "    if d, ok := __" + enumdef.EnumTypeName + "Descriptions[e]; ok {" + "\n"
-			str += "        return d" + "\n"
-			str += "    }" + "\n"
-			str += "    return \"\"" + "\n"
-			str += "}" + "\n"
-			str += "" + "\n"
-		}
-
-		str += "func (e " + enumdef.EnumTypeName + ") VarName() string {" + "\n"
-		str += "    if d, ok := __" + enumdef.EnumTypeName + "Varnames[e]; ok {" + "\n"
-		str += "        return d" + "\n"
-		str += "    }" + "\n"
-		str += "    return \"\"" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func (e " + enumdef.EnumTypeName + ") Meta() enums.EnumMetaValue {" + "\n"
-		if hasDescr {
-			str += "    return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())}"
-		} else {
-			str += "    return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil}"
-		}
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func Parse" + enumdef.EnumTypeName + "(vv string) (" + enumdef.EnumTypeName + ", bool) {" + "\n"
-		str += "    for _, ev := range __" + enumdef.EnumTypeName + "Values {" + "\n"
-		str += "        if string(ev) == vv {" + "\n"
-		str += "            return ev, true" + "\n"
-		str += "        }" + "\n"
-		str += "    }" + "\n"
-		str += "    return \"\", false" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func " + enumdef.EnumTypeName + "Values() []" + enumdef.EnumTypeName + " {" + "\n"
-		str += "    return __" + enumdef.EnumTypeName + "Values" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
-
-		str += "func " + enumdef.EnumTypeName + "ValuesMeta() []enums.EnumMetaValue {" + "\n"
-		str += "    return []enums.EnumMetaValue{" + "\n"
-		for _, v := range enumdef.Values {
-			str += "        " + v.VarName + ".Meta(),\n"
-		}
-		str += "    }" + "\n"
-		str += "}" + "\n"
-		str += "" + "\n"
+	r := make(map[string]any)
 
+	err := json.Unmarshal([]byte(s), &r)
+	if err != nil {
+		return nil, false
 	}
 
-	return str
+	for _, v := range r {
+
+		rv := reflect.ValueOf(v)
+
+		if rv.Kind() == reflect.Ptr && rv.IsNil() {
+			continue
+		}
+		if rv.Kind() == reflect.Bool {
+			continue
+		}
+		if rv.Kind() == reflect.String {
+			continue
+		}
+		if rv.Kind() == reflect.Int64 {
+			continue
+		}
+		if rv.Kind() == reflect.Float64 {
+			continue
+		}
+
+		return nil, false
+	}
+
+	return r, true
+}
+
+func fmtEnumOutput(cs string, enums []EnumDef, pkgname string) string {
+
+	templ := template.New("enum-generate")
+
+	templ = templ.Funcs(template.FuncMap{
+		"boolToStr": func(b bool) string { return langext.Conditional(b, "true", "false") },
+		"deref":     func(v *string) string { return *v },
+		"trimSpace": func(str string) string { return strings.TrimSpace(str) },
+		"hasStr":    func(v EnumDef) bool { return v.Type == "string" },
+		"hasDescr": func(v EnumDef) bool {
+			return langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Description != nil })
+		},
+		"hasData": func(v EnumDef) bool {
+			return len(v.Values) > 0 && langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Data != nil })
+		},
+		"gostr": func(v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"goobj": func(name string, v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"godatakey": func(v string) string {
+			return strings.ToUpper(v[0:1]) + v[1:]
+		},
+		"godatavalue": func(v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"godatatype": func(v any) string {
+			return fmt.Sprintf("%T", v)
+		},
+		"mapindex": func(v map[string]any, k string) any {
+			return v[k]
+		},
+		"generalDataKeys": func(v EnumDef) map[string]string {
+			r0 := make(map[string]int)
+
+			for _, eval := range v.Values {
+				for k := range *eval.Data {
+					if ctr, ok := r0[k]; ok {
+						r0[k] = ctr + 1
+					} else {
+						r0[k] = 1
+					}
+				}
+			}
+
+			r1 := langext.MapToArr(r0)
+			r2 := langext.ArrFilter(r1, func(p langext.MapEntry[string, int]) bool { return p.Value == len(v.Values) })
+			r3 := langext.ArrMap(r2, func(p langext.MapEntry[string, int]) string { return p.Key })
+			r4 := langext.ArrToKVMap(r3, func(p string) string { return p }, func(p string) string { return fmt.Sprintf("%T", (*v.Values[0].Data)[p]) })
+
+			return r4
+		},
+	})
+
+	templ = template.Must(templ.Parse(templateEnumGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"Enums":        enums,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
 }

bfcodegen/enum-generate.template

@@ -0,0 +1,155 @@
+// Code generated by enum-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+import "gogs.mikescher.com/BlackForestBytes/goext/enums"
+
+const ChecksumEnumGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{ $pkgname  := .PkgName }}
+
+{{range .Enums}}
+
+{{ $hasStr   := ( . | hasStr   ) }}
+{{ $hasDescr := ( . | hasDescr ) }}
+{{ $hasData  := ( . | hasData  ) }}
+
+// ================================ {{.EnumTypeName}} ================================
+//
+// File:       {{.FileRelative}}
+// StringEnum: {{$hasStr   | boolToStr}}
+// DescrEnum:  {{$hasDescr | boolToStr}}
+// DataEnum:   {{$hasData  | boolToStr}}
+//
+
+{{ $typename := .EnumTypeName }}
+{{ $enumdef  := . }}
+
+var __{{.EnumTypeName}}Values = []{{.EnumTypeName}}{ {{range .Values}}
+	{{.VarName}}, {{end}}
+}
+
+{{if $hasDescr}}
+var __{{.EnumTypeName}}Descriptions = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: {{.Description | deref | trimSpace | gostr}}, {{end}}
+}
+{{end}}
+
+{{if $hasData}}
+type {{ .EnumTypeName }}Data struct { {{ range $datakey, $datatype := ($enumdef | generalDataKeys)  }}
+    {{ $datakey | godatakey }} {{ $datatype }} `json:"{{ $datakey }}"` {{ end }}
+}
+
+var __{{.EnumTypeName}}Data = map[{{.EnumTypeName}}]{{.EnumTypeName}}Data{ {{range .Values}} {{ $enumvalue := . }}
+	{{.VarName}}: {{ $typename }}Data{ {{ range $datakey, $datatype := $enumdef | generalDataKeys  }}
+	    {{ $datakey | godatakey }}: {{ (mapindex $enumvalue.Data $datakey) | godatavalue }}, {{ end }}
+	}, {{end}}
+}
+{{end}}
+
+var __{{.EnumTypeName}}Varnames = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: "{{.VarName}}", {{end}}
+}
+
+func (e {{.EnumTypeName}}) Valid() bool {
+	return langext.InArray(e, __{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func (e {{.EnumTypeName}}) ValuesAny() []any {
+	return langext.ArrCastToAny(__{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) ValuesMeta() []enums.EnumMetaValue {
+	return {{.EnumTypeName}}ValuesMeta()
+}
+
+{{if $hasStr}}
+func (e {{.EnumTypeName}}) String() string {
+	return string(e)
+}
+{{end}}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) Description() string {
+	if d, ok := __{{.EnumTypeName}}Descriptions[e]; ok {
+		return d
+	}
+	return ""
+}
+{{end}}
+
+{{if $hasData}}
+func (e {{.EnumTypeName}}) Data() {{.EnumTypeName}}Data {
+	if d, ok := __{{.EnumTypeName}}Data[e]; ok {
+		return d
+	}
+	return {{.EnumTypeName}}Data{}
+}
+{{end}}
+
+func (e {{.EnumTypeName}}) VarName() string {
+	if d, ok := __{{.EnumTypeName}}Varnames[e]; ok {
+		return d
+	}
+	return ""
+}
+
+func (e {{.EnumTypeName}}) TypeName() string {
+	return "{{$typename}}"
+}
+
+func (e {{.EnumTypeName}}) PackageName() string {
+	return "{{$pkgname }}"
+}
+
+func (e {{.EnumTypeName}}) Meta() enums.EnumMetaValue {
+    {{if $hasDescr}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())} {{else}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil} {{end}}
+}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) DescriptionMeta() enums.EnumDescriptionMetaValue {
+    return enums.EnumDescriptionMetaValue{VarName: e.VarName(), Value: e, Description: e.Description()}
+}
+{{end}}
+
+func Parse{{.EnumTypeName}}(vv string) ({{.EnumTypeName}}, bool) {
+	for _, ev := range __{{.EnumTypeName}}Values {
+		if string(ev) == vv {
+			return ev, true
+		}
+	}
+	return "", false
+}
+
+func {{.EnumTypeName}}Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func {{.EnumTypeName}}ValuesMeta() []enums.EnumMetaValue {
+	return []enums.EnumMetaValue{ {{range .Values}}
+            {{.VarName}}.Meta(), {{end}}
+	}
+}
+
+{{if $hasDescr}}
+func {{.EnumTypeName}}ValuesDescriptionMeta() []enums.EnumDescriptionMetaValue {
+	return []enums.EnumDescriptionMetaValue{ {{range .Values}}
+            {{.VarName}}.DescriptionMeta(), {{end}}
+	}
+}
+{{end}}
+
+{{end}}
+
+// ================================ ================= ================================
+
+func AllPackageEnums() []enums.Enum {
+    return []enums.Enum{ {{range .Enums}}
+        {{.EnumTypeName}}, {{end}}
+    }
+}

bfcodegen/enum-generate_test.go

@@ -2,6 +2,7 @@ package bfcodegen
 
 import (
 	_ "embed"
+	"fmt"
 	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/tst"
@@ -11,8 +12,11 @@ import (
 	"time"
 )
 
-//go:embed _test_example.tgz
-var ExampleModels []byte
+//go:embed _test_example_1.tgz
+var EnumExampleModels1 []byte
+
+//go:embed _test_example_2.tgz
+var EnumExampleModels2 []byte
 
 func TestGenerateEnumSpecs(t *testing.T) {
 
@@ -20,7 +24,7 @@ func TestGenerateEnumSpecs(t *testing.T) {
 
 	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
 
-	err := os.WriteFile(tmpFile, ExampleModels, 0o777)
+	err := os.WriteFile(tmpFile, EnumExampleModels1, 0o777)
 	tst.AssertNoErr(t, err)
 
 	t.Cleanup(func() { _ = os.Remove(tmpFile) })
@@ -33,10 +37,55 @@ func TestGenerateEnumSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	err = GenerateEnumSpecs(tmpDir, tmpDir+"/enums_gen.go")
+	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
 	tst.AssertNoErr(t, err)
 
-	err = GenerateEnumSpecs(tmpDir, tmpDir+"/enums_gen.go")
+	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
 	tst.AssertNoErr(t, err)
 
+	tst.AssertEqual(t, cs1, cs2)
+	tst.AssertEqual(t, s1, s2)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(s1)
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}
+
+func TestGenerateEnumSpecsData(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, EnumExampleModels2, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true, true)
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(s1)
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
 }

bfcodegen/id-generate.go

@@ -1,10 +1,12 @@
 package bfcodegen
 
 import (
+	"bytes"
+	_ "embed"
 	"errors"
 	"fmt"
+	"go/format"
 	"gogs.mikescher.com/BlackForestBytes/goext"
-	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
 	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rext"
@@ -14,7 +16,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
+	"text/template"
 )
 
 type IDDef struct {
@@ -23,13 +25,22 @@ type IDDef struct {
 	Name         string
 }
 
+type IDGenOptions struct {
+	DebugOutput *bool
+}
+
 var rexIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@id:type).*$`))
 
 var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
 
-func GenerateIDSpecs(sourceDir string, destFile string) error {
+//go:embed id-generate.template
+var templateIDGenerateText string
+
+func GenerateIDSpecs(sourceDir string, destFile string, opt *IDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -75,13 +86,18 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allIDs = append(allIDs, fileIDs...)
 
@@ -94,29 +110,20 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 		return errors.New("no package name found in any file")
 	}
 
-	err = os.WriteFile(destFile, []byte(fmtIDOutput(newChecksum, allIDs, pkgname)), 0o755)
+	fdata, err := format.Source([]byte(fmtIDOutput(newChecksum, allIDs, pkgname)))
 	if err != nil {
 		return err
 	}
 
-	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
+	err = os.WriteFile(destFile, fdata, 0o755)
 	if err != nil {
 		return err
 	}
 
-	if res.CommandTimedOut {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt timed out")
-	}
-	if res.ExitCode != 0 {
-		fmt.Println(res.StdCombined)
-		return errors.New("go fmt did not succeed")
-	}
-
 	return nil
 }
 
-func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
+func processIDFile(basedir string, fn string, debugOutput bool) ([]IDDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -157,7 +164,11 @@ func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
 				FileRelative: rfp,
 				Name:         match.GroupByName("name").Value(),
 			}
-			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
 			ids = append(ids, def)
 		}
 	}
@@ -166,71 +177,22 @@ func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
 }
 
 func fmtIDOutput(cs string, ids []IDDef, pkgname string) string {
-	str := "// Code generated by id-generate.go DO NOT EDIT.\n"
-	str += "\n"
-	str += "package " + pkgname + "\n"
-	str += "\n"
+	templ := template.Must(template.New("id-generate").Parse(templateIDGenerateText))
 
-	str += "import \"go.mongodb.org/mongo-driver/bson\"" + "\n"
-	str += "import \"go.mongodb.org/mongo-driver/bson/bsontype\"" + "\n"
-	str += "import \"go.mongodb.org/mongo-driver/bson/primitive\"" + "\n"
-	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/exerr\"" + "\n"
-	str += "\n"
-
-	str += "const ChecksumIDGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
-	str += "\n"
+	buffer := bytes.Buffer{}
 
 	anyDef := langext.ArrFirstOrNil(ids, func(def IDDef) bool { return def.Name == "AnyID" || def.Name == "AnyId" })
 
-	for _, iddef := range ids {
-
-		str += "// ================================ " + iddef.Name + " (" + iddef.FileRelative + ") ================================" + "\n"
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") MarshalBSONValue() (bsontype.Type, []byte, error) {" + "\n"
-		str += "	if objId, err := primitive.ObjectIDFromHex(string(i)); err == nil {" + "\n"
-		str += "		return bson.MarshalValue(objId)" + "\n"
-		str += "	} else {" + "\n"
-		str += "		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, \"Failed to marshal " + iddef.Name + "(\"+i.String()+\") to ObjectId\").Str(\"value\", string(i)).Type(\"type\", i).Build()" + "\n"
-		str += "	}" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") String() string {" + "\n"
-		str += "	return string(i)" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") ObjID() (primitive.ObjectID, error) {" + "\n"
-		str += "	return primitive.ObjectIDFromHex(string(i))" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		str += "func (i " + iddef.Name + ") Valid() bool {" + "\n"
-		str += "	_, err := primitive.ObjectIDFromHex(string(i))" + "\n"
-		str += "	return err == nil" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
-		if anyDef != nil {
-			str += "func (i " + iddef.Name + ") AsAny() " + anyDef.Name + " {" + "\n"
-			str += "	return " + anyDef.Name + "(i)" + "\n"
-			str += "}" + "\n"
-
-			str += "" + "\n"
-		}
-
-		str += "func New" + iddef.Name + "() " + iddef.Name + " {" + "\n"
-		str += "	return " + iddef.Name + "(primitive.NewObjectID().Hex())" + "\n"
-		str += "}" + "\n"
-
-		str += "" + "\n"
-
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+		"AnyDef":       anyDef,
+	})
+	if err != nil {
+		panic(err)
 	}
 
-	return str
+	return buffer.String()
 }

bfcodegen/id-generate.template

@@ -0,0 +1,47 @@
+// Code generated by id-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "go.mongodb.org/mongo-driver/bson"
+import "go.mongodb.org/mongo-driver/bson/bsontype"
+import "go.mongodb.org/mongo-driver/bson/primitive"
+import "gogs.mikescher.com/BlackForestBytes/goext/exerr"
+
+const ChecksumIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func (i {{.Name}}) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	if objId, err := primitive.ObjectIDFromHex(string(i)); err == nil {
+		return bson.MarshalValue(objId)
+	} else {
+		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, "Failed to marshal {{.Name}}("+i.String()+") to ObjectId").Str("value", string(i)).Type("type", i).Build()
+	}
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) ObjID() (primitive.ObjectID, error) {
+	return primitive.ObjectIDFromHex(string(i))
+}
+
+func (i {{.Name}}) Valid() bool {
+	_, err := primitive.ObjectIDFromHex(string(i))
+	return err == nil
+}
+
+{{if ne $.AnyDef nil}}
+func (i {{.Name}}) AsAny() {{$.AnyDef.Name}} {
+	return {{$.AnyDef.Name}}(i)
+}
+{{end}}
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(primitive.NewObjectID().Hex())
+}
+
+{{end}}

bfcodegen/id-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var IDExampleModels1 []byte
+
+func TestGenerateIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, IDExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", &IDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", &IDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/id_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

cmdext/cmdrunner.go

@@ -133,9 +133,6 @@ func run(opt CommandRunner) (CommandResult, error) {
 
 	case <-stderrFailChan:
 		_ = cmd.Process.Kill()
-		for _, lstr := range opt.listener {
-			lstr.Timeout()
-		}
 
 		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, 32*time.Millisecond); ok {
 			// most of the time the cmd.Process.Kill() should also have finished the pipereader
@@ -160,7 +157,8 @@ func run(opt CommandRunner) (CommandResult, error) {
 		}
 
 	case outobj := <-outputChan:
-		if exiterr, ok := outobj.err.(*exec.ExitError); ok {
+		var exiterr *exec.ExitError
+		if errors.As(outobj.err, &exiterr) {
 			excode := exiterr.ExitCode()
 			for _, lstr := range opt.listener {
 				lstr.Finished(excode)

cmdext/pipereader.go

@@ -32,8 +32,8 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	stdout := ""
 	go func() {
 		buf := make([]byte, 128)
-		for true {
-			n, out := pr.stdout.Read(buf)
+		for {
+			n, err := pr.stdout.Read(buf)
 			if n > 0 {
 				txt := string(buf[:n])
 				stdout += txt
@@ -42,11 +42,11 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 					lstr.ReadRawStdout(buf[:n])
 				}
 			}
-			if out == io.EOF {
+			if err == io.EOF {
 				break
 			}
-			if out != nil {
-				errch <- out
+			if err != nil {
+				errch <- err
 				break
 			}
 		}
@@ -61,7 +61,7 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	stderr := ""
 	go func() {
 		buf := make([]byte, 128)
-		for true {
+		for {
 			n, err := pr.stderr.Read(buf)
 
 			if n > 0 {

confext/confParser.go

@@ -41,12 +41,12 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			continue
 		}
 
-		if rvfield.Kind() == reflect.Struct {
+		envkey, found := rsfield.Tag.Lookup("env")
+		if !found || envkey == "-" {
+			continue
+		}
 
-			envkey, found := rsfield.Tag.Lookup("env")
-			if !found || envkey == "-" {
-				continue
-			}
+		if rvfield.Kind() == reflect.Struct && rvfield.Type() != reflect.TypeOf(time.UnixMilli(0)) {
 
 			subPrefix := prefix
 			if envkey != "" {
@@ -57,10 +57,7 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			if err != nil {
 				return err
 			}
-		}
 
-		envkey := rsfield.Tag.Get("env")
-		if envkey == "" || envkey == "-" {
 			continue
 		}
 

cryptext/passHash.go

@@ -66,7 +66,6 @@ func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bo
 		return int(version), nil, payload, false, nil, true
 	}
 
-	//
 	if version == 2 {
 		if len(split) != 3 {
 			return -1, nil, nil, false, nil, false

cryptext/pronouncablePassword.go

@@ -0,0 +1,263 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"io"
+	"math/big"
+	mathrand "math/rand"
+	"strings"
+)
+
+const (
+	ppStartChar            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppEndChar              = "ABDEFIKMNORSTUXYZ"
+	ppVowel                = "AEIOUY"
+	ppConsonant            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppSegmentLenMin        = 3
+	ppSegmentLenMax        = 7
+	ppMaxRepeatedVowel     = 2
+	ppMaxRepeatedConsonant = 2
+)
+
+var ppContinuation = map[uint8]string{
+	'A': "BCDFGHJKLMNPRSTVWXYZ",
+	'B': "ADFIKLMNORSTUY",
+	'C': "AEIKOUY",
+	'D': "AEILORSUYZ",
+	'E': "BCDFGHJKLMNPRSTVWXYZ",
+	'F': "ADEGIKLOPRTUY",
+	'G': "ABDEFHILMNORSTUY",
+	'H': "AEIOUY",
+	'I': "BCDFGHJKLMNPRSTVWXZ",
+	'J': "AEIOUY",
+	'K': "ADEFHILMNORSTUY",
+	'L': "ADEFGIJKMNOPSTUVWYZ",
+	'M': "ABEFIKOPSTUY",
+	'N': "ABEFIKOPSTUY",
+	'O': "BCDFGHJKLMNPRSTVWXYZ",
+	'P': "AEFIJLORSTUY",
+	'Q': "AEIOUY",
+	'R': "ADEFGHIJKLMNOPSTUVYZ",
+	'S': "ACDEIKLOPTUYZ",
+	'T': "AEHIJOPRSUWY",
+	'U': "BCDFGHJKLMNPRSTVWXZ",
+	'V': "AEIOUY",
+	'W': "AEIOUY",
+	'X': "AEIOUY",
+	'Y': "ABCDFGHKLMNPRSTVXZ",
+	'Z': "AEILOTUY",
+}
+
+var ppLog2Map = map[int]float64{
+	1:  0.00000000,
+	2:  1.00000000,
+	3:  1.58496250,
+	4:  2.00000000,
+	5:  2.32192809,
+	6:  2.58496250,
+	7:  2.80735492,
+	8:  3.00000000,
+	9:  3.16992500,
+	10: 3.32192809,
+	11: 3.45943162,
+	12: 3.58496250,
+	13: 3.70043972,
+	14: 3.80735492,
+	15: 3.90689060,
+	16: 4.00000000,
+	17: 4.08746284,
+	18: 4.16992500,
+	19: 4.24792751,
+	20: 4.32192809,
+	21: 4.39231742,
+	22: 4.45943162,
+	23: 4.52356196,
+	24: 4.58496250,
+	25: 4.64385619,
+	26: 4.70043972,
+	27: 4.75488750,
+	28: 4.80735492,
+	29: 4.85798100,
+	30: 4.90689060,
+	31: 4.95419631,
+	32: 5.00000000,
+}
+
+var (
+	ppVowelMap     = ppMakeSet(ppVowel)
+	ppConsonantMap = ppMakeSet(ppConsonant)
+	ppEndCharMap   = ppMakeSet(ppEndChar)
+)
+
+func ppMakeSet(v string) map[uint8]bool {
+	mp := make(map[uint8]bool, len(v))
+	for _, chr := range v {
+		mp[uint8(chr)] = true
+	}
+	return mp
+}
+
+func ppRandInt(rng io.Reader, max int) int {
+	v, err := rand.Int(rng, big.NewInt(int64(max)))
+	if err != nil {
+		panic(err)
+	}
+	return int(v.Int64())
+}
+
+func ppRand(rng io.Reader, chars string, entropy *float64) uint8 {
+	chr := chars[ppRandInt(rng, len(chars))]
+
+	*entropy = *entropy + ppLog2Map[len(chars)]
+
+	return chr
+}
+
+func ppCharType(chr uint8) (bool, bool) {
+	_, ok1 := ppVowelMap[chr]
+	_, ok2 := ppConsonantMap[chr]
+
+	return ok1, ok2
+}
+
+func ppCharsetRemove(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; !ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func ppCharsetFilter(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func PronouncablePasswordExt(rng io.Reader, pwlen int) (string, float64) {
+
+	// kinda pseudo markov-chain - with a few extra rules and no weights...
+
+	if pwlen <= 0 {
+		return "", 0
+	}
+
+	vowelCount := 0
+	consoCount := 0
+	entropy := float64(0)
+
+	startChar := ppRand(rng, ppStartChar, &entropy)
+
+	result := string(startChar)
+	currentChar := startChar
+
+	isVowel, isConsonant := ppCharType(currentChar)
+	if isVowel {
+		vowelCount = 1
+	}
+	if isConsonant {
+		consoCount = ppMaxRepeatedConsonant
+	}
+
+	segmentLen := 1
+
+	segmentLenTarget := ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+
+	for len(result) < pwlen {
+
+		charset := ppContinuation[currentChar]
+		if vowelCount >= ppMaxRepeatedVowel {
+			charset = ppCharsetRemove(charset, ppVowelMap, false)
+		}
+		if consoCount >= ppMaxRepeatedConsonant {
+			charset = ppCharsetRemove(charset, ppConsonantMap, false)
+		}
+
+		lastOfSegment := false
+		newSegment := false
+
+		if len(result)+1 == pwlen {
+			// last of result
+			charset = ppCharsetFilter(charset, ppEndCharMap, false)
+		} else if segmentLen+1 == segmentLenTarget {
+			// last of segment
+			charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+			if charsetNew != "" {
+				charset = charsetNew
+				lastOfSegment = true
+			}
+		} else if segmentLen >= segmentLenTarget {
+			// (perhaps) start of new segment
+			if _, ok := ppEndCharMap[currentChar]; ok {
+				charset = ppStartChar
+				newSegment = true
+			} else {
+				// continue segment for one more char to (hopefully) find an end-char
+				charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+				if charsetNew != "" {
+					charset = charsetNew
+					lastOfSegment = true
+				}
+			}
+		} else {
+			// normal continuation
+		}
+
+		newChar := ppRand(rng, charset, &entropy)
+		if lastOfSegment {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		} else if newSegment {
+			currentChar = newChar
+			segmentLen = 1
+			result += strings.ToUpper(string(newChar))
+			segmentLenTarget = ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+			vowelCount = 0
+			consoCount = 0
+		} else {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		}
+
+		isVowel, isConsonant := ppCharType(currentChar)
+		if isVowel {
+			vowelCount++
+			consoCount = 0
+		}
+		if isConsonant {
+			vowelCount = 0
+			if newSegment {
+				consoCount = ppMaxRepeatedConsonant
+			} else {
+				consoCount++
+			}
+		}
+	}
+
+	return result, entropy
+}
+
+func PronouncablePassword(len int) string {
+	v, _ := PronouncablePasswordExt(rand.Reader, len)
+	return v
+}
+
+func PronouncablePasswordSeeded(seed int64, len int) string {
+
+	v, _ := PronouncablePasswordExt(mathrand.New(mathrand.NewSource(seed)), len)
+	return v
+}

cryptext/pronouncablePassword_test.go

@@ -0,0 +1,35 @@
+package cryptext
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestPronouncablePasswordExt(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw, entropy := PronouncablePasswordExt(rand.New(rand.NewSource(int64(i))), 16)
+		fmt.Printf("[%.2f] => %s\n", entropy, pw)
+	}
+}
+
+func TestPronouncablePasswordSeeded(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw := PronouncablePasswordSeeded(int64(i), 8)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePassword(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw := PronouncablePassword(i + 1)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePasswordWrongLen(t *testing.T) {
+	PronouncablePassword(0)
+	PronouncablePassword(-1)
+	PronouncablePassword(-2)
+	PronouncablePassword(-3)
+}

cursortoken/filter.go

@@ -4,6 +4,10 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
+type RawFilter interface {
+	FilterQuery() mongo.Pipeline
+}
+
 type Filter interface {
 	FilterQuery() mongo.Pipeline
 	Pagination() (string, SortDirection, string, SortDirection)

dataext/syncMap.go

@@ -0,0 +1,113 @@
+package dataext
+
+import "sync"
+
+type SyncMap[TKey comparable, TData any] struct {
+	data map[TKey]TData
+	lock sync.Mutex
+}
+
+func (s *SyncMap[TKey, TData]) Set(key TKey, data TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	s.data[key] = data
+}
+
+func (s *SyncMap[TKey, TData]) SetIfNotContains(key TKey, data TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if _, existsInPreState := s.data[key]; existsInPreState {
+		return false
+	}
+
+	s.data[key] = data
+
+	return true
+}
+
+func (s *SyncMap[TKey, TData]) Get(key TKey) (TData, bool) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		return v, true
+	} else {
+		return *new(TData), false
+	}
+}
+
+func (s *SyncMap[TKey, TData]) Delete(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	delete(s.data, key)
+
+	return ok
+}
+
+func (s *SyncMap[TKey, TData]) Contains(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	return ok
+}
+
+func (s *SyncMap[TKey, TData]) GetAllKeys() []TKey {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TKey, 0, len(s.data))
+
+	for k := range s.data {
+		r = append(r, k)
+	}
+
+	return r
+}
+
+func (s *SyncMap[TKey, TData]) GetAllValues() []TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TData, 0, len(s.data))
+
+	for _, v := range s.data {
+		r = append(r, v)
+	}
+
+	return r
+}

enums/enum.go

@@ -5,6 +5,8 @@ type Enum interface {
 	ValuesAny() []any
 	ValuesMeta() []EnumMetaValue
 	VarName() string
+	TypeName() string
+	PackageName() string
 }
 
 type StringEnum interface {
@@ -15,10 +17,17 @@ type StringEnum interface {
 type DescriptionEnum interface {
 	Enum
 	Description() string
+	DescriptionMeta() EnumDescriptionMetaValue
 }
 
 type EnumMetaValue struct {
 	VarName     string  `json:"varName"`
-	Value       any     `json:"value"`
+	Value       Enum    `json:"value"`
 	Description *string `json:"description"`
 }
+
+type EnumDescriptionMetaValue struct {
+	VarName     string `json:"varName"`
+	Value       Enum   `json:"value"`
+	Description string `json:"description"`
+}

exerr/builder.go

@@ -68,8 +68,10 @@ func init() {
 }
 
 type Builder struct {
+	wrappedErr      error
 	errorData       *ExErr
 	containsGinData bool
+	noLog           bool
 }
 
 func Get(err error) *Builder {
@@ -88,9 +90,9 @@ func Wrap(err error, msg string) *Builder {
 	if !pkgconfig.RecursiveErrors {
 		v := FromError(err)
 		v.Message = msg
-		return &Builder{errorData: v}
+		return &Builder{wrappedErr: err, errorData: v}
 	}
-	return &Builder{errorData: wrapExErr(FromError(err), msg, CatWrap, 1)}
+	return &Builder{wrappedErr: err, errorData: wrapExErr(FromError(err), msg, CatWrap, 1)}
 }
 
 // ----------------------------------------------------------------------------
@@ -190,6 +192,13 @@ func (b *Builder) System() *Builder {
 
 // ----------------------------------------------------------------------------
 
+func (b *Builder) NoLog() *Builder {
+	b.noLog = true
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
 func (b *Builder) Id(key string, val fmt.Stringer) *Builder {
 	return b.addMeta(key, MDTID, newIDWrap(val))
 }
@@ -275,7 +284,7 @@ func (b *Builder) Any(key string, val any) *Builder {
 }
 
 func (b *Builder) Stringer(key string, val fmt.Stringer) *Builder {
-	if val == nil {
+	if langext.IsNil(val) {
 		return b.addMeta(key, MDTString, "(!nil)")
 	} else {
 		return b.addMeta(key, MDTString, val.String())
@@ -401,16 +410,22 @@ func extractHeader(header map[string][]string) []string {
 
 // Build creates a new error, ready to pass up the stack
 // If the errors is not SevWarn or SevInfo it gets also logged (in short form, without stacktrace) onto stdout
+// Can be gloablly configured with ZeroLogErrTraces and ZeroLogAllTraces
+// Can be locally suppressed with Builder.NoLog()
 func (b *Builder) Build() error {
 	warnOnPkgConfigNotInitialized()
 
-	if pkgconfig.ZeroLogErrTraces && (b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal) {
+	if pkgconfig.DisableErrorWrapping && b.wrappedErr != nil {
+		return b.wrappedErr
+	}
+
+	if pkgconfig.ZeroLogErrTraces && !b.noLog && (b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal) {
 		b.errorData.ShortLog(stackSkipLogger.Error())
-	} else if pkgconfig.ZeroLogAllTraces {
+	} else if pkgconfig.ZeroLogAllTraces && !b.noLog {
 		b.errorData.ShortLog(stackSkipLogger.Error())
 	}
 
-	b.CallListener(MethodBuild)
+	b.errorData.CallListener(MethodBuild)
 
 	return b.errorData
 }
@@ -432,7 +447,7 @@ func (b *Builder) Output(ctx context.Context, g *gin.Context) {
 		b.errorData.Log(stackSkipLogger.Warn())
 	}
 
-	b.CallListener(MethodOutput)
+	b.errorData.CallListener(MethodOutput)
 }
 
 // Print prints the error
@@ -444,7 +459,7 @@ func (b *Builder) Print() {
 		b.errorData.ShortLog(stackSkipLogger.Warn())
 	}
 
-	b.CallListener(MethodPrint)
+	b.errorData.CallListener(MethodPrint)
 }
 
 func (b *Builder) Format(level LogPrintLevel) string {
@@ -457,7 +472,7 @@ func (b *Builder) Fatal() {
 	b.errorData.Severity = SevFatal
 	b.errorData.Log(stackSkipLogger.WithLevel(zerolog.FatalLevel))
 
-	b.CallListener(MethodFatal)
+	b.errorData.CallListener(MethodFatal)
 
 	os.Exit(1)
 }

exerr/data.go

@@ -46,6 +46,11 @@ var (
 	TypeCursorTokenDecode = NewType("CURSOR_TOKEN_DECODE", langext.Ptr(500))
 	TypeMongoFilter       = NewType("MONGO_FILTER", langext.Ptr(500))
 	TypeMongoReflection   = NewType("MONGO_REFLECTION", langext.Ptr(500))
+	TypeMongoInvalidOpt   = NewType("MONGO_INVALIDOPT", langext.Ptr(500))
+
+	TypeSQLQuery  = NewType("SQL_QUERY", langext.Ptr(500))
+	TypeSQLBuild  = NewType("SQL_BUILD", langext.Ptr(500))
+	TypeSQLDecode = NewType("SQL_DECODE", langext.Ptr(500))
 
 	TypeWrap = NewType("Wrap", nil)
 
@@ -56,22 +61,29 @@ var (
 	TypeBindFailHeader   = NewType("BINDFAIL_HEADER", langext.Ptr(400))
 
 	TypeMarshalEntityID = NewType("MARSHAL_ENTITY_ID", langext.Ptr(400))
+	TypeInvalidCSID     = NewType("INVALID_CSID", langext.Ptr(400))
+
+	TypeGoogleStatuscode = NewType("GOOGLE_STATUSCODE", langext.Ptr(400))
+	TypeGoogleResponse   = NewType("GOOGLE_RESPONSE", langext.Ptr(400))
 
 	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
 	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))
 
-	// other values come the used package
+	// other values come from the downstream application that uses goext
 )
 
-var registeredTypes = dataext.SyncSet[string]{}
+var registeredTypes = dataext.SyncMap[string, ErrorType]{}
 
 func NewType(key string, defStatusCode *int) ErrorType {
-	insertOkay := registeredTypes.Add(key)
-	if !insertOkay {
-		panic("Cannot register same ErrType ('" + key + "') more than once")
-	}
+	et := ErrorType{key, defStatusCode}
 
-	return ErrorType{key, defStatusCode}
+	registeredTypes.Set(key, et)
+
+	return et
+}
+
+func ListRegisteredTypes() []ErrorType {
+	return registeredTypes.GetAllValues()
 }
 
 type LogPrintLevel string

exerr/errinit.go

@@ -13,6 +13,7 @@ type ErrorPackageConfig struct {
 	IncludeMetaInGinOutput bool                                             // Log meta fields ( from e.g. `.Str(key, val).Build()` ) to gin in err.Output()
 	ExtendGinOutput        func(err *ExErr, json map[string]any)            // (Optionally) extend the gin output with more fields
 	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any) // (Optionally) extend the gin `__data` output with more fields
+	DisableErrorWrapping   bool                                             // Disables the exerr.Wrap()...Build() function - will always return the original error
 }
 
 type ErrorPackageConfigInit struct {
@@ -23,6 +24,7 @@ type ErrorPackageConfigInit struct {
 	IncludeMetaInGinOutput *bool
 	ExtendGinOutput        func(err *ExErr, json map[string]any)
 	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any)
+	DisableErrorWrapping   *bool
 }
 
 var initialized = false
@@ -35,6 +37,7 @@ var pkgconfig = ErrorPackageConfig{
 	IncludeMetaInGinOutput: true,
 	ExtendGinOutput:        func(err *ExErr, json map[string]any) {},
 	ExtendGinDataOutput:    func(err *ExErr, depth int, json map[string]any) {},
+	DisableErrorWrapping:   false,
 }
 
 // Init initializes the exerr packages
@@ -63,11 +66,16 @@ func Init(cfg ErrorPackageConfigInit) {
 		IncludeMetaInGinOutput: langext.Coalesce(cfg.IncludeMetaInGinOutput, pkgconfig.IncludeMetaInGinOutput),
 		ExtendGinOutput:        ego,
 		ExtendGinDataOutput:    egdo,
+		DisableErrorWrapping:   langext.Coalesce(cfg.DisableErrorWrapping, pkgconfig.DisableErrorWrapping),
 	}
 
 	initialized = true
 }
 
+func Initialized() bool {
+	return initialized
+}
+
 func warnOnPkgConfigNotInitialized() {
 	if !initialized {
 		fmt.Printf("\n")

exerr/exerr.go

@@ -30,7 +30,7 @@ type ExErr struct {
 }
 
 func (ee *ExErr) Error() string {
-	return ee.Message
+	return ee.RecursiveMessage()
 }
 
 // Unwrap must be implemented so that some error.XXX methods work
@@ -164,19 +164,37 @@ func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
 }
 
 func (ee *ExErr) ShortLog(evt *zerolog.Event) {
-	ee.Meta.Apply(evt).Msg(ee.FormatLog(LogPrintShort))
+	ee.Meta.Apply(evt, langext.Ptr(240)).Msg(ee.FormatLog(LogPrintShort))
 }
 
 // RecursiveMessage returns the message to show
 // = first error (top-down) that is not wrapping/foreign/empty
+// = lowest level error (that is not empty)
+// = fallback to self.message
 func (ee *ExErr) RecursiveMessage() string {
+
+	// ==== [1] ==== first error (top-down) that is not wrapping/foreign/empty
+
 	for curr := ee; curr != nil; curr = curr.OriginalError {
 		if curr.Message != "" && curr.Category != CatWrap && curr.Category != CatForeign {
 			return curr.Message
 		}
 	}
 
-	// fallback to self
+	// ==== [2] ==== lowest level error (that is not empty)
+
+	deepestMsg := ""
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Message != "" {
+			deepestMsg = curr.Message
+		}
+	}
+	if deepestMsg != "" {
+		return deepestMsg
+	}
+
+	// ==== [3] ==== fallback to self.message
+
 	return ee.Message
 }
 
@@ -240,6 +258,73 @@ func (ee *ExErr) Depth() int {
 	}
 }
 
+// GetMeta returns the meta value with the specified key
+// this method recurses through all wrapped errors and returns the first matching meta value
+func (ee *ExErr) GetMeta(key string) (any, bool) {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if v, ok := curr.Meta[key]; ok {
+			return v.Value, true
+		}
+	}
+
+	return nil, false
+}
+
+// GetMetaString functions the same as GetMeta, but returns false if the type does not match
+func (ee *ExErr) GetMetaString(key string) (string, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(string); ok {
+			return v2, true
+		}
+	}
+	return "", false
+}
+
+func (ee *ExErr) GetMetaBool(key string) (bool, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(bool); ok {
+			return v2, true
+		}
+	}
+	return false, false
+}
+
+func (ee *ExErr) GetMetaInt(key string) (int, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(int); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaFloat32(key string) (float32, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(float32); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaFloat64(key string) (float64, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(float64); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaTime(key string) (time.Time, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(time.Time); ok {
+			return v2, true
+		}
+	}
+	return time.Time{}, false
+}
+
 // contains test if the supplied error is contained in this error (anywhere in the chain)
 func (ee *ExErr) contains(original *ExErr) (*ExErr, bool) {
 	if original == nil {

exerr/gin.go

@@ -57,6 +57,19 @@ func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) la
 	return ginJson
 }
 
+func (ee *ExErr) ToDefaultAPIJson() (string, error) {
+
+	gjr := json.GoJsonRender{Data: ee.ToAPIJson(true, pkgconfig.ExtendedGinOutput, pkgconfig.IncludeMetaInGinOutput), NilSafeSlices: true, NilSafeMaps: true}
+
+	r, err := gjr.RenderString()
+
+	if err != nil {
+		return "", err
+	}
+
+	return r, nil
+}
+
 // ToAPIJson converts the ExError to a json object
 // (the same object as used in the Output(gin) method)
 //

exerr/listener.go

@@ -25,13 +25,11 @@ func RegisterListener(l Listener) {
 	listener = append(listener, l)
 }
 
-func (b *Builder) CallListener(m Method) {
-	valErr := b.errorData
-
+func (ee *ExErr) CallListener(m Method) {
 	listenerLock.Lock()
 	defer listenerLock.Unlock()
 
 	for _, v := range listener {
-		v(m, valErr)
+		v(m, ee)
 	}
 }

exerr/meta.go

@@ -9,6 +9,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"math"
 	"strconv"
 	"strings"
 	"time"
@@ -217,23 +218,35 @@ func (v MetaValue) ShortString(lim int) string {
 	return "(err)"
 }
 
-func (v MetaValue) Apply(key string, evt *zerolog.Event) *zerolog.Event {
+func (v MetaValue) Apply(key string, evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	switch v.DataType {
 	case MDTString:
-		return evt.Str(key, v.Value.(string))
+		if limitLen == nil {
+			return evt.Str(key, v.Value.(string))
+		} else {
+			return evt.Str(key, langext.StrLimit(v.Value.(string), *limitLen, "..."))
+		}
 	case MDTID:
 		return evt.Str(key, v.Value.(IDWrap).Value)
 	case MDTAny:
 		if v.Value.(AnyWrap).IsError {
 			return evt.Str(key, "(err)")
 		} else {
-			return evt.Str(key, v.Value.(AnyWrap).Json)
+			if limitLen == nil {
+				return evt.Str(key, v.Value.(AnyWrap).Json)
+			} else {
+				return evt.Str(key, langext.StrLimit(v.Value.(AnyWrap).Json, *limitLen, "..."))
+			}
 		}
 	case MDTStringPtr:
 		if langext.IsNil(v.Value) {
 			return evt.Str(key, "<<null>>")
 		}
-		return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		if limitLen == nil {
+			return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		} else {
+			return evt.Str(key, langext.StrLimit(langext.CoalesceString(v.Value.(*string), "<<null>>"), *limitLen, "..."))
+		}
 	case MDTInt:
 		return evt.Int(key, v.Value.(int))
 	case MDTInt8:
@@ -655,6 +668,28 @@ func (v MetaValue) rawValueForJson() any {
 		}
 		return v.Value.(EnumWrap).ValueString
 	}
+	if v.DataType == MDTFloat32 {
+		if math.IsNaN(float64(v.Value.(float32))) {
+			return "float64::NaN"
+		} else if math.IsInf(float64(v.Value.(float32)), +1) {
+			return "float64::+inf"
+		} else if math.IsInf(float64(v.Value.(float32)), -1) {
+			return "float64::-inf"
+		} else {
+			return v.Value
+		}
+	}
+	if v.DataType == MDTFloat64 {
+		if math.IsNaN(v.Value.(float64)) {
+			return "float64::NaN"
+		} else if math.IsInf(v.Value.(float64), +1) {
+			return "float64::+inf"
+		} else if math.IsInf(v.Value.(float64), -1) {
+			return "float64::-inf"
+		} else {
+			return v.Value
+		}
+	}
 	return v.Value
 }
 
@@ -702,9 +737,9 @@ func (mm MetaMap) Any() bool {
 	return len(mm) > 0
 }
 
-func (mm MetaMap) Apply(evt *zerolog.Event) *zerolog.Event {
+func (mm MetaMap) Apply(evt *zerolog.Event, limitLen *int) *zerolog.Event {
 	for key, val := range mm {
-		evt = val.Apply(key, evt)
+		evt = val.Apply(key, evt, limitLen)
 	}
 	return evt
 }

fsext/exists.go

@@ -0,0 +1,36 @@
+package fsext
+
+import "os"
+
+func PathExists(fp string) (bool, error) {
+	_, err := os.Stat(fp)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func FileExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return !stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func DirectoryExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}

ginext/engine.go

@@ -6,20 +6,25 @@ import (
 	"github.com/rs/zerolog/log"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
 	"net"
 	"net/http"
+	"net/http/httptest"
+	"regexp"
 	"strings"
 	"time"
 )
 
 type GinWrapper struct {
 	engine          *gin.Engine
-	SuppressGinLogs bool
+	suppressGinLogs bool
 
-	allowCors      bool
-	ginDebug       bool
-	bufferBody     bool
-	requestTimeout time.Duration
+	allowCors             bool
+	ginDebug              bool
+	bufferBody            bool
+	requestTimeout        time.Duration
+	listenerBeforeRequest []func(g *gin.Context)
+	listenerAfterRequest  []func(g *gin.Context, resp HTTPResponse)
 
 	routeSpecs []ginRouteSpec
 }
@@ -31,40 +36,46 @@ type ginRouteSpec struct {
 	Handler     string
 }
 
+type Options struct {
+	AllowCors             *bool                                     // Add cors handler to allow all CORS requests on the default http methods
+	GinDebug              *bool                                     // Set gin.debug to true (adds more logs)
+	BufferBody            *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
+	Timeout               *time.Duration                            // The default handler timeout
+	ListenerBeforeRequest []func(g *gin.Context)                    // Register listener that are called before the handler method
+	ListenerAfterRequest  []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+}
+
 // NewEngine creates a new (wrapped) ginEngine
-// Parameters are:
-// - [allowCors]    Add cors handler to allow all CORS requests on the default http methods
-// - [ginDebug]     Set gin.debug to true (adds more logs)
-// - [bufferBody]   Buffers the input body stream, this way the ginext error handler can later include the whole request body
-// - [timeout]      The default handler timeout
-func NewEngine(allowCors bool, ginDebug bool, bufferBody bool, timeout time.Duration) *GinWrapper {
+func NewEngine(opt Options) *GinWrapper {
 	engine := gin.New()
 
 	wrapper := &GinWrapper{
-		engine:          engine,
-		SuppressGinLogs: false,
-		allowCors:       allowCors,
-		ginDebug:        ginDebug,
-		bufferBody:      bufferBody,
-		requestTimeout:  timeout,
+		engine:                engine,
+		suppressGinLogs:       false,
+		allowCors:             langext.Coalesce(opt.AllowCors, false),
+		ginDebug:              langext.Coalesce(opt.GinDebug, true),
+		bufferBody:            langext.Coalesce(opt.BufferBody, false),
+		requestTimeout:        langext.Coalesce(opt.Timeout, 24*time.Hour),
+		listenerBeforeRequest: opt.ListenerBeforeRequest,
+		listenerAfterRequest:  opt.ListenerAfterRequest,
 	}
 
 	engine.RedirectFixedPath = false
 	engine.RedirectTrailingSlash = false
 
-	if allowCors {
+	if wrapper.allowCors {
 		engine.Use(CorsMiddleware())
 	}
 
 	// do not debug-print routes
 	gin.DebugPrintRouteFunc = func(_, _, _ string, _ int) {}
 
-	if ginDebug {
+	if !wrapper.ginDebug {
 		gin.SetMode(gin.ReleaseMode)
 
 		ginlogger := gin.Logger()
 		engine.Use(func(context *gin.Context) {
-			if !wrapper.SuppressGinLogs {
+			if !wrapper.suppressGinLogs {
 				ginlogger(context)
 			}
 		})
@@ -126,8 +137,8 @@ func (w *GinWrapper) DebugPrintRoutes() {
 		line := [4]string{
 			spec.Method,
 			spec.URL,
-			strings.Join(spec.Middlewares, " -> "),
-			spec.Handler,
+			strings.Join(langext.ArrMap(spec.Middlewares, w.cleanMiddlewareName), " -> "),
+			w.cleanMiddlewareName(spec.Handler),
 		}
 
 		lines = append(lines, line)
@@ -138,12 +149,47 @@ func (w *GinWrapper) DebugPrintRoutes() {
 		pad[3] = mathext.Max(pad[3], len(line[3]))
 	}
 
+	fmt.Printf("Gin-Routes:\n")
+	fmt.Printf("{\n")
 	for _, line := range lines {
 
-		fmt.Printf("Gin-Route: %s  %s  -->  %s  -->  %s\n",
+		fmt.Printf(" %s  %s  -->  %s  -->  %s\n",
 			langext.StrPadRight("["+line[0]+"]", " ", pad[0]+2),
 			langext.StrPadRight(line[1], " ", pad[1]),
 			langext.StrPadRight(line[2], " ", pad[2]),
 			langext.StrPadRight(line[3], " ", pad[3]))
 	}
+	fmt.Printf("}\n")
+}
+
+func (w *GinWrapper) cleanMiddlewareName(fname string) string {
+
+	funcSuffix := rext.W(regexp.MustCompile(`\.func[0-9]+(?:\.[0-9]+)*$`))
+	if match, ok := funcSuffix.MatchFirst(fname); ok {
+		fname = fname[:len(fname)-match.FullMatch().Length()]
+	}
+
+	if strings.HasSuffix(fname, ".(*GinRoutesWrapper).WithJSONFilter") {
+		fname = "[JSONFilter]"
+	}
+
+	if fname == "ginext.BodyBuffer" {
+		fname = "[BodyBuffer]"
+	}
+
+	skipPrefixes := []string{"api.(*Handler).", "api.", "ginext.", "handler.", "admin-app.", "employee-app.", "employer-app."}
+	for _, pfx := range skipPrefixes {
+		if strings.HasPrefix(fname, pfx) {
+			fname = fname[len(pfx):]
+		}
+	}
+
+	return fname
+}
+
+// ServeHTTP only used for unit tests
+func (w *GinWrapper) ServeHTTP(req *http.Request) *httptest.ResponseRecorder {
+	respRec := httptest.NewRecorder()
+	w.engine.ServeHTTP(respRec, req)
+	return respRec
 }

ginext/funcWrapper.go

@@ -14,7 +14,17 @@ func Wrap(w *GinWrapper, fn WHandlerFunc) gin.HandlerFunc {
 
 		reqctx := g.Request.Context()
 
-		wrap, stackTrace, panicObj := callPanicSafe(fn, PreContext{wrapper: w, ginCtx: g})
+		pctx := PreContext{
+			wrapper:        w,
+			ginCtx:         g,
+			persistantData: &preContextData{},
+		}
+
+		for _, lstr := range w.listenerBeforeRequest {
+			lstr(g)
+		}
+
+		wrap, stackTrace, panicObj := callPanicSafe(fn, pctx)
 		if panicObj != nil {
 
 			fmt.Printf("\n======== ======== STACKTRACE ======== ========\n%s\n======== ======== ======== ========\n\n", stackTrace)
@@ -32,6 +42,17 @@ func Wrap(w *GinWrapper, fn WHandlerFunc) gin.HandlerFunc {
 			panic("Writing in WrapperFunc is not supported")
 		}
 
+		if pctx.persistantData.sessionObj != nil {
+			err := pctx.persistantData.sessionObj.Finish(reqctx, wrap)
+			if err != nil {
+				wrap = Error(exerr.Wrap(err, "Failed to finish session").Any("originalResponse", wrap).Build())
+			}
+		}
+
+		for _, lstr := range w.listenerAfterRequest {
+			lstr(g, wrap)
+		}
+
 		if reqctx.Err() == nil {
 			wrap.Write(g)
 		}

ginext/preContext.go

@@ -1,25 +1,34 @@
 package ginext
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/gin-gonic/gin/binding"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"io"
 	"runtime/debug"
 	"time"
 )
 
 type PreContext struct {
-	ginCtx  *gin.Context
-	wrapper *GinWrapper
-	uri     any
-	query   any
-	body    any
-	form    any
-	header  any
-	timeout *time.Duration
+	ginCtx         *gin.Context
+	wrapper        *GinWrapper
+	uri            any
+	query          any
+	body           any
+	rawbody        *[]byte
+	form           any
+	header         any
+	timeout        *time.Duration
+	persistantData *preContextData // must be a ptr, so that we can get the values back in out Wrap func
+}
+
+type preContextData struct {
+	sessionObj SessionObject
 }
 
 func (pctx *PreContext) URI(uri any) *PreContext {
@@ -37,6 +46,11 @@ func (pctx *PreContext) Body(body any) *PreContext {
 	return pctx
 }
 
+func (pctx *PreContext) RawBody(rawbody *[]byte) *PreContext {
+	pctx.rawbody = rawbody
+	return pctx
+}
+
 func (pctx *PreContext) Form(form any) *PreContext {
 	pctx.form = form
 	return pctx
@@ -52,6 +66,11 @@ func (pctx *PreContext) WithTimeout(to time.Duration) *PreContext {
 	return pctx
 }
 
+func (pctx *PreContext) WithSession(sessionObj SessionObject) *PreContext {
+	pctx.persistantData.sessionObj = sessionObj
+	return pctx
+}
+
 func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
 	if pctx.uri != nil {
 		if err := pctx.ginCtx.ShouldBindUri(pctx.uri); err != nil {
@@ -90,6 +109,23 @@ func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
 		}
 	}
 
+	if pctx.rawbody != nil {
+		if brc, ok := pctx.ginCtx.Request.Body.(dataext.BufferedReadCloser); ok {
+			v, err := brc.BufferedAll()
+			if err != nil {
+				return nil, nil, langext.Ptr(Error(err))
+			}
+			*pctx.rawbody = v
+		} else {
+			buf := &bytes.Buffer{}
+			_, err := io.Copy(buf, pctx.ginCtx.Request.Body)
+			if err != nil {
+				return nil, nil, langext.Ptr(Error(err))
+			}
+			*pctx.rawbody = buf.Bytes()
+		}
+	}
+
 	if pctx.form != nil {
 		if pctx.ginCtx.ContentType() == "multipart/form-data" {
 			if err := pctx.ginCtx.ShouldBindWith(pctx.form, binding.Form); err != nil {
@@ -126,8 +162,17 @@ func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
 	}
 
 	ictx, cancel := context.WithTimeout(context.Background(), langext.Coalesce(pctx.timeout, pctx.wrapper.requestTimeout))
+
 	actx := CreateAppContext(pctx.ginCtx, ictx, cancel)
 
+	if pctx.persistantData.sessionObj != nil {
+		err := pctx.persistantData.sessionObj.Init(pctx.ginCtx, actx)
+		if err != nil {
+			actx.Cancel()
+			return nil, nil, langext.Ptr(Error(exerr.Wrap(err, "Failed to init session").Build()))
+		}
+	}
+
 	return actx, pctx.ginCtx, nil
 }
 

ginext/response.go

@@ -5,8 +5,20 @@ import (
 	"github.com/gin-gonic/gin"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"os"
 )
 
+type cookieval struct {
+	name     string
+	value    string
+	maxAge   int
+	path     string
+	domain   string
+	secure   bool
+	httpOnly bool
+}
+
 type headerval struct {
 	Key string
 	Val string
@@ -15,19 +27,42 @@ type headerval struct {
 type HTTPResponse interface {
 	Write(g *gin.Context)
 	WithHeader(k string, v string) HTTPResponse
+	WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse
+	IsSuccess() bool
+}
+
+type InspectableHTTPResponse interface {
+	HTTPResponse
+
+	Statuscode() int
+	BodyString(g *gin.Context) *string
+	ContentType() string
+	Headers() []string
 }
 
 type jsonHTTPResponse struct {
 	statusCode int
 	data       any
 	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j jsonHTTPResponse) jsonRenderer(g *gin.Context) json.GoJsonRender {
+	var f *string
+	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
+		f = &jsonfilter
+	}
+	return json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f}
 }
 
 func (j jsonHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
-	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true})
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.Render(j.statusCode, j.jsonRenderer(g))
 }
 
 func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {
@@ -35,15 +70,48 @@ func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j jsonHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j jsonHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j jsonHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j jsonHTTPResponse) BodyString(g *gin.Context) *string {
+	if str, err := j.jsonRenderer(g).RenderString(); err == nil {
+		return &str
+	} else {
+		return nil
+	}
+}
+
+func (j jsonHTTPResponse) ContentType() string {
+	return "application/json"
+}
+
+func (j jsonHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type emptyHTTPResponse struct {
 	statusCode int
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j emptyHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Status(j.statusCode)
 }
 
@@ -52,16 +120,45 @@ func (j emptyHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j emptyHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j emptyHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j emptyHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j emptyHTTPResponse) BodyString(*gin.Context) *string {
+	return nil
+}
+
+func (j emptyHTTPResponse) ContentType() string {
+	return ""
+}
+
+func (j emptyHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type textHTTPResponse struct {
 	statusCode int
 	data       string
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j textHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.String(j.statusCode, "%s", j.data)
 }
 
@@ -70,17 +167,46 @@ func (j textHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j textHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j textHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j textHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j textHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(j.data)
+}
+
+func (j textHTTPResponse) ContentType() string {
+	return "text/plain"
+}
+
+func (j textHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type dataHTTPResponse struct {
 	statusCode  int
 	data        []byte
 	contentType string
 	headers     []headerval
+	cookies     []cookieval
 }
 
 func (j dataHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Data(j.statusCode, j.contentType, j.data)
 }
 
@@ -89,11 +215,37 @@ func (j dataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j dataHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j dataHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j dataHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j dataHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(string(j.data))
+}
+
+func (j dataHTTPResponse) ContentType() string {
+	return j.contentType
+}
+
+func (j dataHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type fileHTTPResponse struct {
 	mimetype string
 	filepath string
 	filename *string
 	headers  []headerval
+	cookies  []cookieval
 }
 
 func (j fileHTTPResponse) Write(g *gin.Context) {
@@ -105,6 +257,9 @@ func (j fileHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.File(j.filepath)
 }
 
@@ -113,23 +268,55 @@ func (j fileHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j fileHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j fileHTTPResponse) IsSuccess() bool {
+	return true
+}
+
+func (j fileHTTPResponse) Statuscode() int {
+	return 200
+}
+
+func (j fileHTTPResponse) BodyString(*gin.Context) *string {
+	data, err := os.ReadFile(j.filepath)
+	if err != nil {
+		return nil
+	}
+	return langext.Ptr(string(data))
+}
+
+func (j fileHTTPResponse) ContentType() string {
+	return j.mimetype
+}
+
+func (j fileHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type downloadDataHTTPResponse struct {
 	statusCode int
 	mimetype   string
 	data       []byte
 	filename   *string
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j downloadDataHTTPResponse) Write(g *gin.Context) {
 	g.Header("Content-Type", j.mimetype) // if we don't set it here gin does weird file-sniffing later...
 	if j.filename != nil {
 		g.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", *j.filename))
-
 	}
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Data(j.statusCode, j.mimetype, j.data)
 }
 
@@ -138,13 +325,45 @@ func (j downloadDataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j downloadDataHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j downloadDataHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j downloadDataHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j downloadDataHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(string(j.data))
+}
+
+func (j downloadDataHTTPResponse) ContentType() string {
+	return j.mimetype
+}
+
+func (j downloadDataHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type redirectHTTPResponse struct {
 	statusCode int
 	url        string
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j redirectHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Redirect(j.statusCode, j.url)
 }
 
@@ -153,13 +372,47 @@ func (j redirectHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j redirectHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j redirectHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j redirectHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j redirectHTTPResponse) BodyString(*gin.Context) *string {
+	return nil
+}
+
+func (j redirectHTTPResponse) ContentType() string {
+	return ""
+}
+
+func (j redirectHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type jsonAPIErrResponse struct {
 	err     *exerr.ExErr
 	headers []headerval
+	cookies []cookieval
 }
 
 func (j jsonAPIErrResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	j.err.Output(g)
+
+	j.err.CallListener(exerr.MethodOutput)
 }
 
 func (j jsonAPIErrResponse) WithHeader(k string, v string) HTTPResponse {
@@ -167,6 +420,39 @@ func (j jsonAPIErrResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j jsonAPIErrResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j jsonAPIErrResponse) IsSuccess() bool {
+	return false
+}
+
+func (j jsonAPIErrResponse) Statuscode() int {
+	return langext.Coalesce(j.err.RecursiveStatuscode(), 0)
+}
+
+func (j jsonAPIErrResponse) BodyString(*gin.Context) *string {
+	if str, err := j.err.ToDefaultAPIJson(); err == nil {
+		return &str
+	} else {
+		return nil
+	}
+}
+
+func (j jsonAPIErrResponse) ContentType() string {
+	return "application/json"
+}
+
+func (j jsonAPIErrResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+func (j jsonAPIErrResponse) Unwrap() error {
+	return j.err
+}
+
 func Status(sc int) HTTPResponse {
 	return &emptyHTTPResponse{statusCode: sc}
 }

ginext/routes.go

@@ -3,11 +3,9 @@ package ginext
 import (
 	"github.com/gin-gonic/gin"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
-	"gogs.mikescher.com/BlackForestBytes/goext/rext"
 	"net/http"
 	"path"
 	"reflect"
-	"regexp"
 	"runtime"
 	"strings"
 )
@@ -55,7 +53,11 @@ func (w *GinRoutesWrapper) Group(relativePath string) *GinRoutesWrapper {
 func (w *GinRoutesWrapper) Use(middleware ...gin.HandlerFunc) *GinRoutesWrapper {
 	defHandler := langext.ArrCopy(w.defaultHandler)
 	defHandler = append(defHandler, middleware...)
-	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler, absPath: w.absPath}
+}
+
+func (w *GinRoutesWrapper) WithJSONFilter(filter string) *GinRoutesWrapper {
+	return w.Use(func(g *gin.Context) { g.Set("goext.jsonfilter", filter) })
 }
 
 func (w *GinRoutesWrapper) GET(relativePath string) *GinRouteBuilder {
@@ -109,6 +111,10 @@ func (w *GinRouteBuilder) Use(middleware ...gin.HandlerFunc) *GinRouteBuilder {
 	return w
 }
 
+func (w *GinRouteBuilder) WithJSONFilter(filter string) *GinRouteBuilder {
+	return w.Use(func(g *gin.Context) { g.Set("goext.jsonfilter", filter) })
+}
+
 func (w *GinRouteBuilder) Handle(handler WHandlerFunc) {
 
 	if w.routes.wrapper.bufferBody {
@@ -181,12 +187,6 @@ func nameOfFunction(f any) string {
 		fname = fname[:len(fname)-len("-fm")]
 	}
 
-	suffix := rext.W(regexp.MustCompile(`\.func[0-9]+(?:\.[0-9]+)*$`))
-
-	if match, ok := suffix.MatchFirst(fname); ok {
-		fname = fname[:len(fname)-match.FullMatch().Length()]
-	}
-
 	return fname
 }
 

ginext/session.go

@@ -0,0 +1,11 @@
+package ginext
+
+import (
+	"context"
+	"github.com/gin-gonic/gin"
+)
+
+type SessionObject interface {
+	Init(g *gin.Context, ctx *AppContext) error
+	Finish(ctx context.Context, resp HTTPResponse) error
+}

go.mod

@@ -1,50 +1,57 @@
 module gogs.mikescher.com/BlackForestBytes/goext
 
-go 1.19
+go 1.21
 
 require (
 	github.com/gin-gonic/gin v1.9.1
+	github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.-
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/rs/xid v1.5.0
-	github.com/rs/zerolog v1.31.0
-	go.mongodb.org/mongo-driver v1.12.1
-	golang.org/x/crypto v0.13.0
-	golang.org/x/sys v0.12.0
-	golang.org/x/term v0.12.0
+	github.com/rs/zerolog v1.32.0
+	go.mongodb.org/mongo-driver v1.14.0
+	golang.org/x/crypto v0.21.0
+	golang.org/x/sys v0.18.0
+	golang.org/x/term v0.18.0
 )
 
 require (
-	github.com/bytedance/sonic v1.10.1 // indirect
+	github.com/bytedance/sonic v1.11.2 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
-	github.com/chenzhuoyu/iasm v0.9.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.15.4 // indirect
+	github.com/go-playground/validator/v10 v10.19.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.5.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	golang.org/x/arch v0.5.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.37.6 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/sqlite v1.28.0 // indirect
 )

go.sum

@@ -1,247 +1,238 @@
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
-github.com/bytedance/sonic v1.10.0-rc2 h1:oDfRZ+4m6AYCOC0GFeOCeYqvBmucy1isvouS2K0cPzo=
-github.com/bytedance/sonic v1.10.0-rc2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/bytedance/sonic v1.10.0-rc3 h1:uNSnscRapXTwUgTyOF0GVljYD08p9X/Lbr9MweSV3V0=
-github.com/bytedance/sonic v1.10.0-rc3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk=
-github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
-github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc=
-github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
+github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.0 h1:FwNNv6Vu4z2Onf1++LNzxB/QhitD8wuTdpZzMTGITWo=
+github.com/bytedance/sonic v1.11.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.1 h1:JC0+6c9FoWYYxakaoa+c5QTtJeiSZNeByOBhXtAFSn4=
+github.com/bytedance/sonic v1.11.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A=
+github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
-github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo=
 github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
+github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
+github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=
-github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw=
-github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM=
-github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.3 h1:S+sSpunYjNPDuXkWbK+x+bA7iXiW296KG4dL3X7xUZo=
-github.com/go-playground/validator/v10 v10.15.3/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
-github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs=
-github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
+github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74=
+github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U=
+github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
+github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
-github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
-github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
-github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
-github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
-github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
-github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
-github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
-github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
-github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
+github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
-github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
-github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
-github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
-go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
-go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
-go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
-go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
-go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
+go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
+go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
+go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc=
-golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y=
-golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
+golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
-golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
+golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
-golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw=
+modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
+modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.270"
+const GoextVersion = "0.0.402"
 
-const GoextVersionTimestamp = "2023-09-25T11:35:03+0200"
+const GoextVersionTimestamp = "2024-03-10T12:49:31+0100"

gojson/encode.go

@@ -156,7 +156,6 @@ import (
 // an error.
 func Marshal(v any) ([]byte, error) {
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
 
 	err := e.marshal(v, encOpts{escapeHTML: true})
 	if err != nil {
@@ -164,6 +163,8 @@ func Marshal(v any) ([]byte, error) {
 	}
 	buf := append([]byte(nil), e.Bytes()...)
 
+	encodeStatePool.Put(e)
+
 	return buf, nil
 }
 
@@ -174,9 +175,9 @@ type IndentOpt struct {
 
 // MarshalSafeCollections is like Marshal except it will marshal nil maps and
 // slices as '{}' and '[]' respectfully instead of 'null'
-func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt) ([]byte, error) {
+func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool, indent *IndentOpt, filter *string) ([]byte, error) {
 	e := &encodeState{}
-	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps})
+	err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps, filter: filter})
 	if err != nil {
 		return nil, err
 	}
@@ -393,6 +394,9 @@ type encOpts struct {
 	nilSafeSlices bool
 	// nilSafeMaps marshals a nil maps '{}' instead of 'null'
 	nilSafeMaps bool
+	// filter matches jsonfilter tag of struct
+	// marshals if no jsonfilter is set or otherwise if jsonfilter has the filter value
+	filter *string
 }
 
 type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
@@ -777,6 +781,8 @@ FieldLoop:
 
 		if f.omitEmpty && isEmptyValue(fv) {
 			continue
+		} else if opts.filter != nil && len(f.jsonfilter) > 0 && !f.jsonfilter.Contains(*opts.filter) {
+			continue
 		}
 		e.WriteByte(next)
 		next = ','
@@ -1220,15 +1226,28 @@ type field struct {
 	nameNonEsc  string // `"` + name + `":`
 	nameEscHTML string // `"` + HTMLEscape(name) + `":`
 
-	tag       bool
-	index     []int
-	typ       reflect.Type
-	omitEmpty bool
-	quoted    bool
+	tag        bool
+	index      []int
+	typ        reflect.Type
+	omitEmpty  bool
+	jsonfilter jsonfilter
+	quoted     bool
 
 	encoder encoderFunc
 }
 
+// jsonfilter stores the value of the jsonfilter struct tag
+type jsonfilter []string
+
+func (j jsonfilter) Contains(t string) bool {
+	for _, tag := range j {
+		if t == tag {
+			return true
+		}
+	}
+	return false
+}
+
 // byIndex sorts field by index sequence.
 type byIndex []field
 
@@ -1304,6 +1323,13 @@ func typeFields(t reflect.Type) structFields {
 				if !isValidTag(name) {
 					name = ""
 				}
+
+				var jsonfilter []string
+				jsonfilterTag := sf.Tag.Get("jsonfilter")
+				if jsonfilterTag != "" && jsonfilterTag != "-" {
+					jsonfilter = strings.Split(jsonfilterTag, ",")
+				}
+
 				index := make([]int, len(f.index)+1)
 				copy(index, f.index)
 				index[len(f.index)] = i
@@ -1334,12 +1360,13 @@ func typeFields(t reflect.Type) structFields {
 						name = sf.Name
 					}
 					field := field{
-						name:      name,
-						tag:       tagged,
-						index:     index,
-						typ:       ft,
-						omitEmpty: opts.Contains("omitempty"),
-						quoted:    quoted,
+						name:       name,
+						tag:        tagged,
+						index:      index,
+						typ:        ft,
+						omitEmpty:  opts.Contains("omitempty"),
+						jsonfilter: jsonfilter,
+						quoted:     quoted,
 					}
 					field.nameBytes = []byte(field.name)
 					field.equalFold = foldFunc(field.nameBytes)

gojson/encode_test.go

@@ -1253,6 +1253,10 @@ func TestMarshalSafeCollections(t *testing.T) {
 		nilMapStruct struct {
 			NilMap map[string]interface{} `json:"nil_map"`
 		}
+		testWithFilter struct {
+			Test1 string `json:"test1" jsonfilter:"FILTERONE"`
+			Test2 string `json:"test2" jsonfilter:"FILTERTWO"`
+		}
 	)
 
 	tests := []struct {
@@ -1271,10 +1275,12 @@ func TestMarshalSafeCollections(t *testing.T) {
 		{map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}"},
 		{pNilMap, "null"},
 		{nilMapStruct{}, "{\"nil_map\":{}}"},
+		{testWithFilter{}, "{\"test1\":\"\"}"},
 	}
 
+	filter := "FILTERONE"
 	for i, tt := range tests {
-		b, err := MarshalSafeCollections(tt.in, true, true, nil)
+		b, err := MarshalSafeCollections(tt.in, true, true, nil, &filter)
 		if err != nil {
 			t.Errorf("test %d, unexpected failure: %v", i, err)
 		}

gojson/fold.go

@@ -97,7 +97,10 @@ func equalFoldRight(s, t []byte) bool {
 		t = t[size:]
 
 	}
-	return len(t) == 0
+	if len(t) > 0 {
+		return false
+	}
+	return true
 }
 
 // asciiEqualFold is a specialization of bytes.EqualFold for use when

gojson/fold_test.go

@@ -52,7 +52,9 @@ func TestFold(t *testing.T) {
 }
 
 func TestFoldAgainstUnicode(t *testing.T) {
-	var buf1, buf2 []byte
+	const bufSize = 5
+	buf1 := make([]byte, 0, bufSize)
+	buf2 := make([]byte, 0, bufSize)
 	var runes []rune
 	for i := 0x20; i <= 0x7f; i++ {
 		runes = append(runes, rune(i))
@@ -94,8 +96,12 @@ func TestFoldAgainstUnicode(t *testing.T) {
 				continue
 			}
 			for _, r2 := range runes {
-				buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x')
-				buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x')
+				buf1 := append(buf1[:0], 'x')
+				buf2 := append(buf2[:0], 'x')
+				buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
+				buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
+				buf1 = append(buf1, 'x')
+				buf2 = append(buf2, 'x')
 				want := bytes.EqualFold(buf1, buf2)
 				if got := ff.fold(buf1, buf2); got != want {
 					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)

gojson/gionic.go

@@ -17,6 +17,7 @@ type GoJsonRender struct {
 	NilSafeSlices bool
 	NilSafeMaps   bool
 	Indent        *IndentOpt
+	Filter        *string
 }
 
 func (r GoJsonRender) Render(w http.ResponseWriter) error {
@@ -25,7 +26,7 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
 		header["Content-Type"] = []string{"application/json; charset=utf-8"}
 	}
 
-	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent)
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
 	if err != nil {
 		panic(err)
 	}
@@ -36,6 +37,14 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
 	return nil
 }
 
+func (r GoJsonRender) RenderString() (string, error) {
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
+	if err != nil {
+		panic(err)
+	}
+	return string(jsonBytes), nil
+}
+
 func (r GoJsonRender) WriteContentType(w http.ResponseWriter) {
 	header := w.Header()
 	if val := header["Content-Type"]; len(val) == 0 {

gojson/number_test.go

@@ -116,3 +116,18 @@ func TestNumberIsValid(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkNumberIsValid(b *testing.B) {
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		isValidNumber(s)
+	}
+}
+
+func BenchmarkNumberIsValidRegexp(b *testing.B) {
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		jsonNumberRegexp.MatchString(s)
+	}
+}

gojson/scanner.go

@@ -594,7 +594,7 @@ func (s *scanner) error(c byte, context string) int {
 	return scanError
 }
 
-// quoteChar formats c as a quoted character literal.
+// quoteChar formats c as a quoted character literal
 func quoteChar(c byte) string {
 	// special cases - different from quoted strings
 	if c == '\'' {

gojson/stream.go

@@ -179,11 +179,9 @@ func nonSpace(b []byte) bool {
 
 // An Encoder writes JSON values to an output stream.
 type Encoder struct {
-	w             io.Writer
-	err           error
-	escapeHTML    bool
-	nilSafeSlices bool
-	nilSafeMaps   bool
+	w          io.Writer
+	err        error
+	escapeHTML bool
 
 	indentBuf    *bytes.Buffer
 	indentPrefix string
@@ -204,11 +202,8 @@ func (enc *Encoder) Encode(v any) error {
 	if enc.err != nil {
 		return enc.err
 	}
-
 	e := newEncodeState()
-	defer encodeStatePool.Put(e)
-
-	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
+	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
 	if err != nil {
 		return err
 	}
@@ -236,6 +231,7 @@ func (enc *Encoder) Encode(v any) error {
 	if _, err = enc.w.Write(b); err != nil {
 		enc.err = err
 	}
+	encodeStatePool.Put(e)
 	return err
 }
 
@@ -247,13 +243,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) {
 	enc.indentValue = indent
 }
 
-// SetNilSafeCollection specifies whether to represent nil slices and maps as
-// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
-func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
-	enc.nilSafeSlices = nilSafeSlices
-	enc.nilSafeMaps = nilSafeMaps
-}
-
 // SetEscapeHTML specifies whether problematic HTML characters
 // should be escaped inside JSON quoted strings.
 // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e

gojson/stream_test.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
-	"runtime/debug"
 	"strings"
 	"testing"
 )
@@ -42,7 +41,7 @@ false
 
 func TestEncoder(t *testing.T) {
 	for i := 0; i <= len(streamTest); i++ {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		// Check that enc.SetIndent("", "") turns off indentation.
 		enc.SetIndent(">", ".")
@@ -60,43 +59,6 @@ func TestEncoder(t *testing.T) {
 	}
 }
 
-func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
-	// Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test.
-	percent := debug.SetGCPercent(-1)
-	defer debug.SetGCPercent(percent)
-
-	// Trigger an error in Marshal with cyclic data.
-	type Dummy struct {
-		Name string
-		Next *Dummy
-	}
-	dummy := Dummy{Name: "Dummy"}
-	dummy.Next = &dummy
-
-	var buf bytes.Buffer
-	enc := NewEncoder(&buf)
-	if err := enc.Encode(dummy); err == nil {
-		t.Errorf("Encode(dummy) == nil; want error")
-	}
-
-	type Data struct {
-		A string
-		I int
-	}
-	data := Data{A: "a", I: 1}
-	if err := enc.Encode(data); err != nil {
-		t.Errorf("Marshal(%v) = %v", data, err)
-	}
-
-	var data2 Data
-	if err := Unmarshal(buf.Bytes(), &data2); err != nil {
-		t.Errorf("Unmarshal(%v) = %v", data2, err)
-	}
-	if data2 != data {
-		t.Errorf("expect: %v, but get: %v", data, data2)
-	}
-}
-
 var streamEncodedIndent = `0.1
 "hello"
 null
@@ -115,7 +77,7 @@ false
 `
 
 func TestEncoderIndent(t *testing.T) {
-	var buf strings.Builder
+	var buf bytes.Buffer
 	enc := NewEncoder(&buf)
 	enc.SetIndent(">", ".")
 	for _, v := range streamTest {
@@ -185,7 +147,7 @@ func TestEncoderSetEscapeHTML(t *testing.T) {
 			`{"bar":"\"<html>foobar</html>\""}`,
 		},
 	} {
-		var buf strings.Builder
+		var buf bytes.Buffer
 		enc := NewEncoder(&buf)
 		if err := enc.Encode(tt.v); err != nil {
 			t.Errorf("Encode(%s): %s", tt.name, err)
@@ -347,6 +309,21 @@ func TestBlocking(t *testing.T) {
 	}
 }
 
+func BenchmarkEncoderEncode(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X, Y string
+	}
+	v := &T{"foo", "bar"}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			if err := NewEncoder(io.Discard).Encode(v); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
 type tokenStreamCase struct {
 	json      string
 	expTokens []any
@@ -495,45 +472,3 @@ func TestHTTPDecoding(t *testing.T) {
 		t.Errorf("err = %v; want io.EOF", err)
 	}
 }
-
-func TestEncoderSetNilSafeCollection(t *testing.T) {
-	var (
-		nilSlice  []interface{}
-		pNilSlice *[]interface{}
-		nilMap    map[string]interface{}
-		pNilMap   *map[string]interface{}
-	)
-	for _, tt := range []struct {
-		name        string
-		v           interface{}
-		want        string
-		rescuedWant string
-	}{
-		{"nilSlice", nilSlice, "null", "[]"},
-		{"nonNilSlice", []interface{}{}, "[]", "[]"},
-		{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
-		{"pNilSlice", pNilSlice, "null", "null"},
-		{"nilMap", nilMap, "null", "{}"},
-		{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
-		{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
-		{"pNilMap", pNilMap, "null", "null"},
-	} {
-		var buf bytes.Buffer
-		enc := NewEncoder(&buf)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.want {
-			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
-		}
-		buf.Reset()
-		enc.SetNilSafeCollection(true, true)
-		if err := enc.Encode(tt.v); err != nil {
-			t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
-		}
-		if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
-			t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
-				tt.name, got, tt.want)
-		}
-	}
-}

googleapi/README.md

@@ -0,0 +1,54 @@
+
+Google OAuth Setup (to send mails)
+==================================
+
+
+ - Login @ https://console.cloud.google.com
+
+ - GMail API akivieren: https://console.cloud.google.com/apis/library/gmail.googleapis.com?
+
+ - Create new Project (aka 'BackendMailAPI') @ https://console.cloud.google.com/projectcreate
+   User Type: Intern
+   Anwendungsname: 'BackendMailAPI'
+   Support-Email: ...
+   Authorisierte Domains: 'heydyno.de' (or project domain)
+   Kontakt-Email: ...
+   
+
+ - Unter "Anmeldedaten" neuer OAuth Client erstellen @ https://console.cloud.google.com/apis/credentials
+   Anwendungstyp: Web
+   Name: 'BackendMailOAuth'
+   Redirect-Uri: 'http://localhost/oauth'
+   Client-ID und Client-Key merken
+
+ - Open in Browser:
+   https://accounts.google.com/o/oauth2/v2/auth?redirect_uri=http://localhost/oauth&prompt=consent&response_type=code&client_id={...}&scope=https://www.googleapis.com/auth/gmail.send&access_type=offline
+   Code aus redirected URI merken
+
+ - Code via request einlösen (und refresh_roken merken):
+
+```
+curl --request POST \
+  --url https://oauth2.googleapis.com/token \
+  --data code={...} \
+  --data redirect_uri=http://localhost/oauth \
+  --data client_id={...} \
+  --data client_secret={...} \
+  --data grant_type=authorization_code \
+  --data scope=https://www.googleapis.com/auth/gmail.send
+```
+
+ - Fertig, mit `client_id`, `client_secret` und `refresh_token` kann das package benutzt werden
+
+
+
+
+
+
+
+
+
+
+
+
+

googleapi/attachment.go

@@ -0,0 +1,46 @@
+package googleapi
+
+import (
+	"encoding/base64"
+	"fmt"
+)
+
+type MailAttachment struct {
+	IsInline    bool
+	ContentType string
+	Filename    string
+	Data        []byte
+}
+
+func (a MailAttachment) dump() []string {
+	res := make([]string, 0, 4)
+
+	if a.ContentType != "" {
+		res = append(res, "Content-Type: "+a.ContentType+"; charset=UTF-8")
+	}
+
+	res = append(res, "Content-Transfer-Encoding: base64")
+
+	if a.IsInline {
+		if a.Filename != "" {
+			res = append(res, fmt.Sprintf("Content-Disposition: inline;filename=\"%s\"", a.Filename))
+		} else {
+			res = append(res, "Content-Disposition: inline")
+		}
+	} else {
+		if a.Filename != "" {
+			res = append(res, fmt.Sprintf("Content-Disposition: attachment;filename=\"%s\"", a.Filename))
+		} else {
+			res = append(res, "Content-Disposition: attachment")
+		}
+	}
+
+	b64 := base64.StdEncoding.EncodeToString(a.Data)
+	for i := 0; i < len(b64); i += 80 {
+		res = append(res, b64[i:min(i+80, len(b64))])
+	}
+
+	res = append(res)
+
+	return res
+}

googleapi/body.go

@@ -0,0 +1,6 @@
+package googleapi
+
+type MailBody struct {
+	Plain string
+	HTML  string
+}

googleapi/mimeMessage.go

@@ -0,0 +1,224 @@
+package googleapi
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"mime"
+	"strings"
+	"time"
+)
+
+// https://datatracker.ietf.org/doc/html/rfc2822
+func encodeMimeMail(from string, recipients []string, cc []string, bcc []string, subject string, body MailBody, attachments []MailAttachment) string {
+
+	data := make([]string, 0, 32)
+
+	data = append(data, "Date: "+time.Now().Format(time.RFC1123Z))
+	data = append(data, "MIME-Version: 1.0")
+	data = append(data, "From: "+mime.QEncoding.Encode("UTF-8", from))
+	data = append(data, "To: "+strings.Join(langext.ArrMap(recipients, func(v string) string { return mime.QEncoding.Encode("UTF-8", v) }), ", "))
+	if len(cc) > 0 {
+		data = append(data, "To: "+strings.Join(langext.ArrMap(cc, func(v string) string { return mime.QEncoding.Encode("UTF-8", v) }), ", "))
+	}
+	if len(bcc) > 0 {
+		data = append(data, "Bcc: "+strings.Join(langext.ArrMap(bcc, func(v string) string { return mime.QEncoding.Encode("UTF-8", v) }), ", "))
+	}
+	data = append(data, "Subject: "+mime.QEncoding.Encode("UTF-8", subject))
+
+	hasInlineAttachments := langext.ArrAny(attachments, func(v MailAttachment) bool { return v.IsInline })
+	hasNormalAttachments := langext.ArrAny(attachments, func(v MailAttachment) bool { return !v.IsInline })
+	hasPlain := body.Plain != ""
+	hasHTML := body.HTML != ""
+
+	mixedBoundary := langext.MustRawHexUUID()
+	relatedBoundary := langext.MustRawHexUUID()
+	altBoundary := langext.MustRawHexUUID()
+
+	inlineAttachments := langext.ArrFilter(attachments, func(v MailAttachment) bool { return v.IsInline })
+	normalAttachments := langext.ArrFilter(attachments, func(v MailAttachment) bool { return !v.IsInline })
+
+	if hasInlineAttachments && hasNormalAttachments {
+		// "mixed+related"
+
+		data = append(data, "Content-Type: multipart/mixed; boundary="+mixedBoundary)
+		data = append(data, "")
+		data = append(data, "--"+mixedBoundary)
+
+		data = append(data, "Content-Type: multipart/related; boundary="+relatedBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, relatedBoundary, altBoundary)...)
+		data = append(data, "")
+
+		for i, attachment := range inlineAttachments {
+			data = append(data, "--"+relatedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(inlineAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+relatedBoundary+"--")
+
+		for i, attachment := range normalAttachments {
+			data = append(data, "--"+mixedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(normalAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+mixedBoundary+"--")
+
+	} else if hasNormalAttachments {
+		// "mixed"
+
+		data = append(data, "Content-Type: multipart/mixed; boundary="+mixedBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, mixedBoundary, altBoundary)...)
+		if hasPlain && hasHTML {
+			data = append(data, "")
+		}
+
+		for i, attachment := range normalAttachments {
+			data = append(data, "--"+mixedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(normalAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+mixedBoundary+"--")
+
+	} else if hasInlineAttachments {
+		// "related"
+
+		data = append(data, "Content-Type: multipart/related; boundary="+relatedBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, relatedBoundary, altBoundary)...)
+		data = append(data, "")
+
+		for i, attachment := range inlineAttachments {
+			data = append(data, "--"+relatedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(inlineAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+relatedBoundary+"--")
+
+	} else if hasPlain && hasHTML {
+		// "alternative"
+
+		data = append(data, "Content-Type: multipart/alternative; boundary="+altBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, altBoundary, altBoundary)...)
+		data = append(data, "")
+
+		data = append(data, "--"+altBoundary+"--")
+
+	} else if hasPlain {
+		// "plain"
+
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+
+	} else if hasHTML {
+		// "plain"
+
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+
+	} else {
+		// "empty??"
+
+	}
+
+	return strings.Join(data, "\r\n")
+}
+
+func dumpMailBody(body MailBody, hasInlineAttachments bool, hasNormalAttachments bool, boundary string, boundaryAlt string) []string {
+
+	if body.HTML != "" && body.Plain != "" && !hasInlineAttachments && hasNormalAttachments {
+		data := make([]string, 0, 16)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: multipart/alternative; boundary="+boundaryAlt)
+		data = append(data, "")
+		data = append(data, "--"+boundaryAlt)
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+		data = append(data, "")
+		data = append(data, "--"+boundaryAlt)
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+		data = append(data, "")
+		data = append(data, "--"+boundaryAlt+"--")
+		return data
+	}
+
+	if body.HTML != "" && body.Plain != "" && hasInlineAttachments {
+		data := make([]string, 0, 2)
+		data = append(data, "--"+boundary)
+		data = append(data, body.HTML)
+		return data
+	}
+
+	if body.HTML != "" && body.Plain != "" {
+		data := make([]string, 0, 8)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+		data = append(data, "")
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+		return data
+	}
+
+	if body.HTML != "" {
+		data := make([]string, 0, 2)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+		return data
+	}
+
+	if body.Plain != "" {
+		data := make([]string, 0, 2)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+		return data
+	}
+
+	data := make([]string, 0, 16)
+	data = append(data, "--"+boundary)
+	data = append(data, "Content-Type: text/plain; charset=UTF-8")
+	data = append(data, "Content-Transfer-Encoding: 7bit")
+	data = append(data, "")
+	data = append(data, "") // no content ?!?
+	return data
+}

googleapi/mimeMessage_test.go

@@ -0,0 +1,80 @@
+package googleapi
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"testing"
+)
+
+func TestEncodeMimeMail(t *testing.T) {
+
+	mail := encodeMimeMail(
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail",
+		MailBody{Plain: "Plain Text"},
+		nil)
+
+	verifyMime(mail)
+}
+
+func TestEncodeMimeMail2(t *testing.T) {
+
+	mail := encodeMimeMail(
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail (alternative)",
+		MailBody{
+			Plain: "Plain Text",
+			HTML:  "<html><body><u>Non</u> Pl<i>ai</i>n T<b>ex</b>t</body></html>",
+		},
+		nil)
+
+	verifyMime(mail)
+}
+
+func TestEncodeMimeMail3(t *testing.T) {
+
+	mail := encodeMimeMail(
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail (alternative)",
+		MailBody{
+			HTML: "<html><body><u>Non</u> Pl<i>ai</i>n T<b>ex</b>t</body></html>",
+		},
+		[]MailAttachment{
+			{Data: []byte("HelloWorld"), Filename: "test.txt", IsInline: false, ContentType: "text/plain"},
+		})
+
+	verifyMime(mail)
+}
+
+func TestEncodeMimeMail4(t *testing.T) {
+
+	b := tst.Must(os.ReadFile("test_placeholder.png"))(t)
+
+	mail := encodeMimeMail(
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail (inline)",
+		MailBody{
+			HTML: "<html><body><u>Non</u> Pl<i>ai</i>n T<b>ex</b>t</body></html>",
+		},
+		[]MailAttachment{
+			{Data: b, Filename: "img.png", IsInline: true, ContentType: "image/png"},
+		})
+
+	verifyMime(mail)
+}
+
+func verifyMime(mail string) {
+	//fmt.Printf("%s\n\n", mail)
+}

googleapi/oAuth.go

@@ -0,0 +1,91 @@
+package googleapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type GoogleOAuth interface {
+	AccessToken() (string, error)
+}
+
+type oauth struct {
+	clientID     string
+	clientSecret string
+	refreshToken string
+
+	lock        sync.RWMutex
+	accessToken *string
+	expiryDate  *time.Time
+}
+
+func NewGoogleOAuth(clientid string, clientsecret, refreshtoken string) GoogleOAuth {
+	return &oauth{
+		clientID:     clientid,
+		clientSecret: clientsecret,
+		refreshToken: refreshtoken,
+	}
+}
+
+func (c *oauth) AccessToken() (string, error) {
+	c.lock.RLock()
+	if c.accessToken != nil && c.expiryDate != nil && (*c.expiryDate).After(time.Now()) {
+		c.lock.RUnlock()
+		return *c.accessToken, nil // still valid
+	}
+	c.lock.RUnlock()
+
+	httpclient := http.Client{}
+
+	url := fmt.Sprintf("https://oauth2.googleapis.com/token?client_id=%s&client_secret=%s&grant_type=%s&refresh_token=%s",
+		c.clientID,
+		c.clientSecret,
+		"refresh_token",
+		c.refreshToken)
+
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	reqStartTime := time.Now()
+
+	res, err := httpclient.Do(req)
+
+	type response struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int    `json:"expires_in"`
+		Scope       string `json:"scope"`
+		TokenType   string `json:"token_type"`
+	}
+
+	var r response
+
+	data, err := io.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(data, &r)
+	if err != nil {
+		return "", err
+	}
+
+	if r.ExpiresIn == 0 || r.AccessToken == "" {
+		return "", exerr.New(exerr.TypeGoogleResponse, "google oauth returned no response").Str("body", string(data)).Build()
+	}
+
+	c.lock.Lock()
+	c.expiryDate = langext.Ptr(reqStartTime.Add(timeext.FromSeconds(r.ExpiresIn - 10)))
+	c.accessToken = langext.Ptr(r.AccessToken)
+	c.lock.Unlock()
+
+	return r.AccessToken, nil
+}

googleapi/sendMail.go

@@ -0,0 +1,69 @@
+package googleapi
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"io"
+	"net/http"
+)
+
+type MailRef struct {
+	ID       string   `json:"id"`
+	ThreadID string   `json:"threadId"`
+	LabelIDs []string `json:"labelIds"`
+}
+
+func (c *client) SendMail(ctx context.Context, from string, recipients []string, cc []string, bcc []string, subject string, body MailBody, attachments []MailAttachment) (MailRef, error) {
+
+	mm := encodeMimeMail(from, recipients, cc, bcc, subject, body, attachments)
+
+	tok, err := c.oauth.AccessToken()
+	if err != nil {
+		return MailRef{}, exerr.Wrap(err, "").Build()
+	}
+
+	url := fmt.Sprintf("https://gmail.googleapis.com/gmail/v1/users/%s/messages/send?alt=json&prettyPrint=false", "me")
+
+	msgbody, err := json.Marshal(langext.H{"raw": base64.URLEncoding.EncodeToString([]byte(mm))})
+	if err != nil {
+		return MailRef{}, exerr.Wrap(err, "").Build()
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(msgbody))
+	if err != nil {
+		return MailRef{}, exerr.Wrap(err, "").Build()
+	}
+
+	req.Header.Add("Authorization", "Bearer "+tok)
+	req.Header.Add("X-Goog-Api-Client", "blackforestbytes-goext/"+goext.GoextVersion)
+	req.Header.Add("User-Agent", "blackforestbytes-goext/"+goext.GoextVersion)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return MailRef{}, exerr.Wrap(err, "").Build()
+	}
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return MailRef{}, exerr.Wrap(err, "").Build()
+	}
+
+	if resp.StatusCode != 200 {
+		return MailRef{}, exerr.New(exerr.TypeGoogleStatuscode, "gmail returned non-200 statuscode").Int("sc", resp.StatusCode).Str("body", string(respBody)).Build()
+	}
+
+	var respObj MailRef
+	err = json.Unmarshal(respBody, &respObj)
+	if err != nil {
+		return MailRef{}, exerr.Wrap(err, "").Str("body", string(respBody)).Build()
+	}
+
+	return respObj, nil
+}

googleapi/sendMail_test.go

@@ -0,0 +1,151 @@
+package googleapi
+
+import (
+	"context"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	if !exerr.Initialized() {
+		exerr.Init(exerr.ErrorPackageConfigInit{ZeroLogErrTraces: langext.PFalse, ZeroLogAllTraces: langext.PFalse})
+	}
+	os.Exit(m.Run())
+}
+
+func TestSendMail1(t *testing.T) {
+	t.Skip()
+	return
+
+	auth := NewGoogleOAuth(
+		"554617284247-8di0j6s5dcmlk4lmk4hdf9kdn8scss54.apps.googleusercontent.com",
+		"TODO",
+		"TODO")
+
+	ctx := context.Background()
+
+	gclient := NewGoogleClient(auth)
+
+	mail, err := gclient.SendMail(
+		ctx,
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail",
+		MailBody{Plain: "Plain Text"},
+		nil)
+
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("mail.ID        := %s\n", mail.ID)
+	fmt.Printf("mail.ThreadID  := %s\n", mail.ThreadID)
+	fmt.Printf("mail.LabelIDs  := %v\n", mail.LabelIDs)
+}
+
+func TestSendMail2(t *testing.T) {
+	t.Skip()
+	return
+
+	auth := NewGoogleOAuth(
+		"554617284247-8di0j6s5dcmlk4lmk4hdf9kdn8scss54.apps.googleusercontent.com",
+		"TODO",
+		"TODO")
+
+	ctx := context.Background()
+
+	gclient := NewGoogleClient(auth)
+
+	mail, err := gclient.SendMail(
+		ctx,
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail (alternative)",
+		MailBody{
+			Plain: "Plain Text",
+			HTML:  "<html><body><u>Non</u> Pl<i>ai</i>n T<b>ex</b>t</body></html>",
+		},
+		nil)
+
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("mail.ID        := %s\n", mail.ID)
+	fmt.Printf("mail.ThreadID  := %s\n", mail.ThreadID)
+	fmt.Printf("mail.LabelIDs  := %v\n", mail.LabelIDs)
+}
+
+func TestSendMail3(t *testing.T) {
+	t.Skip()
+	return
+
+	auth := NewGoogleOAuth(
+		"554617284247-8di0j6s5dcmlk4lmk4hdf9kdn8scss54.apps.googleusercontent.com",
+		"TODO",
+		"TODO")
+
+	ctx := context.Background()
+
+	gclient := NewGoogleClient(auth)
+
+	mail, err := gclient.SendMail(
+		ctx,
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail (attach)",
+		MailBody{
+			HTML: "<html><body><u>Non</u> Pl<i>ai</i>n T<b>ex</b>t</body></html>",
+		},
+		[]MailAttachment{
+			{Data: []byte("HelloWorld"), Filename: "test.txt", IsInline: false, ContentType: "text/plain"},
+		})
+
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("mail.ID        := %s\n", mail.ID)
+	fmt.Printf("mail.ThreadID  := %s\n", mail.ThreadID)
+	fmt.Printf("mail.LabelIDs  := %v\n", mail.LabelIDs)
+}
+
+func TestSendMail4(t *testing.T) {
+	t.Skip()
+	return
+
+	auth := NewGoogleOAuth(
+		"554617284247-8di0j6s5dcmlk4lmk4hdf9kdn8scss54.apps.googleusercontent.com",
+		"TODO",
+		"TODO")
+
+	ctx := context.Background()
+
+	gclient := NewGoogleClient(auth)
+
+	b := tst.Must(os.ReadFile("test_placeholder.png"))(t)
+
+	mail, err := gclient.SendMail(
+		ctx,
+		"noreply@heydyno.de",
+		[]string{"trash@mikescher.de"},
+		nil,
+		nil,
+		"Hello Test Mail (inline)",
+		MailBody{
+			HTML: "<html><body><u>Non</u> Pl<i>ai</i>n T<b>ex</b>t</body></html>",
+		},
+		[]MailAttachment{
+			{Data: b, Filename: "img.png", IsInline: true, ContentType: "image/png"},
+		})
+
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("mail.ID        := %s\n", mail.ID)
+	fmt.Printf("mail.ThreadID  := %s\n", mail.ThreadID)
+	fmt.Printf("mail.LabelIDs  := %v\n", mail.LabelIDs)
+}

googleapi/service.go

@@ -0,0 +1,22 @@
+package googleapi
+
+import (
+	"context"
+	"net/http"
+)
+
+type GoogleClient interface {
+	SendMail(ctx context.Context, from string, recipients []string, cc []string, bcc []string, subject string, body MailBody, attachments []MailAttachment) (MailRef, error)
+}
+
+type client struct {
+	oauth GoogleOAuth
+	http  http.Client
+}
+
+func NewGoogleClient(oauth GoogleOAuth) GoogleClient {
+	return &client{
+		oauth: oauth,
+		http:  http.Client{},
+	}
+}

langext/array.go

@@ -265,6 +265,15 @@ func ArrFirstIndex[T comparable](arr []T, needle T) int {
 	return -1
 }
 
+func ArrFirstIndexFunc[T any](arr []T, comp func(v T) bool) int {
+	for i, v := range arr {
+		if comp(v) {
+			return i
+		}
+	}
+	return -1
+}
+
 func ArrLastIndex[T comparable](arr []T, needle T) int {
 	result := -1
 	for i, v := range arr {
@@ -275,6 +284,16 @@ func ArrLastIndex[T comparable](arr []T, needle T) int {
 	return result
 }
 
+func ArrLastIndexFunc[T any](arr []T, comp func(v T) bool) int {
+	result := -1
+	for i, v := range arr {
+		if comp(v) {
+			result = i
+		}
+	}
+	return result
+}
+
 func AddToSet[T comparable](set []T, add T) []T {
 	for _, v := range set {
 		if v == add {
@@ -479,3 +498,33 @@ func JoinString(arr []string, delimiter string) string {
 
 	return str
 }
+
+// ArrChunk splits the array into buckets of max-size `chunkSize`
+// order is being kept.
+// The last chunk may contain less than length elements.
+//
+// (chunkSize == -1) means no chunking
+//
+// see https://www.php.net/manual/en/function.array-chunk.php
+func ArrChunk[T any](arr []T, chunkSize int) [][]T {
+	if chunkSize == -1 {
+		return [][]T{arr}
+	}
+
+	res := make([][]T, 0, 1+len(arr)/chunkSize)
+
+	i := 0
+	for i < len(arr) {
+
+		right := i + chunkSize
+		if right >= len(arr) {
+			right = len(arr)
+		}
+
+		res = append(res, arr[i:right])
+
+		i = right
+	}
+
+	return res
+}

langext/baseAny.go

@@ -0,0 +1,178 @@
+package langext
+
+import (
+	"crypto/rand"
+	"errors"
+	"math"
+	"math/big"
+)
+
+type AnyBaseConverter struct {
+	base    uint64
+	charset []rune
+}
+
+func NewAnyBaseConverter(cs string) AnyBaseConverter {
+	rcs := []rune(cs)
+	return AnyBaseConverter{
+		base:    uint64(len(rcs)),
+		charset: rcs,
+	}
+}
+
+func (bc AnyBaseConverter) Rand(rlen int) string {
+	biBase := big.NewInt(int64(bc.base))
+
+	randMax := big.NewInt(math.MaxInt64)
+
+	r := ""
+
+	for i := 0; i < rlen; i++ {
+		v, err := rand.Int(rand.Reader, randMax)
+		if err != nil {
+			panic(err)
+		}
+
+		r += string(bc.charset[v.Mod(v, biBase).Int64()])
+	}
+
+	return r
+}
+
+func (bc AnyBaseConverter) EncodeUInt64(num uint64) string {
+	if num == 0 {
+		return "0"
+	}
+
+	b := ""
+
+	// loop as long the num is bigger than zero
+	for num > 0 {
+		r := num % bc.base
+
+		num -= r
+		num /= base62Base
+
+		b += string(bc.charset[int(r)])
+	}
+
+	return b
+}
+
+func (bc AnyBaseConverter) DecodeUInt64(str string) (uint64, error) {
+	if str == "" {
+		return 0, errors.New("empty string")
+	}
+
+	result := uint64(0)
+
+	for _, v := range str {
+		result *= base62Base
+
+		pos := ArrFirstIndex(bc.charset, v)
+		if pos == -1 {
+			return 0, errors.New("invalid character: " + string(v))
+		}
+
+		result += uint64(pos)
+	}
+
+	return result, nil
+}
+
+func (bc AnyBaseConverter) Encode(src []byte) string {
+	value := new(big.Int)
+	value.SetBytes(src)
+	return bc.EncodeBigInt(value)
+}
+
+func (bc AnyBaseConverter) EncodeBigInt(src *big.Int) string {
+	value := new(big.Int)
+	value.Set(src)
+
+	isneg := value.Sign() < 0
+
+	answer := ""
+
+	if isneg {
+		value.Neg(value)
+	}
+
+	biBase := big.NewInt(int64(bc.base))
+
+	rem := new(big.Int)
+
+	for value.Sign() > 0 {
+		value.QuoRem(value, biBase, rem)
+		answer = string(bc.charset[rem.Int64()]) + answer
+	}
+
+	if isneg {
+		return "-" + answer
+	} else {
+		return answer
+	}
+}
+
+func (bc AnyBaseConverter) Decode(src string) ([]byte, error) {
+	value, err := bc.DecodeToBigInt(src)
+	if err != nil {
+		return nil, err
+	}
+	return value.Bytes(), nil
+}
+
+func (bc AnyBaseConverter) DecodeToBigInt(_src string) (*big.Int, error) {
+	result := new(big.Int)
+	result.SetInt64(0)
+
+	src := []rune(_src)
+
+	if len(src) == 0 {
+		return nil, errors.New("string is empty")
+	}
+	if bc.base < 2 {
+		return nil, errors.New("not enough digits")
+	}
+
+	i := 0
+
+	sign := new(big.Int)
+	sign.SetInt64(1)
+	if src[i] == '+' {
+		i++
+	} else if src[i] == '-' {
+		i++
+		sign.SetInt64(-1)
+	}
+
+	if i >= len(src) {
+		return nil, errors.New("no digits in input")
+	}
+
+	biBase := big.NewInt(int64(bc.base))
+
+	oldResult := new(big.Int)
+
+	for ; i < len(src); i++ {
+		n := ArrFirstIndex(bc.charset, src[i])
+		if n < 0 {
+			return nil, errors.New("invalid characters in input")
+		}
+
+		oldResult.Set(result)
+
+		result.Mul(result, biBase)
+		result.Add(result, big.NewInt(int64(n)))
+
+		if result.Cmp(oldResult) < 0 {
+			return nil, errors.New("overflow")
+		}
+	}
+
+	if sign.Cmp(big.NewInt(0)) < 0 {
+		result.Neg(result)
+	}
+
+	return result, nil
+}

langext/baseAny_test.go

@@ -0,0 +1,80 @@
+package langext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func _anyEncStr(bc AnyBaseConverter, v string) string {
+	vr := bc.Encode([]byte(v))
+	return vr
+}
+
+func _anyDecStr(bc AnyBaseConverter, v string) string {
+	vr, err := bc.Decode(v)
+	if err != nil {
+		panic(err)
+	}
+	return string(vr)
+}
+
+func TestAnyBase58DefaultEncoding(t *testing.T) {
+	tst.AssertEqual(t, _anyEncStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "Hello"), "9Ajdvzr")
+	tst.AssertEqual(t, _anyEncStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX")
+}
+
+func TestAnyBase58DefaultDecoding(t *testing.T) {
+	tst.AssertEqual(t, _anyDecStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "9Ajdvzr"), "Hello")
+	tst.AssertEqual(t, _anyDecStr(NewAnyBaseConverter("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestAnyBaseDecode(t *testing.T) {
+
+	const (
+		Binary  = "01"
+		Decimal = "0123456789"
+		Hex     = "0123456789ABCDEF"
+		DNA     = "ACGT"
+		Base32  = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+		Base58  = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+		Base62  = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+		Base64  = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+		Base256 = "🚀🪐☄🛰🌌🌑🌒🌓🌔🌕🌖🌗🌘🌍🌏🌎🐉☀💻🖥💾💿😂❤😍🤣😊🙏💕😭😘👍😅👏😁🔥🥰💔💖💙😢🤔😆🙄💪😉☺👌🤗💜😔😎😇🌹🤦🎉💞✌✨🤷😱😌🌸🙌😋💗💚😏💛🙂💓🤩😄😀🖤😃💯🙈👇🎶😒🤭❣😜💋👀😪😑💥🙋😞😩😡🤪👊🥳😥🤤👉💃😳✋😚😝😴🌟😬🙃🍀🌷😻😓⭐✅🥺🌈😈🤘💦✔😣🏃💐☹🎊💘😠☝😕🌺🎂🌻😐🖕💝🙊😹🗣💫💀👑🎵🤞😛🔴😤🌼😫⚽🤙☕🏆🤫👈😮🙆🍻🍃🐶💁😲🌿🧡🎁⚡🌞🎈❌✊👋😰🤨😶🤝🚶💰🍓💢🤟🙁🚨💨🤬✈🎀🍺🤓😙💟🌱😖👶🥴▶➡❓💎💸⬇😨🌚🦋😷🕺⚠🙅😟😵👎🤲🤠🤧📌🔵💅🧐🐾🍒😗🤑🌊🤯🐷☎💧😯💆👆🎤🙇🍑❄🌴💣🐸💌📍🥀🤢👅💡💩👐📸👻🤐🤮🎼🥵🚩🍎🍊👼💍📣🥂"
+	)
+
+	type TestDef struct {
+		FromCS  string
+		FromVal string
+		ToCS    string
+		ToVal   string
+	}
+
+	defs := []TestDef{
+		{Binary, "10100101011100000101010", Decimal, "5421098"},
+		{Decimal, "5421098", DNA, "CCAGGTGAAGGG"},
+		{Decimal, "5421098", DNA, "CCAGGTGAAGGG"},
+		{Decimal, "80085", Base256, "🪐💞🔵"},
+		{Hex, "48656C6C6C20576F526C5421", Base64, "SGVsbGwgV29SbFQh"},
+		{Base64, "SGVsbGw/gV29SbF+Qh", Base32, "CIMVWGY3B7QFO32SNRPZBB"},
+		{Base64, "SGVsbGw/gV29SbF+Qh", Base58, "2fUsGKQUcgQcwSqpvy6"},
+		{Base64, "SGVsbGw/gV29SbF+Qh", Base62, "V34nvybdQ3m3RHk9Sr"},
+	}
+
+	for _, def := range defs {
+
+		d1 := NewAnyBaseConverter(def.FromCS)
+		d2 := NewAnyBaseConverter(def.ToCS)
+
+		v1 := tst.Must(d1.Decode(def.FromVal))(t)
+		v2 := tst.Must(d2.Decode(def.ToVal))(t)
+
+		tst.AssertArrayEqual(t, v1, v2)
+
+		str2 := d2.Encode(v1)
+		tst.AssertEqual(t, str2, def.ToVal)
+
+		str1 := d1.Encode(v2)
+		tst.AssertEqual(t, str1, def.FromVal)
+
+	}
+}

langext/maps.go

@@ -29,6 +29,14 @@ func ArrToMap[T comparable, V any](a []V, keyfunc func(V) T) map[T]V {
 	return result
 }
 
+func ArrToKVMap[T any, K comparable, V any](a []T, keyfunc func(T) K, valfunc func(T) V) map[K]V {
+	result := make(map[K]V, len(a))
+	for _, v := range a {
+		result[keyfunc(v)] = valfunc(v)
+	}
+	return result
+}
+
 func ArrToSet[T comparable](a []T) map[T]bool {
 	result := make(map[T]bool, len(a))
 	for _, v := range a {
@@ -63,3 +71,19 @@ func ForceMap[K comparable, V any](v map[K]V) map[K]V {
 		return v
 	}
 }
+
+func MapMerge[K comparable, V any](base map[K]V, arr ...map[K]V) map[K]V {
+	res := make(map[K]V, len(base)*(1+len(arr)))
+
+	for k, v := range base {
+		res[k] = v
+	}
+
+	for _, m := range arr {
+		for k, v := range m {
+			res[k] = v
+		}
+	}
+
+	return res
+}

langext/must.go

@@ -0,0 +1,21 @@
+package langext
+
+// Must returns a value and panics on error
+//
+// Usage: Must(methodWithError(...))
+func Must[T any](v T, err error) T {
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// MustBool returns a value and panics on missing
+//
+// Usage: MustBool(methodWithOkayReturn(...))
+func MustBool[T any](v T, ok bool) T {
+	if !ok {
+		panic("not ok")
+	}
+	return v
+}

langext/panic.go

@@ -1,7 +1,10 @@
 package langext
 
+import "runtime/debug"
+
 type PanicWrappedErr struct {
 	panic any
+	Stack string
 }
 
 func (p PanicWrappedErr) Error() string {
@@ -15,7 +18,7 @@ func (p PanicWrappedErr) ReoveredObj() any {
 func RunPanicSafe(fn func()) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -27,7 +30,7 @@ func RunPanicSafe(fn func()) (err error) {
 func RunPanicSafeR1(fn func() error) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -38,7 +41,7 @@ func RunPanicSafeR2[T1 any](fn func() (T1, error)) (r1 T1, err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
 			r1 = *new(T1)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -50,7 +53,7 @@ func RunPanicSafeR3[T1 any, T2 any](fn func() (T1, T2, error)) (r1 T1, r2 T2, er
 		if rec := recover(); rec != nil {
 			r1 = *new(T1)
 			r2 = *new(T2)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 
@@ -63,7 +66,7 @@ func RunPanicSafeR4[T1 any, T2 any, T3 any](fn func() (T1, T2, T3, error)) (r1 T
 			r1 = *new(T1)
 			r2 = *new(T2)
 			r3 = *new(T3)
-			err = PanicWrappedErr{panic: rec}
+			err = PanicWrappedErr{panic: rec, Stack: string(debug.Stack())}
 		}
 	}()
 

langext/pointer.go

@@ -10,10 +10,23 @@ var PTrue = Ptr(true)
 // PFalse := &false
 var PFalse = Ptr(false)
 
+// PNil := &nil
+var PNil = Ptr[any](nil)
+
 func Ptr[T any](v T) *T {
 	return &v
 }
 
+func DblPtr[T any](v T) **T {
+	v_ := &v
+	return &v_
+}
+
+func DblPtrNil[T any]() **T {
+	var v *T = nil
+	return &v
+}
+
 func PtrInt32(v int32) *int32 {
 	return &v
 }
@@ -35,7 +48,7 @@ func IsNil(i interface{}) bool {
 		return true
 	}
 	switch reflect.TypeOf(i).Kind() {
-	case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice:
+	case reflect.Ptr, reflect.Map, reflect.Chan, reflect.Slice, reflect.Func, reflect.UnsafePointer:
 		return reflect.ValueOf(i).IsNil()
 	}
 	return false

langext/sort.go

@@ -8,12 +8,28 @@ func Sort[T OrderedConstraint](arr []T) {
 	})
 }
 
+func AsSorted[T OrderedConstraint](arr []T) []T {
+	arr = ArrCopy(arr)
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+	return arr
+}
+
 func SortStable[T OrderedConstraint](arr []T) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return arr[i1] < arr[i2]
 	})
 }
 
+func AsSortedStable[T OrderedConstraint](arr []T) []T {
+	arr = ArrCopy(arr)
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+	return arr
+}
+
 func IsSorted[T OrderedConstraint](arr []T) bool {
 	return sort.SliceIsSorted(arr, func(i1, i2 int) bool {
 		return arr[i1] < arr[i2]
@@ -26,12 +42,28 @@ func SortSlice[T any](arr []T, less func(v1, v2 T) bool) {
 	})
 }
 
+func AsSortedSlice[T any](arr []T, less func(v1, v2 T) bool) []T {
+	arr = ArrCopy(arr)
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+	return arr
+}
+
 func SortSliceStable[T any](arr []T, less func(v1, v2 T) bool) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return less(arr[i1], arr[i2])
 	})
 }
 
+func AsSortedSliceStable[T any](arr []T, less func(v1, v2 T) bool) []T {
+	arr = ArrCopy(arr)
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+	return arr
+}
+
 func IsSliceSorted[T any](arr []T, less func(v1, v2 T) bool) bool {
 	return sort.SliceIsSorted(arr, func(i1, i2 int) bool {
 		return less(arr[i1], arr[i2])
@@ -44,12 +76,28 @@ func SortBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TEle
 	})
 }
 
+func AsSortedBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) []TElem {
+	arr = ArrCopy(arr)
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+	return arr
+}
+
 func SortByStable[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return selector(arr[i1]) < selector(arr[i2])
 	})
 }
 
+func AsSortedByStable[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) []TElem {
+	arr = ArrCopy(arr)
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+	return arr
+}
+
 func IsSortedBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return selector(arr[i1]) < selector(arr[i2])

langext/url.go

@@ -0,0 +1,29 @@
+package langext
+
+import (
+	"fmt"
+	"strings"
+)
+
+func BuildUrl(url, path string, params *map[string]string) string {
+	if path[:1] == "/" && url[len(url)-1:] == "/" {
+		url += path[1:]
+	} else if path[:1] != "/" && url[len(url)-1:] != "/" {
+		url += "/" + path
+	} else {
+		url += path
+	}
+
+	if params == nil {
+		return url
+	}
+
+	for key, value := range *params {
+		if strings.Contains(url, "?") {
+			url += fmt.Sprintf("&%s=%s", key, value)
+		} else {
+			url += fmt.Sprintf("?%s=%s", key, value)
+		}
+	}
+	return url
+}

langext/url_test.go

@@ -0,0 +1,45 @@
+package langext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestBuildUrl(t *testing.T) {
+	tests := []struct {
+		Url    string
+		Path   string
+		Params *map[string]string
+		Want   string
+	}{
+		{
+			Url:    "https://test.heydyno.de/",
+			Path:   "/testing-01",
+			Params: &map[string]string{"param1": "value1"},
+			Want:   "https://test.heydyno.de/testing-01?param1=value1",
+		},
+		{
+			Url:    "https://test.heydyno.de",
+			Path:   "testing-01",
+			Params: &map[string]string{"param1": "value1"},
+			Want:   "https://test.heydyno.de/testing-01?param1=value1",
+		},
+		{
+			Url:    "https://test.heydyno.de",
+			Path:   "/testing-01",
+			Params: nil,
+			Want:   "https://test.heydyno.de/testing-01",
+		},
+		{
+			Url:    "https://test.heydyno.de/",
+			Path:   "testing-01",
+			Params: nil,
+			Want:   "https://test.heydyno.de/testing-01",
+		},
+	}
+
+	for _, test := range tests {
+		res := BuildUrl(test.Url, test.Path, test.Params)
+		tst.AssertEqual(t, res, test.Want)
+	}
+}

mongoext/registry.go

@@ -3,6 +3,9 @@ package mongoext
 import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
 	"reflect"
 )
@@ -16,13 +19,29 @@ func CreateGoExtBsonRegistry() *bsoncodec.Registry {
 	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.RFC3339NanoTime{}), rfctime.RFC3339NanoTime{})
 	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.RFC3339NanoTime{}), rfctime.RFC3339NanoTime{})
 
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.UnixTime{}), rfctime.UnixTime{})
+	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.UnixTime{}), rfctime.UnixTime{})
+
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.UnixMilliTime{}), rfctime.UnixMilliTime{})
+	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.UnixMilliTime{}), rfctime.UnixMilliTime{})
+
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.UnixNanoTime{}), rfctime.UnixNanoTime{})
+	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.UnixNanoTime{}), rfctime.UnixNanoTime{})
+
 	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.Date{}), rfctime.Date{})
 	rb.RegisterTypeDecoder(reflect.TypeOf(&rfctime.Date{}), rfctime.Date{})
 
+	rb.RegisterTypeDecoder(reflect.TypeOf(rfctime.SecondsF64(0)), rfctime.SecondsF64(0))
+	rb.RegisterTypeDecoder(reflect.TypeOf(langext.Ptr(rfctime.SecondsF64(0))), rfctime.SecondsF64(0))
+
 	bsoncodec.DefaultValueEncoders{}.RegisterDefaultEncoders(rb)
 	bsoncodec.DefaultValueDecoders{}.RegisterDefaultDecoders(rb)
 
 	bson.PrimitiveCodecs{}.RegisterPrimitiveCodecs(rb)
 
+	// otherwise we get []primitve.E when unmarshalling into any
+	// which will result in {'key': .., 'value': ...}[] json when json-marshalling
+	rb.RegisterTypeMapEntry(bsontype.EmbeddedDocument, reflect.TypeOf(primitive.M{}))
+
 	return rb.Build()
 }

pagination/filter.go

@@ -0,0 +1,28 @@
+package pagination
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type MongoFilter interface {
+	FilterQuery() mongo.Pipeline
+	Sort() bson.D
+}
+
+type dynamicFilter struct {
+	pipeline mongo.Pipeline
+	sort     bson.D
+}
+
+func (d dynamicFilter) FilterQuery() mongo.Pipeline {
+	return d.pipeline
+}
+
+func (d dynamicFilter) Sort() bson.D {
+	return d.sort
+}
+
+func CreateFilter(pipeline mongo.Pipeline, sort bson.D) MongoFilter {
+	return dynamicFilter{pipeline: pipeline, sort: sort}
+}

pagination/pagination.go

@@ -0,0 +1,19 @@
+package pagination
+
+type Pagination struct {
+	Page             int `json:"page"`             // page (first page == 1)
+	Limit            int `json:"limit"`            // max-page-size
+	TotalPages       int `json:"totalPages"`       // total page-count
+	TotalItems       int `json:"totalItems"`       // total items-count
+	CurrentPageCount int `json:"currentPageCount"` // item-count in current page ( == len(data) )
+}
+
+func CalcPaginationTotalPages(totalItems int, limit int) int {
+	if totalItems == 0 {
+		return 0
+	}
+	if limit == 0 {
+		return 0
+	}
+	return 1 + (totalItems-1)/limit
+}

reflectext/convertToMap.go

@@ -0,0 +1,72 @@
+package reflectext
+
+import "reflect"
+
+func ConvertStructToMap(v any) any {
+	return reflectToMap(reflect.ValueOf(v))
+}
+
+func reflectToMap(fv reflect.Value) any {
+
+	if fv.Kind() == reflect.Ptr {
+
+		if fv.IsNil() {
+			return nil
+		} else {
+			return reflectToMap(fv.Elem())
+		}
+
+	}
+
+	if fv.Kind() == reflect.Func {
+
+		// skip
+		return nil
+
+	}
+
+	if fv.Kind() == reflect.Array {
+
+		arrlen := fv.Len()
+		arr := make([]any, arrlen)
+		for i := 0; i < arrlen; i++ {
+			arr[i] = reflectToMap(fv.Index(i))
+		}
+		return arr
+
+	}
+
+	if fv.Kind() == reflect.Slice {
+
+		arrlen := fv.Len()
+		arr := make([]any, arrlen)
+		for i := 0; i < arrlen; i++ {
+			arr[i] = reflectToMap(fv.Index(i))
+		}
+		return arr
+
+	}
+
+	if fv.Kind() == reflect.Chan {
+
+		// skip
+		return nil
+
+	}
+
+	if fv.Kind() == reflect.Struct {
+
+		res := make(map[string]any)
+
+		for i := 0; i < fv.NumField(); i++ {
+			if fv.Type().Field(i).IsExported() {
+				res[fv.Type().Field(i).Name] = reflectToMap(fv.Field(i))
+			}
+		}
+
+		return res
+
+	}
+
+	return fv.Interface()
+}

reflectext/mapAccess.go

@@ -0,0 +1,98 @@
+package reflectext
+
+import (
+	"reflect"
+	"strings"
+)
+
+// GetMapPath returns the value deep inside a hierahically nested map structure
+// eg:
+// x := langext.H{"K1": langext.H{"K2": 665}}
+// GetMapPath[int](x, "K1.K2") == 665
+func GetMapPath[TData any](mapval any, path string) (TData, bool) {
+	var ok bool
+
+	split := strings.Split(path, ".")
+
+	for i, key := range split {
+
+		if i < len(split)-1 {
+			mapval, ok = GetMapField[any](mapval, key)
+			if !ok {
+				return *new(TData), false
+			}
+		} else {
+			return GetMapField[TData](mapval, key)
+		}
+	}
+
+	return *new(TData), false
+}
+
+// GetMapField gets the value of a map, without knowing the actual types (mapval is any)
+// eg:
+// x := langext.H{"K1": 665}
+// GetMapPath[int](x, "K1") == 665
+//
+// works with aliased types and autom. dereferences pointes
+func GetMapField[TData any, TKey comparable](mapval any, key TKey) (TData, bool) {
+
+	rval := reflect.ValueOf(mapval)
+
+	for rval.Kind() == reflect.Ptr && !rval.IsNil() {
+		rval = rval.Elem()
+	}
+
+	if rval.Kind() != reflect.Map {
+		return *new(TData), false // mapval is not a map
+	}
+
+	kval := reflect.ValueOf(key)
+
+	if !kval.Type().AssignableTo(rval.Type().Key()) {
+		return *new(TData), false // key cannot index mapval
+	}
+
+	eval := rval.MapIndex(kval)
+	if !eval.IsValid() {
+		return *new(TData), false // key does not exist in mapval
+	}
+
+	destType := reflect.TypeOf(new(TData)).Elem()
+
+	if eval.Type() == destType {
+		return eval.Interface().(TData), true
+	}
+
+	if eval.CanConvert(destType) && !preventConvert(eval.Type(), destType) {
+		return eval.Convert(destType).Interface().(TData), true
+	}
+
+	if (eval.Kind() == reflect.Ptr || eval.Kind() == reflect.Interface) && eval.IsNil() && destType.Kind() == reflect.Ptr {
+		return *new(TData), false // special case: mapval[key] is nil
+	}
+
+	for (eval.Kind() == reflect.Ptr || eval.Kind() == reflect.Interface) && !eval.IsNil() {
+		eval = eval.Elem()
+
+		if eval.Type() == destType {
+			return eval.Interface().(TData), true
+		}
+
+		if eval.CanConvert(destType) && !preventConvert(eval.Type(), destType) {
+			return eval.Convert(destType).Interface().(TData), true
+		}
+	}
+
+	return *new(TData), false // mapval[key] is not of type TData
+}
+
+func preventConvert(t1 reflect.Type, t2 reflect.Type) bool {
+	if t1.Kind() == reflect.String && t1.Kind() != reflect.String {
+		return true
+	}
+	if t2.Kind() == reflect.String && t1.Kind() != reflect.String {
+		return true
+	}
+	return false
+}

reflectext/mapAccess_test.go

@@ -0,0 +1,55 @@
+package reflectext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestGetMapPath(t *testing.T) {
+	type PseudoInt = int64
+
+	mymap2 := map[string]map[string]any{"Test": {"Second": 3}}
+
+	var maany2 any = mymap2
+
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[int](maany2, "Test.Second")), "3 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[int](maany2, "Test2.Second")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[int](maany2, "Test.Second2")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[string](maany2, "Test.Second")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[string](maany2, "Test2.Second")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[string](maany2, "Test.Second2")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[PseudoInt](maany2, "Test.Second")), "3 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[PseudoInt](maany2, "Test2.Second")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[PseudoInt](maany2, "Test.Second2")), "0 false")
+}
+
+func TestGetMapField(t *testing.T) {
+	type PseudoInt = int64
+
+	mymap1 := map[string]any{"Test": 12}
+	mymap2 := map[string]int{"Test": 12}
+
+	var maany1 any = mymap1
+	var maany2 any = mymap2
+
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany1, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany1, "Test2")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany1, "Test")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany1, "Test2")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany1, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany1, "Test2")), "0 false")
+
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany2, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany2, "Test2")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany2, "Test")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany2, "Test2")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany2, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany2, "Test2")), "0 false")
+}
+
+func main2() {
+}
+
+func main() {
+}

rfctime/seconds.go

@@ -2,7 +2,14 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
 	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"reflect"
 	"time"
 )
 
@@ -54,6 +61,63 @@ func (d SecondsF64) MarshalJSON() ([]byte, error) {
 	return json.Marshal(secs)
 }
 
+func (d *SecondsF64) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*d = SecondsF64(0)
+		return nil
+	}
+	if bt != bson.TypeDouble {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into SecondsF64", bt))
+	}
+	var secValue float64
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&secValue)
+	if err != nil {
+		return err
+	}
+	*d = SecondsF64(int64(secValue * float64(time.Second)))
+	return nil
+}
+
+func (d SecondsF64) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(d.Seconds())
+}
+
+func (d SecondsF64) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = d.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&d))
+	} else {
+		val.Set(reflect.ValueOf(d))
+	}
+
+	return nil
+}
+
 func NewSecondsF64(t time.Duration) SecondsF64 {
 	return SecondsF64(t)
 }

rfctime/time.go

@@ -0,0 +1,148 @@
+package rfctime
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Time struct {
+	Hour       int
+	Minute     int
+	Second     int
+	NanoSecond int
+}
+
+func (t Time) Serialize() string {
+	return fmt.Sprintf("%04d:%02d:%02d.%09d", t.Hour, t.Minute, t.Second, t.NanoSecond)
+}
+
+func (t Time) SerializeShort() string {
+	if t.NanoSecond == 0 && t.Second == 0 {
+		return fmt.Sprintf("%02d:%02d", t.Hour, t.Minute)
+	} else if t.NanoSecond == 0 {
+		return fmt.Sprintf("%02d:%02d:%02d", t.Hour, t.Minute, t.Second)
+	} else {
+		return fmt.Sprintf("%02d:%02d:%02d.%09d", t.Hour, t.Minute, t.Second, t.NanoSecond)
+	}
+}
+
+func (t *Time) Deserialize(v string) error {
+
+	var h, m, s, ns string
+
+	split1 := strings.Split(v, ".")
+
+	if len(split1) == 2 {
+
+		split2 := strings.Split(split1[0], ":")
+		if len(split2) == 3 {
+
+			h = split2[0]
+			m = split2[1]
+			s = split2[2]
+			ns = split1[1]
+
+		} else {
+			return fmt.Errorf("invalid time format: %s", v)
+		}
+
+	} else if len(split1) == 1 {
+
+		split2 := strings.Split(split1[0], ":")
+		if len(split2) == 2 {
+
+			h = split2[0]
+			m = split2[1]
+			s = "0"
+			ns = "0"
+
+		} else if len(split2) == 3 {
+
+			h = split2[0]
+			m = split2[1]
+			s = split2[2]
+			ns = "0"
+
+		} else {
+			return fmt.Errorf("invalid time format: %s", v)
+		}
+
+	} else {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	ns = langext.StrPadRight(ns, "0", 9)
+
+	hh, err := strconv.ParseInt(h, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	mm, err := strconv.ParseInt(m, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	ss, err := strconv.ParseInt(s, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	nss, err := strconv.ParseInt(ns, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	t.Hour = int(hh)
+	t.Minute = int(mm)
+	t.Second = int(ss)
+	t.NanoSecond = int(nss)
+
+	return nil
+}
+
+func (t Time) FormatStr() string {
+	return "15:04:05.999999999"
+}
+
+func (t Time) GoString() string {
+	return fmt.Sprintf("rfctime.NewTime(%d, %d, %d, %d)", t.Hour, t.Minute, t.Second, t.NanoSecond)
+}
+
+func (t Time) String() string {
+	return fmt.Sprintf("%04d:%02d:%02d.%09d", t.Hour, t.Minute, t.Second, t.NanoSecond)
+}
+
+func NewTime(h int, m int, s int, ns int) Time {
+	return Time{
+		Hour:       h,
+		Minute:     m,
+		Second:     s,
+		NanoSecond: ns,
+	}
+}
+
+func NewTimeFromTS(t time.Time) Time {
+	return Time{
+		Hour:       t.Hour(),
+		Minute:     t.Minute(),
+		Second:     t.Second(),
+		NanoSecond: t.Nanosecond(),
+	}
+}
+
+func NowTime(loc *time.Location) Time {
+	now := time.Now().In(loc)
+	return NewTime(now.Hour(), now.Minute(), now.Second(), now.Nanosecond())
+}
+
+func NowTimeLoc() Time {
+	return NowTime(time.UTC)
+}
+
+func NowTimeUTC() Time {
+	return NowTime(time.Local)
+}

rfctime/unix.go

@@ -2,6 +2,13 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -59,6 +66,63 @@ func (t *UnixTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *UnixTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*t = UnixTime{}
+		return nil
+	}
+	if bt != bson.TypeDateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into UnixTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = UnixTime(tt)
+	return nil
+}
+
+func (t UnixTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t UnixTime) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = t.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&t))
+	} else {
+		val.Set(reflect.ValueOf(t))
+	}
+
+	return nil
+}
+
 func (t UnixTime) Serialize() string {
 	return strconv.FormatInt(t.Time().Unix(), 10)
 }

rfctime/unixMilli.go

@@ -2,6 +2,13 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -59,6 +66,63 @@ func (t *UnixMilliTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *UnixMilliTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*t = UnixMilliTime{}
+		return nil
+	}
+	if bt != bson.TypeDateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into UnixMilliTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = UnixMilliTime(tt)
+	return nil
+}
+
+func (t UnixMilliTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t UnixMilliTime) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = t.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&t))
+	} else {
+		val.Set(reflect.ValueOf(t))
+	}
+
+	return nil
+}
+
 func (t UnixMilliTime) Serialize() string {
 	return strconv.FormatInt(t.Time().UnixMilli(), 10)
 }

rfctime/unixNano.go

@@ -2,6 +2,13 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsoncodec"
+	"go.mongodb.org/mongo-driver/bson/bsonrw"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"reflect"
 	"strconv"
 	"time"
 )
@@ -59,6 +66,63 @@ func (t *UnixNanoTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *UnixNanoTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt == bson.TypeNull {
+		// we can't set nil in UnmarshalBSONValue (so we use default(struct))
+		// Use mongoext.CreateGoExtBsonRegistry if you need to unmarsh pointer values
+		// https://stackoverflow.com/questions/75167597
+		// https://jira.mongodb.org/browse/GODRIVER-2252
+		*t = UnixNanoTime{}
+		return nil
+	}
+	if bt != bson.TypeDateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into UnixNanoTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = UnixNanoTime(tt)
+	return nil
+}
+
+func (t UnixNanoTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t UnixNanoTime) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		if !val.CanSet() {
+			return errors.New("ValueUnmarshalerDecodeValue")
+		}
+		val.Set(reflect.New(val.Type().Elem()))
+	}
+
+	tp, src, err := bsonrw.Copier{}.CopyValueToBytes(vr)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr && len(src) == 0 {
+		val.Set(reflect.Zero(val.Type()))
+		return nil
+	}
+
+	err = t.UnmarshalBSONValue(tp, src)
+	if err != nil {
+		return err
+	}
+
+	if val.Kind() == reflect.Ptr {
+		val.Set(reflect.ValueOf(&t))
+	} else {
+		val.Set(reflect.ValueOf(t))
+	}
+
+	return nil
+}
+
 func (t UnixNanoTime) Serialize() string {
 	return strconv.FormatInt(t.Time().UnixNano(), 10)
 }

scn/scn.go

@@ -0,0 +1,13 @@
+package scn
+
+type Connection struct {
+	uid   string
+	token string
+}
+
+func New(userid string, token string) *Connection {
+	return &Connection{
+		uid:   userid,
+		token: token,
+	}
+}

scn/send.go

@@ -0,0 +1,171 @@
+package scn
+
+import (
+	"bytes"
+	"context"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"io"
+	"net/http"
+	"time"
+)
+
+var (
+	ErrAuthFailed        = exerr.NewType("GOEXT_SCN_AUTHFAILED", nil)
+	ErrQuota             = exerr.NewType("GOEXT_SCN_QUOTAEXCEEDED", nil)
+	ErrBadRequest        = exerr.NewType("GOEXT_SCN_BADREQUEST", nil)
+	ErrInternalServerErr = exerr.NewType("GOEXT_SCN_INTERNALSERVER", nil)
+	ErrOther             = exerr.NewType("GOEXT_SCN_OTHER", nil)
+)
+
+type MessageResponse struct {
+	ErrHighlight int    `json:"errhighlight"`
+	Error        int    `json:"error"`
+	IsPro        bool   `json:"is_pro"`
+	Message      string `json:"message"`
+	Messagecount int    `json:"messagecount"`
+	Quota        int    `json:"quota"`
+	QuotaMax     int    `json:"quota_max"`
+	SCNMessageID string `json:"scn_msg_id"`
+	Success      bool   `json:"success"`
+	SuppressSend bool   `json:"suppress_send"`
+}
+
+type MessageErrResponse struct {
+	Errhighlight int    `json:"errhighlight"`
+	Error        int    `json:"error"`
+	Message      string `json:"message"`
+	Success      bool   `json:"success"`
+}
+
+type MessageBuilder struct {
+	conn       *Connection
+	title      string
+	content    *string
+	channel    *string
+	time       *time.Time
+	sendername *string
+	priority   *int
+}
+
+func (c *Connection) Message(title string) *MessageBuilder {
+	return &MessageBuilder{conn: c, title: title}
+}
+
+func (c *MessageBuilder) Channel(channel string) *MessageBuilder {
+	c.channel = &channel
+	return c
+}
+
+func (c *MessageBuilder) Content(content string) *MessageBuilder {
+	c.content = &content
+	return c
+}
+
+func (c *MessageBuilder) Time(t time.Time) *MessageBuilder {
+	c.time = &t
+	return c
+}
+
+func (c *MessageBuilder) SenderName(sn string) *MessageBuilder {
+	c.sendername = &sn
+	return c
+}
+
+func (c *MessageBuilder) Priority(p int) *MessageBuilder {
+	c.priority = &p
+	return c
+}
+
+func (c *MessageBuilder) Send(ctx context.Context) (MessageResponse, error) {
+	client := http.Client{Timeout: 5 * time.Second}
+
+	body := langext.H{}
+
+	body["user_id"] = c.conn.uid
+	body["key"] = c.conn.token
+
+	if c.channel != nil {
+		body["channel"] = *c.channel
+	}
+
+	body["title"] = c.title
+
+	if c.content != nil {
+		body["content"] = *c.content
+	}
+
+	if c.sendername != nil {
+		body["content"] = *c.sendername
+	}
+
+	if c.time != nil {
+		body["timestamp"] = c.time.Unix()
+	}
+
+	if c.priority != nil {
+		body["priority"] = *c.priority
+	}
+
+	body["msg_id"] = langext.MustHexUUID()
+
+	rawbody, err := json.Marshal(body)
+	if err != nil {
+		return MessageResponse{}, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", "https://simplecloudnotifier.de/", bytes.NewBuffer(rawbody))
+	if err != nil {
+		return MessageResponse{}, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return MessageResponse{}, err
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+
+		raw, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return MessageResponse{}, err
+		}
+
+		var mr MessageResponse
+		err = json.Unmarshal(raw, &mr)
+		if err != nil {
+			return MessageResponse{}, err
+		}
+
+		return mr, nil
+	} else {
+		errMessage := resp.Status
+
+		if raw, err := io.ReadAll(resp.Body); err == nil {
+			var mr MessageErrResponse
+			if err = json.Unmarshal(raw, &mr); err == nil {
+				errMessage = mr.Message
+			}
+		}
+
+		if resp.StatusCode == 400 {
+			return MessageResponse{}, exerr.New(ErrBadRequest, errMessage).Build()
+		}
+		if resp.StatusCode == 401 {
+			return MessageResponse{}, exerr.New(ErrAuthFailed, errMessage).Build()
+		}
+		if resp.StatusCode == 403 {
+			return MessageResponse{}, exerr.New(ErrQuota, errMessage).Build()
+		}
+		if resp.StatusCode == 500 {
+			return MessageResponse{}, exerr.New(ErrInternalServerErr, errMessage).Build()
+		}
+
+		return MessageResponse{}, exerr.New(ErrOther, errMessage).Build()
+	}
+
+}

scn/send_test.go

@@ -0,0 +1,24 @@
+package scn
+
+import (
+	"context"
+	"fmt"
+	"testing"
+)
+
+func TestSendSCN(t *testing.T) {
+	t.Skip()
+	return
+
+	mr, err := New("TODO", "TODO").
+		Message("Hello Test").
+		Content("This is a test / goext").
+		Send(context.Background())
+
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%+v\n", mr)
+
+}

sq/builder.go

@@ -0,0 +1,199 @@
+package sq
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"reflect"
+	"strings"
+)
+
+func BuildUpdateStatement[TData any](q Queryable, tableName string, obj TData, idColumn string) (string, PP, error) {
+	rval := reflect.ValueOf(obj)
+	rtyp := rval.Type()
+
+	params := PP{}
+
+	setClauses := make([]string, 0)
+
+	matchClause := ""
+
+	for i := 0; i < rtyp.NumField(); i++ {
+
+		rsfield := rtyp.Field(i)
+		rvfield := rval.Field(i)
+
+		if !rsfield.IsExported() {
+			continue
+		}
+
+		columnName := rsfield.Tag.Get("db")
+		if columnName == "" || columnName == "-" {
+			continue
+		}
+
+		if idColumn == columnName {
+			idValue, err := convertValueToDB(q, rvfield.Interface())
+			if err != nil {
+				return "", nil, err
+			}
+
+			matchClause = fmt.Sprintf("(%s = :%s)", columnName, params.Add(idValue))
+
+			continue
+		}
+
+		if rsfield.Type.Kind() == reflect.Ptr && rvfield.IsNil() {
+
+			setClauses = append(setClauses, fmt.Sprintf("%s = NULL", columnName))
+
+		} else {
+
+			val, err := convertValueToDB(q, rvfield.Interface())
+			if err != nil {
+				return "", nil, err
+			}
+
+			setClauses = append(setClauses, fmt.Sprintf("%s = :%s", columnName, params.Add(val)))
+
+		}
+	}
+
+	if len(setClauses) == 0 {
+		return "", nil, exerr.New(exerr.TypeSQLBuild, "no updates clauses found in object").Build()
+	}
+
+	if matchClause == "" {
+		return "", nil, exerr.New(exerr.TypeSQLBuild, "id column not found in object").Build()
+	}
+
+	//goland:noinspection SqlNoDataSourceInspection
+	return fmt.Sprintf("UPDATE %s SET %s WHERE %s", tableName, strings.Join(setClauses, ", "), matchClause), params, nil
+}
+
+func BuildInsertStatement[TData any](q Queryable, tableName string, obj TData) (string, PP, error) {
+	rval := reflect.ValueOf(obj)
+	rtyp := rval.Type()
+
+	params := PP{}
+
+	fields := make([]string, 0)
+	values := make([]string, 0)
+
+	for i := 0; i < rtyp.NumField(); i++ {
+
+		rsfield := rtyp.Field(i)
+		rvfield := rval.Field(i)
+
+		if !rsfield.IsExported() {
+			continue
+		}
+
+		columnName := rsfield.Tag.Get("db")
+		if columnName == "" || columnName == "-" {
+			continue
+		}
+
+		if rsfield.Type.Kind() == reflect.Ptr && rvfield.IsNil() {
+
+			fields = append(fields, columnName)
+			values = append(values, "NULL")
+
+		} else {
+
+			val, err := convertValueToDB(q, rvfield.Interface())
+			if err != nil {
+				return "", nil, err
+			}
+
+			fields = append(fields, columnName)
+			values = append(values, ":"+params.Add(val))
+
+		}
+	}
+
+	if len(fields) == 0 {
+		return "", nil, exerr.New(exerr.TypeSQLBuild, "no fields found in object").Build()
+	}
+
+	//goland:noinspection SqlNoDataSourceInspection
+	return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, strings.Join(fields, ", "), strings.Join(values, ", ")), params, nil
+}
+
+func BuildInsertMultipleStatement[TData any](q Queryable, tableName string, vArr []TData) (string, PP, error) {
+
+	if len(vArr) == 0 {
+		return "", nil, errors.New("no data supplied")
+	}
+
+	rtyp := reflect.ValueOf(vArr[0]).Type()
+
+	sqlPrefix := ""
+	{
+		columns := make([]string, 0)
+
+		for i := 0; i < rtyp.NumField(); i++ {
+			rsfield := rtyp.Field(i)
+
+			if !rsfield.IsExported() {
+				continue
+			}
+
+			columnName := rsfield.Tag.Get("db")
+			if columnName == "" || columnName == "-" {
+				continue
+			}
+
+			columns = append(columns, "\""+columnName+"\"")
+		}
+
+		sqlPrefix = fmt.Sprintf("INSERT"+" INTO \"%s\" (%s) VALUES", tableName, strings.Join(columns, ", "))
+	}
+
+	pp := PP{}
+
+	sqlValuesArr := make([]string, 0)
+
+	for _, v := range vArr {
+
+		rval := reflect.ValueOf(v)
+
+		params := make([]string, 0)
+
+		for i := 0; i < rtyp.NumField(); i++ {
+
+			rsfield := rtyp.Field(i)
+			rvfield := rval.Field(i)
+
+			if !rsfield.IsExported() {
+				continue
+			}
+
+			columnName := rsfield.Tag.Get("db")
+			if columnName == "" || columnName == "-" {
+				continue
+			}
+
+			if rsfield.Type.Kind() == reflect.Ptr && rvfield.IsNil() {
+
+				params = append(params, "NULL")
+
+			} else {
+
+				val, err := convertValueToDB(q, rvfield.Interface())
+				if err != nil {
+					return "", nil, err
+				}
+
+				params = append(params, ":"+pp.Add(val))
+
+			}
+		}
+
+		sqlValuesArr = append(sqlValuesArr, fmt.Sprintf("(%s)", strings.Join(params, ", ")))
+	}
+
+	sqlstr := fmt.Sprintf("%s %s", sqlPrefix, strings.Join(sqlValuesArr, ", "))
+
+	return sqlstr, pp, nil
+}

sq/builder_test.go

@@ -0,0 +1,86 @@
+package sq
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"github.com/glebarez/go-sqlite"
+	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"path/filepath"
+	"testing"
+)
+
+type dummyQueryable struct {
+}
+
+func (d dummyQueryable) Exec(ctx context.Context, sql string, prep PP) (sql.Result, error) {
+	return nil, errors.New("err")
+}
+
+func (d dummyQueryable) Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error) {
+	return nil, errors.New("err")
+}
+
+func (d dummyQueryable) ListConverter() []DBTypeConverter {
+	return nil
+}
+
+func TestCreateUpdateStatement(t *testing.T) {
+
+	type request struct {
+		ID        string  `json:"id"        db:"id"`
+		Timestamp int     `json:"timestamp" db:"timestamp"`
+		StrVal    string  `json:"strVal"    db:"str_val"`
+		FloatVal  float64 `json:"floatVal"    db:"float_val"`
+		Dummy     bool    `json:"dummyBool" db:"dummy_bool"`
+		JsonVal   JsonObj `json:"jsonVal" db:"json_val"`
+	}
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
+
+	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
+	tst.AssertNoErr(t, err)
+
+	sqlStr, pp, err := BuildUpdateStatement(db, "requests", request{
+		ID:        "9927",
+		Timestamp: 12321,
+		StrVal:    "hello world",
+		Dummy:     true,
+		FloatVal:  3.14159,
+		JsonVal: JsonObj{
+			"firs":   1,
+			"second": true,
+		},
+	}, "id")
+
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("\n\n")
+
+	fmt.Printf("######## PP:\n")
+	for k, v := range pp {
+		fmt.Printf("['%s']       =>       %+v\n", k, v)
+	}
+	fmt.Printf("\n\n")
+
+	fmt.Printf("######## SQL:\n%+v\n\n", sqlStr)
+
+	fmt.Printf("\n\n")
+
+}

sq/commentTrimmer.go

@@ -0,0 +1,32 @@
+package sq
+
+import (
+	"context"
+	"strings"
+)
+
+var CommentTrimmer = NewPreListener(fnTrimComments)
+
+func fnTrimComments(ctx context.Context, cmdtype string, id *uint16, sql *string, params *PP) error {
+
+	res := make([]string, 0)
+
+	for _, s := range strings.Split(*sql, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(s), "--") {
+			continue
+		}
+
+		idx := strings.Index(s, "--")
+		if idx != -1 {
+			s = s[:idx]
+		}
+
+		s = strings.TrimRight(s, " \t\r\n")
+
+		res = append(res, s)
+	}
+
+	*sql = strings.Join(res, "\n")
+
+	return nil
+}

sq/converter.go

@@ -4,17 +4,9 @@ import (
 	"errors"
 	"fmt"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
-	"time"
+	"reflect"
 )
 
-//TODO UNFINISHED
-// this is not finished
-// idea was that we can register converter in the database struct
-// they get inherited from the transactions
-// and when marshallingunmarshaling (sq.Query | sq.QueryAll)
-// or marshaling (sq.InsertSingle)
-// the types get converter automatically...
-
 type DBTypeConverter interface {
 	ModelTypeString() string
 	DBTypeString() string
@@ -22,37 +14,16 @@ type DBTypeConverter interface {
 	DBToModel(v any) (any, error)
 }
 
-var ConverterBoolToBit = NewDBTypeConverter[bool, int](func(v bool) (int, error) {
-	return langext.Conditional(v, 1, 0), nil
-}, func(v int) (bool, error) {
-	if v == 0 {
-		return false, nil
-	}
-	if v == 1 {
-		return true, nil
-	}
-	return false, errors.New(fmt.Sprintf("invalid valud for boolean: '%d'", v))
-})
+type DBDataConstraint interface {
+	string | langext.NumberConstraint | []byte
+}
 
-var ConverterTimeToUnixMillis = NewDBTypeConverter[time.Time, int64](func(v time.Time) (int64, error) {
-	return v.UnixMilli(), nil
-}, func(v int64) (time.Time, error) {
-	return time.UnixMilli(v), nil
-})
+type DatabaseConvertible[TModelData any, TDBData DBDataConstraint] interface {
+	MarshalToDB(v TModelData) (TDBData, error)
+	UnmarshalToModel(v TDBData) (TModelData, error)
+}
 
-var ConverterOptTimeToUnixMillis = NewDBTypeConverter[*time.Time, *int64](func(v *time.Time) (*int64, error) {
-	if v == nil {
-		return nil, nil
-	}
-	return langext.Ptr(v.UnixMilli()), nil
-}, func(v *int64) (*time.Time, error) {
-	if v == nil {
-		return nil, nil
-	}
-	return langext.Ptr(time.UnixMilli(*v)), nil
-})
-
-type dbTypeConverterImpl[TModelData any, TDBData any] struct {
+type dbTypeConverterImpl[TModelData any, TDBData DBDataConstraint] struct {
 	dbTypeString    string
 	modelTypeString string
 	todb            func(v TModelData) (TDBData, error)
@@ -81,7 +52,7 @@ func (t *dbTypeConverterImpl[TModelData, TDBData]) DBToModel(v any) (any, error)
 	return nil, errors.New(fmt.Sprintf("Unexpected value in DBTypeConverter, expected '%s', found '%T'", t.dbTypeString, v))
 }
 
-func NewDBTypeConverter[TModelData any, TDBData any](todb func(v TModelData) (TDBData, error), tomodel func(v TDBData) (TModelData, error)) DBTypeConverter {
+func NewDBTypeConverter[TModelData any, TDBData DBDataConstraint](todb func(v TModelData) (TDBData, error), tomodel func(v TDBData) (TModelData, error)) DBTypeConverter {
 	return &dbTypeConverterImpl[TModelData, TDBData]{
 		dbTypeString:    fmt.Sprintf("%T", *new(TDBData)),
 		modelTypeString: fmt.Sprintf("%T", *new(TModelData)),
@@ -89,3 +60,45 @@ func NewDBTypeConverter[TModelData any, TDBData any](todb func(v TModelData) (TD
 		tomodel:         tomodel,
 	}
 }
+
+func NewAutoDBTypeConverter[TDBData DBDataConstraint, TModelData DatabaseConvertible[TModelData, TDBData]](obj TModelData) DBTypeConverter {
+	return &dbTypeConverterImpl[TModelData, TDBData]{
+		dbTypeString:    fmt.Sprintf("%T", *new(TDBData)),
+		modelTypeString: fmt.Sprintf("%T", *new(TModelData)),
+		todb:            obj.MarshalToDB,
+		tomodel:         obj.UnmarshalToModel,
+	}
+}
+
+func convertValueToDB(q Queryable, value any) (any, error) {
+	modelTypeStr := fmt.Sprintf("%T", value)
+
+	for _, conv := range q.ListConverter() {
+		if conv.ModelTypeString() == modelTypeStr {
+			return conv.ModelToDB(value)
+		}
+	}
+
+	if value != nil && reflect.TypeOf(value).Kind() == reflect.Ptr {
+		vof := reflect.ValueOf(value)
+		if vof.IsNil() {
+			return nil, nil
+		} else {
+			return convertValueToDB(q, vof.Elem().Interface())
+		}
+	}
+
+	return value, nil
+}
+
+func convertValueToModel(q Queryable, value any, destinationType string) (any, error) {
+	dbTypeString := fmt.Sprintf("%T", value)
+
+	for _, conv := range q.ListConverter() {
+		if conv.ModelTypeString() == destinationType && conv.DBTypeString() == dbTypeString {
+			return conv.DBToModel(value)
+		}
+	}
+
+	return value, nil
+}

sq/converterDefault.go

@@ -0,0 +1,161 @@
+package sq
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ========================== COMMON DATATYPES ==========================
+
+var ConverterBoolToBit = NewDBTypeConverter[bool, int64](func(v bool) (int64, error) {
+	return langext.Conditional(v, int64(1), int64(0)), nil
+}, func(v int64) (bool, error) {
+	if v == 0 {
+		return false, nil
+	}
+	if v == 1 {
+		return true, nil
+	}
+	return false, errors.New(fmt.Sprintf("invalid valud for boolean: '%d'", v))
+})
+
+var ConverterTimeToUnixMillis = NewDBTypeConverter[time.Time, int64](func(v time.Time) (int64, error) {
+	return v.UnixMilli(), nil
+}, func(v int64) (time.Time, error) {
+	return time.UnixMilli(v), nil
+})
+
+// ========================== RFCTIME ==========================
+
+var ConverterRFCUnixMilliTimeToUnixMillis = NewDBTypeConverter[rfctime.UnixMilliTime, int64](func(v rfctime.UnixMilliTime) (int64, error) {
+	return v.UnixMilli(), nil
+}, func(v int64) (rfctime.UnixMilliTime, error) {
+	return rfctime.NewUnixMilli(time.UnixMilli(v)), nil
+})
+
+var ConverterRFCUnixNanoTimeToUnixNanos = NewDBTypeConverter[rfctime.UnixNanoTime, int64](func(v rfctime.UnixNanoTime) (int64, error) {
+	return v.UnixNano(), nil
+}, func(v int64) (rfctime.UnixNanoTime, error) {
+	return rfctime.NewUnixNano(time.Unix(0, v)), nil
+})
+
+var ConverterRFCUnixTimeToUnixSeconds = NewDBTypeConverter[rfctime.UnixTime, int64](func(v rfctime.UnixTime) (int64, error) {
+	return v.Unix(), nil
+}, func(v int64) (rfctime.UnixTime, error) {
+	return rfctime.NewUnix(time.Unix(v, 0)), nil
+})
+
+// ConverterRFC339TimeToString
+// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
+var ConverterRFC339TimeToString = NewDBTypeConverter[rfctime.RFC3339Time, string](func(v rfctime.RFC3339Time) (string, error) {
+	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05"), nil
+}, func(v string) (rfctime.RFC3339Time, error) {
+	t, err := time.Parse("2006-01-02 15:04:05", v)
+	if err != nil {
+		return rfctime.RFC3339Time{}, err
+	}
+	return rfctime.NewRFC3339(t), nil
+})
+
+// ConverterRFC339NanoTimeToString
+// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
+var ConverterRFC339NanoTimeToString = NewDBTypeConverter[rfctime.RFC3339NanoTime, string](func(v rfctime.RFC3339NanoTime) (string, error) {
+	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05.999999999"), nil
+}, func(v string) (rfctime.RFC3339NanoTime, error) {
+	t, err := time.ParseInLocation("2006-01-02 15:04:05.999999999", v, time.UTC)
+	if err != nil {
+		return rfctime.RFC3339NanoTime{}, err
+	}
+	return rfctime.NewRFC3339Nano(t), nil
+})
+
+var ConverterRFCDateToString = NewDBTypeConverter[rfctime.Date, string](func(v rfctime.Date) (string, error) {
+	return fmt.Sprintf("%04d-%02d-%02d", v.Year, v.Month, v.Day), nil
+}, func(v string) (rfctime.Date, error) {
+	split := strings.Split(v, "-")
+	if len(split) != 3 {
+		return rfctime.Date{}, errors.New("invalid date format: " + v)
+	}
+	year, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	month, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	day, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+
+	return rfctime.Date{Year: int(year), Month: int(month), Day: int(day)}, nil
+})
+
+var ConverterRFCTimeToString = NewDBTypeConverter[rfctime.Time, string](func(v rfctime.Time) (string, error) {
+	return v.SerializeShort(), nil
+}, func(v string) (rfctime.Time, error) {
+	res := rfctime.Time{}
+	err := res.Deserialize(v)
+	if err != nil {
+		return rfctime.Time{}, err
+	}
+	return res, nil
+})
+
+var ConverterRFCSecondsF64ToString = NewDBTypeConverter[rfctime.SecondsF64, float64](func(v rfctime.SecondsF64) (float64, error) {
+	return v.Seconds(), nil
+}, func(v float64) (rfctime.SecondsF64, error) {
+	return rfctime.NewSecondsF64(timeext.FromSeconds(v)), nil
+})
+
+// ========================== JSON ==========================
+
+var ConverterJsonObjToString = NewAutoDBTypeConverter(JsonObj{})
+
+var ConverterJsonArrToString = NewAutoDBTypeConverter(JsonArr{})
+
+// Json[T] must be registered manually for each gen-type
+
+// ========================== EXERR ==========================
+
+var ConverterExErrCategoryToString = NewDBTypeConverter[exerr.ErrorCategory, string](func(v exerr.ErrorCategory) (string, error) {
+	return v.Category, nil
+}, func(v string) (exerr.ErrorCategory, error) {
+	for _, cat := range exerr.AllCategories {
+		if cat.Category == v {
+			return cat, nil
+		}
+	}
+	return exerr.CatUser, errors.New("failed to convert '" + v + "' to exerr.ErrorCategory")
+})
+
+var ConverterExErrSeverityToString = NewDBTypeConverter[exerr.ErrorSeverity, string](func(v exerr.ErrorSeverity) (string, error) {
+	return v.Severity, nil
+}, func(v string) (exerr.ErrorSeverity, error) {
+	for _, sev := range exerr.AllSeverities {
+		if sev.Severity == v {
+			return sev, nil
+		}
+	}
+	return exerr.SevErr, errors.New("failed to convert '" + v + "' to exerr.ErrorSeverity")
+})
+
+var ConverterExErrTypeToString = NewDBTypeConverter[exerr.ErrorType, string](func(v exerr.ErrorType) (string, error) {
+	return v.Key, nil
+}, func(v string) (exerr.ErrorType, error) {
+	for _, etp := range exerr.ListRegisteredTypes() {
+		if etp.Key == v {
+			return etp, nil
+		}
+	}
+
+	return exerr.NewType(v, nil), nil
+})

sq/database.go

@@ -4,16 +4,24 @@ import (
 	"context"
 	"database/sql"
 	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"sync"
 )
 
 type DB interface {
-	Exec(ctx context.Context, sql string, prep PP) (sql.Result, error)
-	Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error)
+	Queryable
+
 	Ping(ctx context.Context) error
 	BeginTransaction(ctx context.Context, iso sql.IsolationLevel) (Tx, error)
 	AddListener(listener Listener)
 	Exit() error
+	RegisterConverter(DBTypeConverter)
+}
+
+type DBOptions struct {
+	RegisterDefaultConverter *bool
+	RegisterCommentTrimmer   *bool
 }
 
 type database struct {
@@ -21,15 +29,26 @@ type database struct {
 	txctr uint16
 	lock  sync.Mutex
 	lstr  []Listener
+	conv  []DBTypeConverter
 }
 
-func NewDB(db *sqlx.DB) DB {
-	return &database{
+func NewDB(db *sqlx.DB, opt DBOptions) DB {
+	sqdb := &database{
 		db:    db,
 		txctr: 0,
 		lock:  sync.Mutex{},
 		lstr:  make([]Listener, 0),
 	}
+
+	if langext.Coalesce(opt.RegisterDefaultConverter, true) {
+		sqdb.registerDefaultConverter()
+	}
+
+	if langext.Coalesce(opt.RegisterCommentTrimmer, true) {
+		sqdb.AddListener(CommentTrimmer)
+	}
+
+	return sqdb
 }
 
 func (db *database) AddListener(listener Listener) {
@@ -41,7 +60,7 @@ func (db *database) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Resul
 	for _, v := range db.lstr {
 		err := v.PreExec(ctx, nil, &sqlstr, &prep)
 		if err != nil {
-			return nil, err
+			return nil, exerr.Wrap(err, "failed to call SQL pre-exec listener").Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 		}
 	}
 
@@ -52,7 +71,7 @@ func (db *database) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Resul
 	}
 
 	if err != nil {
-		return nil, err
+		return nil, exerr.Wrap(err, "Failed to [exec] sql statement").Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 	}
 	return res, nil
 }
@@ -62,7 +81,7 @@ func (db *database) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx.Ro
 	for _, v := range db.lstr {
 		err := v.PreQuery(ctx, nil, &sqlstr, &prep)
 		if err != nil {
-			return nil, err
+			return nil, exerr.Wrap(err, "failed to call SQL pre-query listener").Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 		}
 	}
 
@@ -73,7 +92,7 @@ func (db *database) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx.Ro
 	}
 
 	if err != nil {
-		return nil, err
+		return nil, exerr.Wrap(err, "Failed to [query] sql statement").Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 	}
 	return rows, nil
 }
@@ -93,7 +112,7 @@ func (db *database) Ping(ctx context.Context) error {
 	}
 
 	if err != nil {
-		return err
+		return exerr.Wrap(err, "Failed to [ping] sql database").Build()
 	}
 	return nil
 }
@@ -113,16 +132,47 @@ func (db *database) BeginTransaction(ctx context.Context, iso sql.IsolationLevel
 
 	xtx, err := db.db.BeginTxx(ctx, &sql.TxOptions{Isolation: iso})
 	if err != nil {
-		return nil, err
+		return nil, exerr.Wrap(err, "Failed to start sql transaction").Build()
 	}
 
 	for _, v := range db.lstr {
 		v.PostTxBegin(txid, err)
 	}
 
-	return NewTransaction(xtx, txid, db.lstr), nil
+	return NewTransaction(xtx, txid, db), nil
 }
 
 func (db *database) Exit() error {
 	return db.db.Close()
 }
+
+func (db *database) ListConverter() []DBTypeConverter {
+	return db.conv
+}
+
+func (db *database) RegisterConverter(conv DBTypeConverter) {
+	db.conv = langext.ArrFilter(db.conv, func(v DBTypeConverter) bool { return v.ModelTypeString() != conv.ModelTypeString() })
+	db.conv = append(db.conv, conv)
+}
+
+func (db *database) registerDefaultConverter() {
+	db.RegisterConverter(ConverterBoolToBit)
+
+	db.RegisterConverter(ConverterTimeToUnixMillis)
+
+	db.RegisterConverter(ConverterRFCUnixMilliTimeToUnixMillis)
+	db.RegisterConverter(ConverterRFCUnixNanoTimeToUnixNanos)
+	db.RegisterConverter(ConverterRFCUnixTimeToUnixSeconds)
+	db.RegisterConverter(ConverterRFC339TimeToString)
+	db.RegisterConverter(ConverterRFC339NanoTimeToString)
+	db.RegisterConverter(ConverterRFCDateToString)
+	db.RegisterConverter(ConverterRFCTimeToString)
+	db.RegisterConverter(ConverterRFCSecondsF64ToString)
+
+	db.RegisterConverter(ConverterJsonObjToString)
+	db.RegisterConverter(ConverterJsonArrToString)
+
+	db.RegisterConverter(ConverterExErrCategoryToString)
+	db.RegisterConverter(ConverterExErrSeverityToString)
+	db.RegisterConverter(ConverterExErrTypeToString)
+}

sq/filter.go

@@ -0,0 +1,49 @@
+package sq
+
+import ct "gogs.mikescher.com/BlackForestBytes/goext/cursortoken"
+
+type FilterSort struct {
+	Field     string
+	Direction ct.SortDirection
+}
+
+type PaginateFilter interface {
+	SQL(params PP) (filterClause string, joinClause string, joinTables []string)
+	Sort() []FilterSort
+}
+
+type genericPaginateFilter struct {
+	sql  func(params PP) (filterClause string, joinClause string, joinTables []string)
+	sort func() []FilterSort
+}
+
+func (g genericPaginateFilter) SQL(params PP) (filterClause string, joinClause string, joinTables []string) {
+	return g.sql(params)
+}
+
+func (g genericPaginateFilter) Sort() []FilterSort {
+	return g.sort()
+}
+
+func NewPaginateFilter(sql func(params PP) (filterClause string, joinClause string, joinTables []string), sort []FilterSort) PaginateFilter {
+	return genericPaginateFilter{
+		sql: func(params PP) (filterClause string, joinClause string, joinTables []string) {
+			return sql(params)
+		},
+		sort: func() []FilterSort {
+			return sort
+		},
+	}
+}
+
+func NewSimplePaginateFilter(filterClause string, filterParams PP, sort []FilterSort) PaginateFilter {
+	return genericPaginateFilter{
+		sql: func(params PP) (string, string, []string) {
+			params.AddAll(filterParams)
+			return filterClause, "", nil
+		},
+		sort: func() []FilterSort {
+			return sort
+		},
+	}
+}

sq/hasher.go

@@ -13,7 +13,9 @@ import (
 	"strings"
 )
 
-func HashSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
+// HashMattnSqliteSchema
+// use if github.com/glebarez/go-sqlite
+func HashMattnSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
 	dbdir := os.TempDir()
 	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
 
@@ -29,7 +31,35 @@ func HashSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
 		return "", err
 	}
 
-	db := NewDB(xdb)
+	db := NewDB(xdb, DBOptions{})
+
+	_, err = db.Exec(ctx, schemaStr, PP{})
+	if err != nil {
+		return "", err
+	}
+
+	return HashSqliteDatabase(ctx, db)
+}
+
+// HashGoSqliteSchema
+// use if mattn/go-sqlite3
+func HashGoSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
+	dbdir := os.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	err := os.MkdirAll(dbdir, os.ModePerm)
+	if err != nil {
+		return "", err
+	}
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb, err := sqlx.Open("sqlite", url)
+	if err != nil {
+		return "", err
+	}
+
+	db := NewDB(xdb, DBOptions{})
 
 	_, err = db.Exec(ctx, schemaStr, PP{})
 	if err != nil {
@@ -91,7 +121,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 	if err != nil {
 		return "", err
 	}
-	tableList, err := ScanAll[tabInfo](rowsTableList, SModeFast, Unsafe, true)
+	tableList, err := ScanAll[tabInfo](ctx, db, rowsTableList, SModeFast, Unsafe, true)
 	if err != nil {
 		return "", err
 	}
@@ -113,7 +143,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 				return "", err
 			}
 
-			columnList, err := ScanAll[colInfo](rowsColumnList, SModeFast, Unsafe, true)
+			columnList, err := ScanAll[colInfo](ctx, db, rowsColumnList, SModeFast, Unsafe, true)
 			if err != nil {
 				return "", err
 			}
@@ -128,7 +158,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 			if err != nil {
 				return "", err
 			}
-			idxList, err := ScanAll[idxInfo](rowsIdxList, SModeFast, Unsafe, true)
+			idxList, err := ScanAll[idxInfo](ctx, db, rowsIdxList, SModeFast, Unsafe, true)
 			if err != nil {
 				return "", err
 			}
@@ -143,7 +173,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 			if err != nil {
 				return "", err
 			}
-			fkyList, err := ScanAll[fkyInfo](rowsIdxList, SModeFast, Unsafe, true)
+			fkyList, err := ScanAll[fkyInfo](ctx, db, rowsIdxList, SModeFast, Unsafe, true)
 			if err != nil {
 				return "", err
 			}

sq/json.go

@@ -0,0 +1,59 @@
+package sq
+
+import "encoding/json"
+
+type JsonObj map[string]any
+
+func (j JsonObj) MarshalToDB(v JsonObj) (string, error) {
+	mrsh, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}
+
+func (j JsonObj) UnmarshalToModel(v string) (JsonObj, error) {
+	var mrsh JsonObj
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return JsonObj{}, err
+	}
+	return mrsh, nil
+}
+
+type JsonArr []any
+
+func (j JsonArr) MarshalToDB(v JsonArr) (string, error) {
+	mrsh, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}
+
+func (j JsonArr) UnmarshalToModel(v string) (JsonArr, error) {
+	var mrsh JsonArr
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return JsonArr{}, err
+	}
+	return mrsh, nil
+}
+
+type AutoJson[T any] struct {
+	Value T
+}
+
+func (j AutoJson[T]) MarshalToDB(v AutoJson[T]) (string, error) {
+	mrsh, err := json.Marshal(v.Value)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}
+
+func (j AutoJson[T]) UnmarshalToModel(v string) (AutoJson[T], error) {
+	mrsh := *new(T)
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return AutoJson[T]{}, err
+	}
+	return AutoJson[T]{Value: mrsh}, nil
+}

sq/listener.go

@@ -17,3 +17,172 @@ type Listener interface {
 	PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 	PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 }
+
+type genListener struct {
+	prePing        func(ctx context.Context) error
+	preTxBegin     func(ctx context.Context, txid uint16) error
+	preTxCommit    func(txid uint16) error
+	preTxRollback  func(txid uint16) error
+	preQuery       func(ctx context.Context, txID *uint16, sql *string, params *PP) error
+	preExec        func(ctx context.Context, txID *uint16, sql *string, params *PP) error
+	postPing       func(result error)
+	postTxBegin    func(txid uint16, result error)
+	postTxCommit   func(txid uint16, result error)
+	postTxRollback func(txid uint16, result error)
+	postQuery      func(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+	postExec       func(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+}
+
+func (g genListener) PrePing(ctx context.Context) error {
+	if g.prePing != nil {
+		return g.prePing(ctx)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxBegin(ctx context.Context, txid uint16) error {
+	if g.preTxBegin != nil {
+		return g.preTxBegin(ctx, txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxCommit(txid uint16) error {
+	if g.preTxCommit != nil {
+		return g.preTxCommit(txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxRollback(txid uint16) error {
+	if g.preTxRollback != nil {
+		return g.preTxRollback(txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreQuery(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+	if g.preQuery != nil {
+		return g.preQuery(ctx, txID, sql, params)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreExec(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+	if g.preExec != nil {
+		return g.preExec(ctx, txID, sql, params)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PostPing(result error) {
+	if g.postPing != nil {
+		g.postPing(result)
+	}
+}
+
+func (g genListener) PostTxBegin(txid uint16, result error) {
+	if g.postTxBegin != nil {
+		g.postTxBegin(txid, result)
+	}
+}
+
+func (g genListener) PostTxCommit(txid uint16, result error) {
+	if g.postTxCommit != nil {
+		g.postTxCommit(txid, result)
+	}
+}
+
+func (g genListener) PostTxRollback(txid uint16, result error) {
+	if g.postTxRollback != nil {
+		g.postTxRollback(txid, result)
+	}
+}
+
+func (g genListener) PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+	if g.postQuery != nil {
+		g.postQuery(txID, sqlOriginal, sqlReal, params)
+	}
+}
+
+func (g genListener) PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+	if g.postExec != nil {
+		g.postExec(txID, sqlOriginal, sqlReal, params)
+	}
+}
+
+func NewPrePingListener(f func(ctx context.Context) error) Listener {
+	return genListener{prePing: f}
+}
+
+func NewPreTxBeginListener(f func(ctx context.Context, txid uint16) error) Listener {
+	return genListener{preTxBegin: f}
+}
+
+func NewPreTxCommitListener(f func(txid uint16) error) Listener {
+	return genListener{preTxCommit: f}
+}
+
+func NewPreTxRollbackListener(f func(txid uint16) error) Listener {
+	return genListener{preTxRollback: f}
+}
+
+func NewPreQueryListener(f func(ctx context.Context, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{preQuery: f}
+}
+
+func NewPreExecListener(f func(ctx context.Context, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{preExec: f}
+}
+
+func NewPreListener(f func(ctx context.Context, cmdtype string, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{
+		preExec: func(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+			return f(ctx, "EXEC", txID, sql, params)
+		},
+		preQuery: func(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+			return f(ctx, "QUERY", txID, sql, params)
+		},
+	}
+}
+
+func NewPostPingListener(f func(result error)) Listener {
+	return genListener{postPing: f}
+}
+
+func NewPostTxBeginListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxBegin: f}
+}
+
+func NewPostTxCommitListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxCommit: f}
+}
+
+func NewPostTxRollbackListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxRollback: f}
+}
+
+func NewPostQueryListener(f func(txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{postQuery: f}
+}
+
+func NewPostExecListener(f func(txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{postExec: f}
+}
+
+func NewPostListener(f func(cmdtype string, txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{
+		postExec: func(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+			f("EXEC", txID, sqlOriginal, sqlReal, params)
+		},
+		postQuery: func(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+			f("QUERY", txID, sqlOriginal, sqlReal, params)
+		},
+	}
+}

sq/main_test.go

@@ -0,0 +1,15 @@
+package sq
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	if !exerr.Initialized() {
+		exerr.Init(exerr.ErrorPackageConfigInit{ZeroLogErrTraces: langext.PFalse, ZeroLogAllTraces: langext.PFalse})
+	}
+	os.Exit(m.Run())
+}