v0.0.386 InsertAndQuerySingle

(cherry picked from commit c0443af63b)

.gitea/workflows/tests.yml

@@ -0,0 +1,55 @@
+
+# https://docs.gitea.com/next/usage/actions/quickstart
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
+
+name: Build Docker and Deploy
+run-name: Build & Deploy ${{ gitea.ref }} on ${{ gitea.actor }}
+
+on:
+  push:
+    branches:
+      - '*'
+      - '**'
+
+
+jobs:
+  run_tests:
+    name: Run goext test-suite  
+    runs-on: bfb-cicd-latest
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Setup go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: '${{ gitea.workspace }}/go.mod'
+
+      - name: Setup packages
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: curl python3
+          version: 1.0
+
+      - name: go version
+        run: go version
+
+      - name: Run tests
+        run: cd "${{ gitea.workspace }}" && make test
+
+      - name: Send failure mail
+        if: failure()
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.fastmail.com
+          server_port: 465
+          secure: true
+          username: ${{secrets.MAIL_USERNAME}}
+          password: ${{secrets.MAIL_PASSWORD}}
+          subject: Pipeline on '${{ gitea.repository }}' failed
+          to: ${{ steps.commiter_info.outputs.MAIL }}
+          from: Gitea Actions <gitea_actions@blackforestbytes.de>
+          body: "Go to https://gogs.blackforestbytes.com/${{ gitea.repository }}/actions"
+

.idea/goext.iml

@@ -1,6 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="WEB_MODULE" version="4">
-  <component name="Go" enabled="true" />
+  <component name="Go" enabled="true">
+    <buildTags>
+      <option name="goVersion" value="1.19" />
+    </buildTags>
+  </component>
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
     <orderEntry type="inheritedJdk" />

.idea/golinter.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoLinterSettings">
+    <option name="checkGoLinterExe" value="false" />
+  </component>
+</project>

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +1,7 @@
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="GoMixedReceiverTypes" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
     <inspection_tool class="LanguageDetectionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
     <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
       <option name="processCode" value="true" />

.idea/sqldialects.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/sq/sq_test.go" dialect="SQLite" />
+  </component>
+</project>

Makefile

@@ -3,7 +3,15 @@ run:
 	echo "This is a library - can't be run" && false
 
 test:
-	go test ./...
+	# go test ./...
+	which gotestsum || go install gotest.tools/gotestsum@latest
+	gotestsum --format "testname" -- -tags="timetzdata sqlite_fts5 sqlite_foreign_keys" "./..."
+
+test-in-docker:
+	tag="goext_temp_test_image:$(shell uuidgen | tr -d '-')";        \
+	docker build --tag $$tag . -f .gitea/workflows/Dockerfile_tests; \
+	docker run --rm $$tag;                                           \
+	docker rmi $$tag
 
 version:
 	_data/version.sh

README.md

@@ -5,4 +5,41 @@ A collection of general & useful library methods
 
 This should not have any heavy dependencies (gin, mongo, etc) and add missing basic language features...
 
-Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
+Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
+
+
+### Packages:
+
+| Name        | Maintainer | Description                                                                                                   |
+|-------------|------------|---------------------------------------------------------------------------------------------------------------|
+| langext     | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
+| mathext     | Mike       | Utility/Helper functions for math                                                                             |
+| cryptext    | Mike       | Utility/Helper functions for encryption                                                                       |
+| syncext     | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
+| dataext     | Mike       | Various useful data structures                                                                                |
+| zipext      | Mike       | Utility for zip/gzip/tar etc                                                                                  |
+| reflectext  | Mike       | Utility for golang reflection                                                                                 |
+| fsext       | Mike       | Utility for filesytem access                                                                                  |
+|             |            |                                                                                                               |
+| mongoext    | Mike       | Utility/Helper functions for mongodb                                                                          |
+| cursortoken | Mike       | MongoDB cursortoken implementation                                                                            |
+| pagination  | Mike       | Pagination implementation                                                                                     |
+|             |            |                                                                                                               |
+| totpext     | Mike       | Implementation of TOTP (2-Factor-Auth)                                                                        |
+| termext     | Mike       | Utilities for terminals (mostly color output)                                                                 |
+| confext     | Mike       | Parses environment configuration into structs                                                                 |
+| cmdext      | Mike       | Runner for external commands/processes                                                                        |
+|             |            |                                                                                                               |
+| sq          | Mike       | Utility functions for sql based databases                                                                     |
+| tst         | Mike       | Utility functions for unit tests                                                                              |
+|             |            |                                                                                                               |
+| rfctime     | Mike       | Classes for time seriallization, with different marshallign method for mongo and json                         |
+| gojson      | Mike       | Same interface for marshalling/unmarshalling as go/json, except with proper serialization of null arrays/maps |
+|             |            |                                                                                                               |
+| bfcodegen   | Mike       | Various codegen tools (run via go generate)                                                                   |
+|             |            |                                                                                                               |
+| rext        | Mike       | Regex Wrapper, wraps regexp with a better interface                                                           |
+| wmo         | Mike       | Mongo Wrapper, wraps mongodb with a better interface                                                          |
+|             |            |                                                                                                               |
+| scn         | Mike       | SimpleCloudNotifier                                                                                           |
+|             |            |                                                                                                               |

TODO.md

@@ -0,0 +1,7 @@
+
+
+ - cronext
+
+ - rfctime.DateOnly
+ - rfctime.HMSTimeOnly
+ - rfctime.NanoTimeOnly

_data/version.sh

@@ -7,6 +7,29 @@ set -o pipefail  # Return value of a pipeline is the value of the last (rightmos
 IFS=$'\n\t'      # Set $IFS to only newline and tab.
 
 
+function black() { echo -e "\x1B[30m $1 \x1B[0m"; }
+function red()   { echo -e "\x1B[31m $1 \x1B[0m"; }
+function green() { echo -e "\x1B[32m $1 \x1B[0m"; }
+function yellow(){ echo -e "\x1B[33m $1 \x1B[0m"; }
+function blue()  { echo -e "\x1B[34m $1 \x1B[0m"; }
+function purple(){ echo -e "\x1B[35m $1 \x1B[0m"; }
+function cyan()  { echo -e "\x1B[36m $1 \x1B[0m"; }
+function white() { echo -e "\x1B[37m $1 \x1B[0m"; }
+
+if [ "$( git rev-parse --abbrev-ref HEAD )" != "master" ]; then
+  >&2 red "[ERROR] Can only create versions of <master>"
+  exit 1
+fi
+
+echo ""
+echo -n "Insert optional commit message: "
+read commitMessage
+echo ""
+
+git pull --ff
+
+go get -u ./...
+
 curr_vers=$(git describe --tags --abbrev=0 | sed 's/v//g')
 
 next_ver=$(echo "$curr_vers" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{if(length($NF+1)>length($NF))$(NF-1)++; $NF=sprintf("%0*d", length($NF), ($NF+1)%(10^length($NF))); print}')
@@ -16,9 +39,22 @@ echo "> Current Version: ${curr_vers}"
 echo "> Next    Version: ${next_ver}"
 echo ""
 
+printf "package goext\n\nconst GoextVersion = \"%s\"\n\nconst GoextVersionTimestamp = \"%s\"\n" "${next_ver}" "$( date +"%Y-%m-%dT%H:%M:%S%z" )" > "goextVersion.go"
+
 git add --verbose .
 
-git commit -a -m "v${next_ver}"
+msg="v${next_ver}"
+
+if [[ "$commitMessage" != "" ]]; then
+  msg="${msg} ${commitMessage}"
+fi
+
+
+if [ $# -gt 0 ]; then
+  msg="$1"
+fi
+
+git commit -a -m "${msg}"
 
 git tag "v${next_ver}"
 

bfcodegen/csid-generate.go

@@ -0,0 +1,182 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"go/format"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type CSIDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+	Prefix       string
+}
+
+var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
+
+var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed csid-generate.template
+var templateCSIDGenerateText string
+
+func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexCSIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[CSIDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[CSIDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\n")
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	fdata, err := format.Source([]byte(fmtCSIDOutput(newChecksum, allIDs, pkgname)))
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(destFile, fdata, 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexCSIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexCSIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := CSIDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+				Prefix:       match.GroupByName("prefix").Value(),
+			}
+			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtCSIDOutput(cs string, ids []CSIDDef, pkgname string) string {
+	templ := template.Must(template.New("csid-generate").Parse(templateCSIDGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/csid-generate.template

@@ -0,0 +1,190 @@
+// Code generated by csid-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "crypto/rand"
+import "crypto/sha256"
+import "fmt"
+import "github.com/go-playground/validator/v10"
+import "github.com/rs/zerolog/log"
+import "gogs.mikescher.com/BlackForestBytes/goext/exerr"
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+import "gogs.mikescher.com/BlackForestBytes/goext/rext"
+import "math/big"
+import "reflect"
+import "regexp"
+import "strings"
+
+const ChecksumCharsetIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+const idlen = 24
+
+const checklen = 1
+
+const idCharset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+const idCharsetLen = len(idCharset)
+
+var charSetReverseMap = generateCharsetMap()
+
+const ({{range .IDs}}
+	prefix{{.Name}} = "{{.Prefix}}" {{end}}
+)
+
+var ({{range .IDs}}
+	regex{{.Name}} = generateRegex(prefix{{.Name}}) {{end}}
+)
+
+func generateRegex(prefix string) rext.Regex {
+	return rext.W(regexp.MustCompile(fmt.Sprintf("^%s[%s]{%d}[%s]{%d}$", prefix, idCharset, idlen-len(prefix)-checklen, idCharset, checklen)))
+}
+
+func generateCharsetMap() []int {
+	result := make([]int, 128)
+	for i := 0; i < len(result); i++ {
+		result[i] = -1
+	}
+	for idx, chr := range idCharset {
+		result[int(chr)] = idx
+	}
+	return result
+}
+
+func generateID(prefix string) string {
+	k := ""
+	csMax := big.NewInt(int64(idCharsetLen))
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		v, err := rand.Int(rand.Reader, csMax)
+		if err != nil {
+			panic(err)
+		}
+		v64 := v.Int64()
+		k += string(idCharset[v64])
+		checksum = (checksum + int(v64)) % (idCharsetLen)
+	}
+	checkstr := string(idCharset[checksum%idCharsetLen])
+	return prefix + k + checkstr
+}
+
+func generateIDFromSeed(prefix string, seed string) string {
+	h := sha256.New()
+
+	iddata := ""
+	for len(iddata) < idlen-len(prefix)-checklen {
+		h.Write([]byte(seed))
+		bs := h.Sum(nil)
+		iddata += langext.NewAnyBaseConverter(idCharset).Encode(bs)
+	}
+
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		ichr := int(iddata[i])
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	return prefix + iddata[:(idlen-len(prefix)-checklen)] + checkstr
+}
+
+func validateID(prefix string, value string) error {
+	if len(value) != idlen {
+		return exerr.New(exerr.TypeInvalidCSID, "id has the wrong length").Str("value", value).Build()
+	}
+
+	if !strings.HasPrefix(value, prefix) {
+		return exerr.New(exerr.TypeInvalidCSID, "id is missing the correct prefix").Str("value", value).Str("prefix", prefix).Build()
+	}
+
+	checksum := 0
+	for i := len(prefix); i < len(value)-checklen; i++ {
+		ichr := int(value[i])
+		if ichr < 0 || ichr >= len(charSetReverseMap) || charSetReverseMap[ichr] == -1 {
+			return exerr.New(exerr.TypeInvalidCSID, "id contains invalid characters").Str("value", value).Build()
+		}
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	if !strings.HasSuffix(value, checkstr) {
+		return exerr.New(exerr.TypeInvalidCSID, "id checkstring is invalid").Str("value", value).Str("checkstr", checkstr).Build()
+	}
+
+	return nil
+}
+
+func getRawData(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[len(prefix) : idlen-checklen]
+}
+
+func getCheckString(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[idlen-checklen:]
+}
+
+func ValidateEntityID(vfl validator.FieldLevel) bool {
+	if !vfl.Field().CanInterface() {
+		log.Error().Msgf("Failed to validate EntityID (cannot interface ?!?)")
+		return false
+	}
+
+	ifvalue := vfl.Field().Interface()
+
+	if value1, ok := ifvalue.(EntityID); ok {
+
+		if vfl.Field().Type().Kind() == reflect.Pointer && langext.IsNil(value1) {
+			return true
+		}
+
+		if err := value1.Valid(); err != nil {
+			log.Debug().Msgf("Failed to validate EntityID '%s' (%s)", value1.String(), err.Error())
+			return false
+		} else {
+			return true
+		}
+
+	} else {
+		log.Error().Msgf("Failed to validate EntityID (wrong type: %T)", ifvalue)
+		return false
+	}
+}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(generateID(prefix{{.Name}}))
+}
+
+func (id {{.Name}}) Valid() error {
+	return validateID(prefix{{.Name}}, string(id))
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) Prefix() string {
+	return prefix{{.Name}}
+}
+
+func (id {{.Name}}) Raw() string {
+	return getRawData(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) CheckString() string {
+	return getCheckString(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) Regex() rext.Regex {
+	return regex{{.Name}}
+}
+
+{{end}}

bfcodegen/csid-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var CSIDExampleModels1 []byte
+
+func TestGenerateCSIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, CSIDExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/csid_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/enum-generate.go

@@ -0,0 +1,355 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go/format"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type EnumDefVal struct {
+	VarName     string
+	Value       string
+	Description *string
+	Data        *map[string]any
+	RawComment  *string
+}
+
+type EnumDef struct {
+	File         string
+	FileRelative string
+	EnumTypeName string
+	Type         string
+	Values       []EnumDefVal
+}
+
+var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
+
+var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:\s\-.]+"|[0-9]+))\s*(//(?P<comm>.*))?.*$`))
+
+var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed enum-generate.template
+var templateEnumGenerateText string
+
+func GenerateEnumSpecs(sourceDir string, destFile string) error {
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexEnumChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, true)
+	if err != nil {
+		return err
+	}
+
+	if !changed {
+		return nil
+	}
+
+	err = os.WriteFile(destFile, []byte(gocode), 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool) (string, string, bool, error) {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return "", "", false, err
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return "", "", false, err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[EnumGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[EnumGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return "", oldChecksum, false, nil
+	}
+
+	allEnums := make([]EnumDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return "", "", false, err
+		}
+
+		fmt.Printf("\n")
+
+		allEnums = append(allEnums, fileEnums...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return "", "", false, errors.New("no package name found in any file")
+	}
+
+	rdata := fmtEnumOutput(newChecksum, allEnums, pkgname)
+
+	if !gofmt {
+		return rdata, newChecksum, true, nil
+	}
+
+	fdata, err := format.Source([]byte(rdata))
+	if err != nil {
+		return "", "", false, err
+	}
+
+	return string(fdata), newChecksum, true, nil
+}
+
+func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	enums := make([]EnumDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexEnumPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexEnumDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := EnumDef{
+				File:         fn,
+				FileRelative: rfp,
+				EnumTypeName: match.GroupByName("name").Value(),
+				Type:         match.GroupByName("type").Value(),
+				Values:       make([]EnumDefVal, 0),
+			}
+			enums = append(enums, def)
+			fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+		}
+
+		if match, ok := rexEnumValueDef.MatchFirst(line); ok {
+			typename := match.GroupByName("type").Value()
+
+			comment := match.GroupByNameOrEmpty("comm").ValueOrNil()
+			var descr *string = nil
+			var data *map[string]any = nil
+			if comment != nil {
+				comment = langext.Ptr(strings.TrimSpace(*comment))
+				if strings.HasPrefix(*comment, "{") {
+					if v, ok := tryParseDataComment(*comment); ok {
+						data = &v
+						if anyDataDescr, ok := v["description"]; ok {
+							if dataDescr, ok := anyDataDescr.(string); ok {
+								descr = &dataDescr
+							}
+						}
+					} else {
+						descr = comment
+					}
+				} else {
+					descr = comment
+				}
+			}
+
+			def := EnumDefVal{
+				VarName:     match.GroupByName("name").Value(),
+				Value:       match.GroupByName("value").Value(),
+				RawComment:  comment,
+				Description: descr,
+				Data:        data,
+			}
+
+			found := false
+			for i, v := range enums {
+				if v.EnumTypeName == typename {
+					enums[i].Values = append(enums[i].Values, def)
+					found = true
+					if def.Description != nil {
+						fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
+					} else {
+						fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+					}
+					break
+				}
+			}
+			if !found {
+				fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+			}
+		}
+	}
+
+	return enums, pkgname, nil
+}
+
+func tryParseDataComment(s string) (map[string]any, bool) {
+
+	r := make(map[string]any)
+
+	err := json.Unmarshal([]byte(s), &r)
+	if err != nil {
+		return nil, false
+	}
+
+	for _, v := range r {
+
+		rv := reflect.ValueOf(v)
+
+		if rv.Kind() == reflect.Ptr && rv.IsNil() {
+			continue
+		}
+		if rv.Kind() == reflect.Bool {
+			continue
+		}
+		if rv.Kind() == reflect.String {
+			continue
+		}
+		if rv.Kind() == reflect.Int64 {
+			continue
+		}
+		if rv.Kind() == reflect.Float64 {
+			continue
+		}
+
+		return nil, false
+	}
+
+	return r, true
+}
+
+func fmtEnumOutput(cs string, enums []EnumDef, pkgname string) string {
+
+	templ := template.New("enum-generate")
+
+	templ = templ.Funcs(template.FuncMap{
+		"boolToStr": func(b bool) string { return langext.Conditional(b, "true", "false") },
+		"deref":     func(v *string) string { return *v },
+		"trimSpace": func(str string) string { return strings.TrimSpace(str) },
+		"hasStr":    func(v EnumDef) bool { return v.Type == "string" },
+		"hasDescr": func(v EnumDef) bool {
+			return langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Description != nil })
+		},
+		"hasData": func(v EnumDef) bool {
+			return len(v.Values) > 0 && langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Data != nil })
+		},
+		"gostr": func(v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"goobj": func(name string, v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"godatakey": func(v string) string {
+			return strings.ToUpper(v[0:1]) + v[1:]
+		},
+		"godatavalue": func(v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"godatatype": func(v any) string {
+			return fmt.Sprintf("%T", v)
+		},
+		"mapindex": func(v map[string]any, k string) any {
+			return v[k]
+		},
+		"generalDataKeys": func(v EnumDef) map[string]string {
+			r0 := make(map[string]int)
+
+			for _, eval := range v.Values {
+				for k := range *eval.Data {
+					if ctr, ok := r0[k]; ok {
+						r0[k] = ctr + 1
+					} else {
+						r0[k] = 1
+					}
+				}
+			}
+
+			r1 := langext.MapToArr(r0)
+			r2 := langext.ArrFilter(r1, func(p langext.MapEntry[string, int]) bool { return p.Value == len(v.Values) })
+			r3 := langext.ArrMap(r2, func(p langext.MapEntry[string, int]) string { return p.Key })
+			r4 := langext.ArrToKVMap(r3, func(p string) string { return p }, func(p string) string { return fmt.Sprintf("%T", (*v.Values[0].Data)[p]) })
+
+			return r4
+		},
+	})
+
+	templ = template.Must(templ.Parse(templateEnumGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"Enums":        enums,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/enum-generate.template

@@ -0,0 +1,137 @@
+// Code generated by enum-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+import "gogs.mikescher.com/BlackForestBytes/goext/enums"
+
+const ChecksumEnumGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{range .Enums}}
+
+{{ $hasStr   := ( . | hasStr   ) }}
+{{ $hasDescr := ( . | hasDescr ) }}
+{{ $hasData  := ( . | hasData  ) }}
+
+// ================================ {{.EnumTypeName}} ================================
+//
+// File:       {{.FileRelative}}
+// StringEnum: {{$hasStr   | boolToStr}}
+// DescrEnum:  {{$hasDescr | boolToStr}}
+// DataEnum:   {{$hasData  | boolToStr}}
+//
+
+{{ $typename := .EnumTypeName }}
+{{ $enumdef  := . }}
+
+var __{{.EnumTypeName}}Values = []{{.EnumTypeName}}{ {{range .Values}}
+	{{.VarName}}, {{end}}
+}
+
+{{if $hasDescr}}
+var __{{.EnumTypeName}}Descriptions = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: {{.Description | deref | trimSpace | gostr}}, {{end}}
+}
+{{end}}
+
+{{if $hasData}}
+type {{ .EnumTypeName }}Data struct { {{ range $datakey, $datatype := ($enumdef | generalDataKeys)  }}
+    {{ $datakey | godatakey }} {{ $datatype }} `json:"{{ $datakey }}"` {{ end }}
+}
+
+var __{{.EnumTypeName}}Data = map[{{.EnumTypeName}}]{{.EnumTypeName}}Data{ {{range .Values}} {{ $enumvalue := . }}
+	{{.VarName}}: {{ $typename }}Data{ {{ range $datakey, $datatype := $enumdef | generalDataKeys  }}
+	    {{ $datakey | godatakey }}: {{ (mapindex $enumvalue.Data $datakey) | godatavalue }}, {{ end }}
+	}, {{end}}
+}
+{{end}}
+
+var __{{.EnumTypeName}}Varnames = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: "{{.VarName}}", {{end}}
+}
+
+func (e {{.EnumTypeName}}) Valid() bool {
+	return langext.InArray(e, __{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func (e {{.EnumTypeName}}) ValuesAny() []any {
+	return langext.ArrCastToAny(__{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) ValuesMeta() []enums.EnumMetaValue {
+	return {{.EnumTypeName}}ValuesMeta()
+}
+
+{{if $hasStr}}
+func (e {{.EnumTypeName}}) String() string {
+	return string(e)
+}
+{{end}}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) Description() string {
+	if d, ok := __{{.EnumTypeName}}Descriptions[e]; ok {
+		return d
+	}
+	return ""
+}
+{{end}}
+
+{{if $hasData}}
+func (e {{.EnumTypeName}}) Data() {{.EnumTypeName}}Data {
+	if d, ok := __{{.EnumTypeName}}Data[e]; ok {
+		return d
+	}
+	return {{.EnumTypeName}}Data{}
+}
+{{end}}
+
+func (e {{.EnumTypeName}}) VarName() string {
+	if d, ok := __{{.EnumTypeName}}Varnames[e]; ok {
+		return d
+	}
+	return ""
+}
+
+func (e {{.EnumTypeName}}) Meta() enums.EnumMetaValue {
+    {{if $hasDescr}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())} {{else}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil} {{end}}
+}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) DescriptionMeta() enums.EnumDescriptionMetaValue {
+    return enums.EnumDescriptionMetaValue{VarName: e.VarName(), Value: e, Description: e.Description()}
+}
+{{end}}
+
+func Parse{{.EnumTypeName}}(vv string) ({{.EnumTypeName}}, bool) {
+	for _, ev := range __{{.EnumTypeName}}Values {
+		if string(ev) == vv {
+			return ev, true
+		}
+	}
+	return "", false
+}
+
+func {{.EnumTypeName}}Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func {{.EnumTypeName}}ValuesMeta() []enums.EnumMetaValue {
+	return []enums.EnumMetaValue{ {{range .Values}}
+            {{.VarName}}.Meta(), {{end}}
+	}
+}
+
+{{if $hasDescr}}
+func {{.EnumTypeName}}ValuesDescriptionMeta() []enums.EnumDescriptionMetaValue {
+	return []enums.EnumDescriptionMetaValue{ {{range .Values}}
+            {{.VarName}}.DescriptionMeta(), {{end}}
+	}
+}
+{{end}}
+
+{{end}}

bfcodegen/enum-generate_test.go

@@ -0,0 +1,91 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var EnumExampleModels1 []byte
+
+//go:embed _test_example_2.tgz
+var EnumExampleModels2 []byte
+
+func TestGenerateEnumSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, EnumExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	tst.AssertNoErr(t, err)
+
+	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, cs1, cs2)
+	tst.AssertEqual(t, s1, s2)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(s1)
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}
+
+func TestGenerateEnumSpecsData(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, EnumExampleModels2, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true)
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(s1)
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/id-generate.go

@@ -0,0 +1,183 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"go/format"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type IDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+}
+
+var rexIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@id:type).*$`))
+
+var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed id-generate.template
+var templateIDGenerateText string
+
+func GenerateIDSpecs(sourceDir string, destFile string) error {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[IDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[IDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]IDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\n")
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	fdata, err := format.Source([]byte(fmtIDOutput(newChecksum, allIDs, pkgname)))
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(destFile, fdata, 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]IDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := IDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+			}
+			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtIDOutput(cs string, ids []IDDef, pkgname string) string {
+	templ := template.Must(template.New("id-generate").Parse(templateIDGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	anyDef := langext.ArrFirstOrNil(ids, func(def IDDef) bool { return def.Name == "AnyID" || def.Name == "AnyId" })
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+		"AnyDef":       anyDef,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/id-generate.template

@@ -0,0 +1,47 @@
+// Code generated by id-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "go.mongodb.org/mongo-driver/bson"
+import "go.mongodb.org/mongo-driver/bson/bsontype"
+import "go.mongodb.org/mongo-driver/bson/primitive"
+import "gogs.mikescher.com/BlackForestBytes/goext/exerr"
+
+const ChecksumIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func (i {{.Name}}) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	if objId, err := primitive.ObjectIDFromHex(string(i)); err == nil {
+		return bson.MarshalValue(objId)
+	} else {
+		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, "Failed to marshal {{.Name}}("+i.String()+") to ObjectId").Str("value", string(i)).Type("type", i).Build()
+	}
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) ObjID() (primitive.ObjectID, error) {
+	return primitive.ObjectIDFromHex(string(i))
+}
+
+func (i {{.Name}}) Valid() bool {
+	_, err := primitive.ObjectIDFromHex(string(i))
+	return err == nil
+}
+
+{{if ne $.AnyDef nil}}
+func (i {{.Name}}) AsAny() {{$.AnyDef.Name}} {
+	return {{$.AnyDef.Name}}(i)
+}
+{{end}}
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(primitive.NewObjectID().Hex())
+}
+
+{{end}}

bfcodegen/id-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var IDExampleModels1 []byte
+
+func TestGenerateIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, IDExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/id_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

cmdext/builder.go

@@ -0,0 +1,100 @@
+package cmdext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"time"
+)
+
+type CommandRunner struct {
+	program          string
+	args             []string
+	timeout          *time.Duration
+	env              []string
+	listener         []CommandListener
+	enforceExitCodes *[]int
+	enforceNoTimeout bool
+	enforceNoStderr  bool
+}
+
+func Runner(program string) *CommandRunner {
+	return &CommandRunner{
+		program:          program,
+		args:             make([]string, 0),
+		timeout:          nil,
+		env:              make([]string, 0),
+		listener:         make([]CommandListener, 0),
+		enforceExitCodes: nil,
+		enforceNoTimeout: false,
+		enforceNoStderr:  false,
+	}
+}
+
+func (r *CommandRunner) Arg(arg string) *CommandRunner {
+	r.args = append(r.args, arg)
+	return r
+}
+
+func (r *CommandRunner) Args(arg []string) *CommandRunner {
+	r.args = append(r.args, arg...)
+	return r
+}
+
+func (r *CommandRunner) Timeout(timeout time.Duration) *CommandRunner {
+	r.timeout = &timeout
+	return r
+}
+
+func (r *CommandRunner) Env(key, value string) *CommandRunner {
+	r.env = append(r.env, fmt.Sprintf("%s=%s", key, value))
+	return r
+}
+
+func (r *CommandRunner) RawEnv(env string) *CommandRunner {
+	r.env = append(r.env, env)
+	return r
+}
+
+func (r *CommandRunner) Envs(env []string) *CommandRunner {
+	r.env = append(r.env, env...)
+	return r
+}
+
+func (r *CommandRunner) EnsureExitcode(arg ...int) *CommandRunner {
+	r.enforceExitCodes = langext.Ptr(langext.ForceArray(arg))
+	return r
+}
+
+func (r *CommandRunner) FailOnExitCode() *CommandRunner {
+	r.enforceExitCodes = langext.Ptr([]int{0})
+	return r
+}
+
+func (r *CommandRunner) FailOnTimeout() *CommandRunner {
+	r.enforceNoTimeout = true
+	return r
+}
+
+func (r *CommandRunner) FailOnStderr() *CommandRunner {
+	r.enforceNoStderr = true
+	return r
+}
+
+func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
+	r.listener = append(r.listener, lstr)
+	return r
+}
+
+func (r *CommandRunner) ListenStdout(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStdoutLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) ListenStderr(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStderrLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) Run() (CommandResult, error) {
+	return run(*r)
+}

cmdext/cmdrunner.go

@@ -1,11 +1,18 @@
 package cmdext
 
 import (
-	"bufio"
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
 	"os/exec"
 	"time"
 )
 
+var ErrExitCode = errors.New("process exited with an unexpected exitcode")
+var ErrTimeout = errors.New("process did not exit after the specified timeout")
+var ErrStderrPrint = errors.New("process did print to stderr stream")
+
 type CommandResult struct {
 	StdOut          string
 	StdErr          string
@@ -14,9 +21,10 @@ type CommandResult struct {
 	CommandTimedOut bool
 }
 
-func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+func run(opt CommandRunner) (CommandResult, error) {
+	cmd := exec.Command(opt.program, opt.args...)
 
-	cmd := exec.Command(program, args...)
+	cmd.Env = append(cmd.Env, opt.env...)
 
 	stdoutPipe, err := cmd.StdoutPipe()
 	if err != nil {
@@ -28,115 +36,161 @@ func RunCommand(program string, args []string, timeout *time.Duration) (CommandR
 		return CommandResult{}, err
 	}
 
+	preader := pipeReader{
+		lineBufferSize: langext.Ptr(128 * 1024 * 1024), // 128MB max size of a single line, is hopefully enough....
+		stdout:         stdoutPipe,
+		stderr:         stderrPipe,
+	}
+
 	err = cmd.Start()
 	if err != nil {
 		return CommandResult{}, err
 	}
 
-	errch := make(chan error, 1)
-	go func() { errch <- cmd.Wait() }()
+	type resultObj struct {
+		stdout      string
+		stderr      string
+		stdcombined string
+		err         error
+	}
 
-	combch := make(chan string, 32)
-	stopCombch := make(chan bool)
+	stderrFailChan := make(chan bool)
 
-	stdout := ""
+	outputChan := make(chan resultObj)
 	go func() {
-		scanner := bufio.NewScanner(stdoutPipe)
-		for scanner.Scan() {
-			txt := scanner.Text()
-			stdout += txt
-			combch <- txt
+		// we need to first fully read the pipes and then call Wait
+		// see https://pkg.go.dev/os/exec#Cmd.StdoutPipe
+
+		listener := make([]CommandListener, 0)
+		listener = append(listener, opt.listener...)
+
+		if opt.enforceNoStderr {
+			listener = append(listener, genericCommandListener{
+				_readRawStderr: langext.Ptr(func(v []byte) {
+					if len(v) > 0 {
+						stderrFailChan <- true
+					}
+				}),
+			})
 		}
-	}()
 
-	stderr := ""
-	go func() {
-		scanner := bufio.NewScanner(stderrPipe)
-		for scanner.Scan() {
-			txt := scanner.Text()
-			stderr += txt
-			combch <- txt
-		}
-	}()
-
-	defer func() {
-		stopCombch <- true
-	}()
-
-	stdcombined := ""
-	go func() {
-		for {
-			select {
-			case txt := <-combch:
-				stdcombined += txt
-			case <-stopCombch:
-				return
-			}
-		}
-	}()
-
-	if timeout != nil {
-
-		select {
-
-		case <-time.After(*timeout):
+		stdout, stderr, stdcombined, err := preader.Read(listener)
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
 			_ = cmd.Process.Kill()
-			return CommandResult{
-				StdOut:          stdout,
-				StdErr:          stderr,
-				StdCombined:     stdcombined,
+			return
+		}
+
+		err = cmd.Wait()
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+		} else {
+			outputChan <- resultObj{stdout, stderr, stdcombined, nil}
+		}
+
+	}()
+
+	var timeoutChan <-chan time.Time = make(chan time.Time, 1)
+	if opt.timeout != nil {
+		timeoutChan = time.After(*opt.timeout)
+	}
+
+	select {
+
+	case <-timeoutChan:
+		_ = cmd.Process.Kill()
+		for _, lstr := range opt.listener {
+			lstr.Timeout()
+		}
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
+			// most of the time the cmd.Process.Kill() should also ahve finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
 				ExitCode:        -1,
 				CommandTimedOut: true,
-			}, nil
-
-		case err := <-errch:
-			if exiterr, ok := err.(*exec.ExitError); ok {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        exiterr.ExitCode(),
-					CommandTimedOut: false,
-				}, nil
-			} else if err != nil {
-				return CommandResult{}, err
-			} else {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        0,
-					CommandTimedOut: false,
-				}, nil
 			}
-
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
 		}
 
-	} else {
+	case <-stderrFailChan:
+		_ = cmd.Process.Kill()
 
-		select {
-
-		case err := <-errch:
-			if exiterr, ok := err.(*exec.ExitError); ok {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        exiterr.ExitCode(),
-					CommandTimedOut: false,
-				}, nil
-			} else if err != nil {
-				return CommandResult{}, err
-			} else {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        0,
-					CommandTimedOut: false,
-				}, nil
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, 32*time.Millisecond); ok {
+			// most of the time the cmd.Process.Kill() should also have finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
+				ExitCode:        -1,
+				CommandTimedOut: false,
 			}
+			return res, ErrStderrPrint
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: false,
+			}
+			return res, ErrStderrPrint
+		}
 
+	case outobj := <-outputChan:
+		var exiterr *exec.ExitError
+		if errors.As(outobj.err, &exiterr) {
+			excode := exiterr.ExitCode()
+			for _, lstr := range opt.listener {
+				lstr.Finished(excode)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        excode,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(excode, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		} else if err != nil {
+			return CommandResult{}, err
+		} else {
+			for _, lstr := range opt.listener {
+				lstr.Finished(0)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        0,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(0, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
 		}
 	}
 }

cmdext/cmdrunner_test.go

@@ -0,0 +1,348 @@
+package cmdext
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestStdout(t *testing.T) {
+
+	res1, err := Runner("printf").Arg("hello").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "hello" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "hello\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStdcombined(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"1\", file=sys.stderr, flush=True); time.sleep(0.1); print(\"2\", file=sys.stdout, flush=True); time.sleep(0.1); print(\"3\", file=sys.stderr, flush=True)").
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "1\n3\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "2\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "1\n2\n3\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialRead(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", flush=True); time.sleep(5); print(\"cant see me\", flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadStderr(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, flush=True); time.sleep(5); print(\"cant see me\", file=sys.stderr, flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStdout(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stdout, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "message101" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "message101" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushed(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", end=''); time.sleep(5); print(\"cant see me\", end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushedStderr(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, end=''); time.sleep(5); print(\"cant see me\", file=sys.stderr, end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestListener(t *testing.T) {
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys;" +
+			"import time;" +
+			"print(\"message 1\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 2\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 3\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 4\", file=sys.stderr, flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 5\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"final\");").
+		ListenStdout(func(s string) { fmt.Printf("@@STDOUT <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		ListenStderr(func(s string) { fmt.Printf("@@STDERR <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		Timeout(10 * time.Second).
+		Run()
+
+	if err != nil {
+		panic(err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+}
+
+func TestLongStdout(t *testing.T) {
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"X\" * 125001 + \"\\n\"); print(\"Y\" * 125001 + \"\\n\"); print(\"Z\" * 125001 + \"\\n\");").
+		Timeout(5000 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if len(res1.StdOut) != 375009 {
+		t.Errorf("len(res1.StdOut) == '%v'", len(res1.StdOut))
+	}
+
+}
+
+func TestFailOnTimeout(t *testing.T) {
+
+	_, err := Runner("sleep").Arg("2").Timeout(200 * time.Millisecond).FailOnTimeout().Run()
+	if !errors.Is(err, ErrTimeout) {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestFailOnStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").FailOnStderr().Run()
+	if err == nil {
+		t.Errorf("no err")
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != -1 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestFailOnExitcode(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).FailOnExitCode().Run()
+	if !errors.Is(err, ErrExitCode) {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode1(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).EnsureExitcode(1).Run()
+	if err != nil {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode2(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200*time.Millisecond).EnsureExitcode(0, 2, 3).Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}

cmdext/helper.go

@@ -0,0 +1,12 @@
+package cmdext
+
+import "time"
+
+func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+	b := Runner(program)
+	b = b.Args(args)
+	if timeout != nil {
+		b = b.Timeout(*timeout)
+	}
+	return b.Run()
+}

cmdext/listener.go

@@ -0,0 +1,57 @@
+package cmdext
+
+type CommandListener interface {
+	ReadRawStdout([]byte)
+	ReadRawStderr([]byte)
+
+	ReadStdoutLine(string)
+	ReadStderrLine(string)
+
+	Finished(int)
+	Timeout()
+}
+
+type genericCommandListener struct {
+	_readRawStdout  *func([]byte)
+	_readRawStderr  *func([]byte)
+	_readStdoutLine *func(string)
+	_readStderrLine *func(string)
+	_finished       *func(int)
+	_timeout        *func()
+}
+
+func (g genericCommandListener) ReadRawStdout(v []byte) {
+	if g._readRawStdout != nil {
+		(*g._readRawStdout)(v)
+	}
+}
+
+func (g genericCommandListener) ReadRawStderr(v []byte) {
+	if g._readRawStderr != nil {
+		(*g._readRawStderr)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStdoutLine(v string) {
+	if g._readStdoutLine != nil {
+		(*g._readStdoutLine)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStderrLine(v string) {
+	if g._readStderrLine != nil {
+		(*g._readStderrLine)(v)
+	}
+}
+
+func (g genericCommandListener) Finished(v int) {
+	if g._finished != nil {
+		(*g._finished)(v)
+	}
+}
+
+func (g genericCommandListener) Timeout() {
+	if g._timeout != nil {
+		(*g._timeout)()
+	}
+}

cmdext/pipereader.go

@@ -0,0 +1,158 @@
+package cmdext
+
+import (
+	"bufio"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
+	"io"
+	"sync"
+)
+
+type pipeReader struct {
+	lineBufferSize *int
+	stdout         io.ReadCloser
+	stderr         io.ReadCloser
+}
+
+// Read ready stdout and stdin until finished
+// also splits both pipes into lines and calld the listener
+func (pr *pipeReader) Read(listener []CommandListener) (string, string, string, error) {
+	type combevt struct {
+		line string
+		stop bool
+	}
+
+	errch := make(chan error, 8)
+
+	wg := sync.WaitGroup{}
+
+	// [1] read raw stdout
+
+	wg.Add(1)
+	stdoutBufferReader, stdoutBufferWriter := io.Pipe()
+	stdout := ""
+	go func() {
+		buf := make([]byte, 128)
+		for {
+			n, err := pr.stdout.Read(buf)
+			if n > 0 {
+				txt := string(buf[:n])
+				stdout += txt
+				_, _ = stdoutBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStdout(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stdoutBufferWriter.Close()
+		wg.Done()
+	}()
+
+	// [2] read raw stderr
+
+	wg.Add(1)
+	stderrBufferReader, stderrBufferWriter := io.Pipe()
+	stderr := ""
+	go func() {
+		buf := make([]byte, 128)
+		for {
+			n, err := pr.stderr.Read(buf)
+
+			if n > 0 {
+				txt := string(buf[:n])
+				stderr += txt
+				_, _ = stderrBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStderr(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stderrBufferWriter.Close()
+		wg.Done()
+	}()
+
+	combch := make(chan combevt, 32)
+
+	// [3] collect stdout line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stdoutBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStdoutLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [4] collect stderr line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stderrBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStderrLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [5] combine stdcombined
+
+	wg.Add(1)
+	stdcombined := ""
+	go func() {
+		stopctr := 0
+		for stopctr < 2 {
+			vvv := <-combch
+			if vvv.stop {
+				stopctr++
+			} else {
+				stdcombined += vvv.line + "\n" // this comes from bufio.Scanner and has no newlines...
+			}
+		}
+		wg.Done()
+	}()
+
+	// wait for all (5) goroutines to finish
+	wg.Wait()
+
+	if err, ok := syncext.ReadNonBlocking(errch); ok {
+		return "", "", "", err
+	}
+
+	return stdout, stderr, stdcombined, nil
+}

confext/confParser.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -22,10 +23,10 @@ import (
 //
 // sub-structs are recursively parsed (if they have an env tag) and the env-variable keys are delimited by the delim parameter
 // sub-structs with `env:""` are also parsed, but the delimited is skipped (they are handled as if they were one level higher)
-func ApplyEnvOverrides[T any](c *T, delim string) error {
+func ApplyEnvOverrides[T any](prefix string, c *T, delim string) error {
 	rval := reflect.ValueOf(c).Elem()
 
-	return processEnvOverrides(rval, delim, "")
+	return processEnvOverrides(rval, delim, prefix)
 }
 
 func processEnvOverrides(rval reflect.Value, delim string, prefix string) error {
@@ -40,12 +41,12 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			continue
 		}
 
-		if rvfield.Kind() == reflect.Struct {
+		envkey, found := rsfield.Tag.Lookup("env")
+		if !found || envkey == "-" {
+			continue
+		}
 
-			envkey, found := rsfield.Tag.Lookup("env")
-			if !found || envkey == "-" {
-				continue
-			}
+		if rvfield.Kind() == reflect.Struct && rvfield.Type() != reflect.TypeOf(time.UnixMilli(0)) {
 
 			subPrefix := prefix
 			if envkey != "" {
@@ -56,10 +57,7 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			if err != nil {
 				return err
 			}
-		}
 
-		envkey := rsfield.Tag.Get("env")
-		if envkey == "" || envkey == "-" {
 			continue
 		}
 
@@ -70,103 +68,128 @@ func processEnvOverrides(rval reflect.Value, delim string, prefix string) error
 			continue
 		}
 
-		if rvfield.Type() == reflect.TypeOf("") {
+		if rvfield.Type().Kind() == reflect.Pointer {
 
-			rvfield.Set(reflect.ValueOf(envval))
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type().Elem())
+			if err != nil {
+				return err
+			}
+
+			// converts reflect.Value to pointer
+			ptrval := reflect.New(rvfield.Type().Elem())
+			ptrval.Elem().Set(newval)
+
+			rvfield.Set(ptrval)
 
 			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
 
-		} else if rvfield.Type() == reflect.TypeOf(int(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", fullEnvKey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int64(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 64)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", fullEnvKey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int64(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int32(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 32)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int32(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int8(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 8)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int8(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(time.Duration(0)) {
-
-			dur, err := timeext.ParseDurationShortString(envval)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", fullEnvKey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(dur))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, dur.String())
-
-		} else if rvfield.Type() == reflect.TypeOf(time.UnixMilli(0)) {
-
-			tim, err := time.Parse(time.RFC3339Nano, envval)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", fullEnvKey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(tim))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, tim.String())
-
-		} else if rvfield.Type().ConvertibleTo(reflect.TypeOf(int(0))) {
-
-			envint, err := strconv.ParseInt(envval, 10, 8)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvfield.Type().Name(), fullEnvKey, envval))
-			}
-
-			envcvl := reflect.ValueOf(envint).Convert(rvfield.Type())
-
-			rvfield.Set(envcvl)
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%v'\n", fullEnvKey, envcvl.Interface())
-
-		} else if rvfield.Type().ConvertibleTo(reflect.TypeOf("")) {
-
-			envcvl := reflect.ValueOf(envval).Convert(rvfield.Type())
-
-			rvfield.Set(envcvl)
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%v'\n", fullEnvKey, envcvl.Interface())
-
 		} else {
-			return errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvfield.Kind().String(), rvfield.Type().String()))
+
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type())
+			if err != nil {
+				return err
+			}
+
+			rvfield.Set(newval)
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
+
 		}
+
 	}
 
 	return nil
 }
+
+func parseEnvToValue(envval string, fullEnvKey string, rvtype reflect.Type) (reflect.Value, error) {
+	if rvtype == reflect.TypeOf("") {
+
+		return reflect.ValueOf(envval), nil
+
+	} else if rvtype == reflect.TypeOf(int(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int64(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 64)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int64(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int32(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 32)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int32(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int8(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int8(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(time.Duration(0)) {
+
+		dur, err := timeext.ParseDurationShortString(envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(dur), nil
+
+	} else if rvtype == reflect.TypeOf(time.UnixMilli(0)) {
+
+		tim, err := time.Parse(time.RFC3339Nano, envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(tim), nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf(int(0))) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+		envcvl := reflect.ValueOf(envint).Convert(rvtype)
+
+		return envcvl, nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf(false)) {
+
+		if strings.TrimSpace(strings.ToLower(envval)) == "true" {
+			return reflect.ValueOf(true).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "false" {
+			return reflect.ValueOf(false).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "1" {
+			return reflect.ValueOf(true).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "0" {
+			return reflect.ValueOf(false).Convert(rvtype), nil
+		} else {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,bool> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf("")) {
+
+		envcvl := reflect.ValueOf(envval).Convert(rvtype)
+		return envcvl, nil
+
+	} else {
+		return reflect.Value{}, errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvtype.Kind().String(), rvtype.String()))
+	}
+}

confext/confParser_test.go

@@ -2,6 +2,7 @@ package confext
 
 import (
 	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"testing"
 	"time"
 )
@@ -41,13 +42,13 @@ func TestApplyEnvOverridesNoop(t *testing.T) {
 
 	output := input
 
-	err := ApplyEnvOverrides(&output, ".")
+	err := ApplyEnvOverrides("", &output, ".")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
 	}
 
-	assertEqual(t, input, output)
+	tst.AssertEqual(t, input, output)
 }
 
 func TestApplyEnvOverridesSimple(t *testing.T) {
@@ -67,6 +68,7 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 		V7 aliasstring   `env:"TEST_V7"`
 		V8 time.Duration `env:"TEST_V8"`
 		V9 time.Time     `env:"TEST_V9"`
+		VA bool          `env:"TEST_VA"`
 	}
 
 	data := testdata{
@@ -81,6 +83,7 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 		V7: "7",
 		V8: 9,
 		V9: time.Unix(1671102873, 0),
+		VA: false,
 	}
 
 	t.Setenv("TEST_V1", "846")
@@ -92,22 +95,24 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 	t.Setenv("TEST_V7", "AAAAAA")
 	t.Setenv("TEST_V8", "1min4s")
 	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+	t.Setenv("TEST_VA", "true")
 
-	err := ApplyEnvOverrides(&data, ".")
+	err := ApplyEnvOverrides("", &data, ".")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
 	}
 
-	assertEqual(t, data.V1, 846)
-	assertEqual(t, data.V2, "hello_world")
-	assertEqual(t, data.V3, 6)
-	assertEqual(t, data.V4, 333)
-	assertEqual(t, data.V5, -937)
-	assertEqual(t, data.V6, 70)
-	assertEqual(t, data.V7, "AAAAAA")
-	assertEqual(t, data.V8, time.Second*64)
-	assertEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.V1, 846)
+	tst.AssertEqual(t, data.V2, "hello_world")
+	tst.AssertEqual(t, data.V3, 6)
+	tst.AssertEqual(t, data.V4, 333)
+	tst.AssertEqual(t, data.V5, -937)
+	tst.AssertEqual(t, data.V6, 70)
+	tst.AssertEqual(t, data.V7, "AAAAAA")
+	tst.AssertEqual(t, data.V8, time.Second*64)
+	tst.AssertEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.VA, true)
 }
 
 func TestApplyEnvOverridesRecursive(t *testing.T) {
@@ -182,35 +187,83 @@ func TestApplyEnvOverridesRecursive(t *testing.T) {
 	t.Setenv("SUB_V3", "33min")
 	t.Setenv("SUB_V4", "2044-01-01T00:00:00Z")
 
-	err := ApplyEnvOverrides(&data, "_")
+	err := ApplyEnvOverrides("", &data, "_")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
 	}
 
-	assertEqual(t, data.V1, 999)
-	assertEqual(t, data.VX, "2")
-	assertEqual(t, data.V5, "no")
-	assertEqual(t, data.Sub1.V1, 3)
-	assertEqual(t, data.Sub1.VX, "4")
-	assertEqual(t, data.Sub1.V2, "5")
-	assertEqual(t, data.Sub1.V8, time.Second*6)
-	assertEqual(t, data.Sub1.V9, time.Unix(947206861, 0).UTC())
-	assertEqual(t, data.Sub2.V1, 846)
-	assertEqual(t, data.Sub2.VX, "9")
-	assertEqual(t, data.Sub2.V2, "222_hello_world")
-	assertEqual(t, data.Sub2.V8, time.Second*64)
-	assertEqual(t, data.Sub2.V9, time.Unix(1257894000, 0).UTC())
-	assertEqual(t, data.Sub3.V1, 33846)
-	assertEqual(t, data.Sub3.VX, "14")
-	assertEqual(t, data.Sub3.V2, "33_hello_world")
-	assertEqual(t, data.Sub3.V8, time.Second*1984)
-	assertEqual(t, data.Sub3.V9, time.Unix(2015276400, 0).UTC())
-	assertEqual(t, data.Sub4.V1, 11)
-	assertEqual(t, data.Sub4.VX, "19")
-	assertEqual(t, data.Sub4.V2, "22")
-	assertEqual(t, data.Sub4.V8, time.Second*1980)
-	assertEqual(t, data.Sub4.V9, time.Unix(2335219200, 0).UTC())
+	tst.AssertEqual(t, data.V1, 999)
+	tst.AssertEqual(t, data.VX, "2")
+	tst.AssertEqual(t, data.V5, "no")
+	tst.AssertEqual(t, data.Sub1.V1, 3)
+	tst.AssertEqual(t, data.Sub1.VX, "4")
+	tst.AssertEqual(t, data.Sub1.V2, "5")
+	tst.AssertEqual(t, data.Sub1.V8, time.Second*6)
+	tst.AssertEqual(t, data.Sub1.V9, time.Unix(947206861, 0).UTC())
+	tst.AssertEqual(t, data.Sub2.V1, 846)
+	tst.AssertEqual(t, data.Sub2.VX, "9")
+	tst.AssertEqual(t, data.Sub2.V2, "222_hello_world")
+	tst.AssertEqual(t, data.Sub2.V8, time.Second*64)
+	tst.AssertEqual(t, data.Sub2.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.Sub3.V1, 33846)
+	tst.AssertEqual(t, data.Sub3.VX, "14")
+	tst.AssertEqual(t, data.Sub3.V2, "33_hello_world")
+	tst.AssertEqual(t, data.Sub3.V8, time.Second*1984)
+	tst.AssertEqual(t, data.Sub3.V9, time.Unix(2015276400, 0).UTC())
+	tst.AssertEqual(t, data.Sub4.V1, 11)
+	tst.AssertEqual(t, data.Sub4.VX, "19")
+	tst.AssertEqual(t, data.Sub4.V2, "22")
+	tst.AssertEqual(t, data.Sub4.V8, time.Second*1980)
+	tst.AssertEqual(t, data.Sub4.V9, time.Unix(2335219200, 0).UTC())
+}
+
+func TestApplyEnvOverridesPointer(t *testing.T) {
+
+	type aliasint int
+	type aliasstring string
+
+	type testdata struct {
+		V1 *int           `env:"TEST_V1"`
+		VX *string        ``
+		V2 *string        `env:"TEST_V2"`
+		V3 *int8          `env:"TEST_V3"`
+		V4 *int32         `env:"TEST_V4"`
+		V5 *int64         `env:"TEST_V5"`
+		V6 *aliasint      `env:"TEST_V6"`
+		VY *aliasint      ``
+		V7 *aliasstring   `env:"TEST_V7"`
+		V8 *time.Duration `env:"TEST_V8"`
+		V9 *time.Time     `env:"TEST_V9"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("TEST_V1", "846")
+	t.Setenv("TEST_V2", "hello_world")
+	t.Setenv("TEST_V3", "6")
+	t.Setenv("TEST_V4", "333")
+	t.Setenv("TEST_V5", "-937")
+	t.Setenv("TEST_V6", "070")
+	t.Setenv("TEST_V7", "AAAAAA")
+	t.Setenv("TEST_V8", "1min4s")
+	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertDeRefEqual(t, data.V1, 846)
+	tst.AssertDeRefEqual(t, data.V2, "hello_world")
+	tst.AssertDeRefEqual(t, data.V3, 6)
+	tst.AssertDeRefEqual(t, data.V4, 333)
+	tst.AssertDeRefEqual(t, data.V5, -937)
+	tst.AssertDeRefEqual(t, data.V6, 70)
+	tst.AssertDeRefEqual(t, data.V7, "AAAAAA")
+	tst.AssertDeRefEqual(t, data.V8, time.Second*64)
+	tst.AssertDeRefEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
 }
 
 func assertEqual[T comparable](t *testing.T, actual T, expected T) {
@@ -218,3 +271,12 @@ func assertEqual[T comparable](t *testing.T, actual T, expected T) {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
+
+func assertPtrEqual[T comparable](t *testing.T, actual *T, expected T) {
+	if actual == nil {
+		t.Errorf("values differ: Actual: NIL, Expected: '%v'", expected)
+	}
+	if *actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

cryptext/aes.go

@@ -1,10 +1,13 @@
 package cryptext
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
-	"encoding/base64"
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
 	"errors"
 	"golang.org/x/crypto/scrypt"
 	"io"
@@ -12,35 +15,90 @@ import (
 
 // https://stackoverflow.com/a/18819040/1761622
 
-func EncryptAESSimple(password, text []byte) ([]byte, error) {
-
-	key, err := scrypt.Key(password, nil, 32768, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
-	if err != nil {
-		return nil, err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return nil, err
-	}
-
-	b := base64.StdEncoding.EncodeToString(text)
-	ciphertext := make([]byte, aes.BlockSize+len(b))
-
-	iv := ciphertext[:aes.BlockSize]
-	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
-		return nil, err
-	}
-
-	cfb := cipher.NewCFBEncrypter(block, iv)
-	cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
-
-	return ciphertext, nil
+type aesPayload struct {
+	Salt    []byte `json:"s"`
+	IV      []byte `json:"i"`
+	Data    []byte `json:"d"`
+	Rounds  int    `json:"r"`
+	Version uint   `json:"v"`
 }
 
-func DecryptAESSimple(password, text []byte) ([]byte, error) {
+func EncryptAESSimple(password []byte, data []byte, rounds int) (string, error) {
 
-	key, err := scrypt.Key(password, nil, 32768, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
+	salt := make([]byte, 8)
+	_, err := io.ReadFull(rand.Reader, salt)
+	if err != nil {
+		return "", err
+	}
+
+	key, err := scrypt.Key(password, salt, rounds, 8, 1, 32)
+	if err != nil {
+		return "", err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	h := sha256.New()
+	h.Write(data)
+	checksum := h.Sum(nil)
+	if len(checksum) != 32 {
+		return "", errors.New("wrong cs size")
+	}
+
+	ciphertext := make([]byte, 32+len(data))
+
+	iv := make([]byte, aes.BlockSize)
+	_, err = io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return "", err
+	}
+
+	combinedData := make([]byte, 0, 32+len(data))
+	combinedData = append(combinedData, checksum...)
+	combinedData = append(combinedData, data...)
+
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext, combinedData)
+
+	pl := aesPayload{
+		Salt:    salt,
+		IV:      iv,
+		Data:    ciphertext,
+		Version: 1,
+		Rounds:  rounds,
+	}
+
+	jbin, err := json.Marshal(pl)
+	if err != nil {
+		return "", err
+	}
+
+	res := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(jbin)
+
+	return res, nil
+}
+
+func DecryptAESSimple(password []byte, encText string) ([]byte, error) {
+
+	jbin, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(encText)
+	if err != nil {
+		return nil, err
+	}
+
+	var pl aesPayload
+	err = json.Unmarshal(jbin, &pl)
+	if err != nil {
+		return nil, err
+	}
+
+	if pl.Version != 1 {
+		return nil, errors.New("unsupported version")
+	}
+
+	key, err := scrypt.Key(password, pl.Salt, pl.Rounds, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
 	if err != nil {
 		return nil, err
 	}
@@ -50,18 +108,24 @@ func DecryptAESSimple(password, text []byte) ([]byte, error) {
 		return nil, err
 	}
 
-	if len(text) < aes.BlockSize {
-		return nil, errors.New("ciphertext too short")
+	dest := make([]byte, len(pl.Data))
+
+	cfb := cipher.NewCFBDecrypter(block, pl.IV)
+	cfb.XORKeyStream(dest, pl.Data)
+
+	if len(dest) < 32 {
+		return nil, errors.New("payload too small")
 	}
 
-	iv := text[:aes.BlockSize]
-	text = text[aes.BlockSize:]
-	cfb := cipher.NewCFBDecrypter(block, iv)
-	cfb.XORKeyStream(text, text)
+	chck := dest[:32]
+	data := dest[32:]
 
-	data, err := base64.StdEncoding.DecodeString(string(text))
-	if err != nil {
-		return nil, err
+	h := sha256.New()
+	h.Write(data)
+	chck2 := h.Sum(nil)
+
+	if !bytes.Equal(chck, chck2) {
+		return nil, errors.New("checksum mismatch")
 	}
 
 	return data, nil

cryptext/aes_test.go

@@ -1,6 +1,10 @@
 package cryptext
 
-import "testing"
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
 
 func TestEncryptAESSimple(t *testing.T) {
 
@@ -8,15 +12,25 @@ func TestEncryptAESSimple(t *testing.T) {
 
 	str1 := []byte("Hello World")
 
-	str2, err := EncryptAESSimple(pw, str1)
+	str2, err := EncryptAESSimple(pw, str1, 512)
 	if err != nil {
 		panic(err)
 	}
 
+	fmt.Printf("%s\n", str2)
+
 	str3, err := DecryptAESSimple(pw, str2)
 	if err != nil {
 		panic(err)
 	}
 
-	assertEqual(t, string(str1), string(str3))
+	tst.AssertEqual(t, string(str1), string(str3))
+
+	str4, err := EncryptAESSimple(pw, str3, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	tst.AssertNotEqual(t, string(str2), string(str4))
+
 }

cryptext/hash_test.go

@@ -1,25 +1,20 @@
 package cryptext
 
 import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
 func TestStrSha256(t *testing.T) {
-	assertEqual(t, StrSha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	assertEqual(t, StrSha256("0"), "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9")
-	assertEqual(t, StrSha256("80085"), "b3786e141d65638ad8a98173e26b5f6a53c927737b23ff31fb1843937250f44b")
-	assertEqual(t, StrSha256("Hello World"), "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e")
+	tst.AssertEqual(t, StrSha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	tst.AssertEqual(t, StrSha256("0"), "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9")
+	tst.AssertEqual(t, StrSha256("80085"), "b3786e141d65638ad8a98173e26b5f6a53c927737b23ff31fb1843937250f44b")
+	tst.AssertEqual(t, StrSha256("Hello World"), "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e")
 }
 
 func TestBytesSha256(t *testing.T) {
-	assertEqual(t, BytesSha256([]byte{}), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	assertEqual(t, BytesSha256([]byte{0}), "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d")
-	assertEqual(t, BytesSha256([]byte{128}), "76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71")
-	assertEqual(t, BytesSha256([]byte{0, 1, 2, 4, 8, 16, 32, 64, 128, 255}), "55016a318ba538e00123c736b2a8b6db368d00e7e25727547655b653e5853603")
-}
-
-func assertEqual(t *testing.T, actual string, expected string) {
-	if actual != expected {
-		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
-	}
+	tst.AssertEqual(t, BytesSha256([]byte{}), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	tst.AssertEqual(t, BytesSha256([]byte{0}), "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d")
+	tst.AssertEqual(t, BytesSha256([]byte{128}), "76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71")
+	tst.AssertEqual(t, BytesSha256([]byte{0, 1, 2, 4, 8, 16, 32, 64, 128, 255}), "55016a318ba538e00123c736b2a8b6db368d00e7e25727547655b653e5853603")
 }

cryptext/passHash.go

@@ -3,6 +3,7 @@ package cryptext
 import (
 	"crypto/rand"
 	"crypto/sha256"
+	"crypto/sha512"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
@@ -14,14 +15,15 @@ import (
 	"strings"
 )
 
-const LatestPassHashVersion = 4
+const LatestPassHashVersion = 5
 
 // PassHash
-// - [v0]: plaintext password ( `0|...` )
-// - [v1]: sha256(plaintext)
-// - [v2]: seed | sha256<seed>(plaintext)
-// - [v3]: seed | sha256<seed>(plaintext) | [hex(totp)]
-// - [v4]: bcrypt(plaintext) | [hex(totp)]
+// - [v0]: plaintext password ( `0|...` )                  // simple, used to write PW's directly in DB
+// - [v1]: sha256(plaintext)                               // simple hashing
+// - [v2]: seed | sha256<seed>(plaintext)                  // add seed
+// - [v3]: seed | sha256<seed>(plaintext) | [hex(totp)]    // add TOTP support
+// - [v4]: bcrypt(plaintext) | [hex(totp)]                 // use proper bcrypt
+// - [v5]: bcrypt(sha512(plaintext)) | [hex(totp)]         // hash pw before bcrypt (otherwise max pw-len = 72)
 type PassHash string
 
 func (ph PassHash) Valid() bool {
@@ -64,7 +66,6 @@ func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bo
 		return int(version), nil, payload, false, nil, true
 	}
 
-	//
 	if version == 2 {
 		if len(split) != 3 {
 			return -1, nil, nil, false, nil, false
@@ -109,7 +110,21 @@ func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bo
 		totp := false
 		totpsecret := make([]byte, 0)
 		if split[2] != "0" {
-			totpsecret, err = hex.DecodeString(split[3])
+			totpsecret, err = hex.DecodeString(split[2])
+			totp = true
+		}
+		return int(version), nil, payload, totp, totpsecret, true
+	}
+
+	if version == 5 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload := []byte(split[1])
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[2] != "0" {
+			totpsecret, err = hex.DecodeString(split[2])
 			totp = true
 		}
 		return int(version), nil, payload, totp, totpsecret, true
@@ -156,6 +171,14 @@ func (ph PassHash) Verify(plainpass string, totp *string) bool {
 		}
 	}
 
+	if version == 5 {
+		if !hastotp {
+			return bcrypt.CompareHashAndPassword(payload, hash512(plainpass)) == nil
+		} else {
+			return bcrypt.CompareHashAndPassword(payload, hash512(plainpass)) == nil && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
 	return false
 }
 
@@ -209,6 +232,12 @@ func (ph PassHash) ClearTOTP() (PassHash, error) {
 		return PassHash(strings.Join(split, "|")), nil
 	}
 
+	if version == 5 {
+		split := strings.Split(string(ph), "|")
+		split[2] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
 	return "", errors.New("unknown version")
 }
 
@@ -242,6 +271,12 @@ func (ph PassHash) WithTOTP(totpSecret []byte) (PassHash, error) {
 		return PassHash(strings.Join(split, "|")), nil
 	}
 
+	if version == 5 {
+		split := strings.Split(string(ph), "|")
+		split[2] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
 	return "", errors.New("unknown version")
 }
 
@@ -271,6 +306,10 @@ func (ph PassHash) Change(newPlainPass string) (PassHash, error) {
 		return HashPasswordV4(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
 	}
 
+	if version == 5 {
+		return HashPasswordV5(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
 	return "", errors.New("unknown version")
 }
 
@@ -279,7 +318,24 @@ func (ph PassHash) String() string {
 }
 
 func HashPassword(plainpass string, totpSecret []byte) (PassHash, error) {
-	return HashPasswordV4(plainpass, totpSecret)
+	return HashPasswordV5(plainpass, totpSecret)
+}
+
+func HashPasswordV5(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	payload, err := bcrypt.GenerateFromPassword(hash512(plainpass), bcrypt.MinCost)
+	if err != nil {
+		return "", err
+	}
+
+	return PassHash(fmt.Sprintf("5|%s|%s", string(payload), strtotp)), nil
 }
 
 func HashPasswordV4(plainpass string, totpSecret []byte) (PassHash, error) {
@@ -340,6 +396,13 @@ func HashPasswordV0(plainpass string) (PassHash, error) {
 	return PassHash(fmt.Sprintf("0|%s", plainpass)), nil
 }
 
+func hash512(s string) []byte {
+	h := sha512.New()
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
 func hash256(s string) []byte {
 	h := sha256.New()
 	h.Write([]byte(s))

cryptext/passHash_test.go

@@ -0,0 +1,210 @@
+package cryptext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/totpext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestPassHash1(t *testing.T) {
+	ph, err := HashPassword("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashTOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPassword("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashUpgrade_V0(t *testing.T) {
+	ph, err := HashPasswordV0("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V1(t *testing.T) {
+	ph, err := HashPasswordV1("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V2(t *testing.T) {
+	ph, err := HashPasswordV2("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V3(t *testing.T) {
+	ph, err := HashPasswordV3("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V3_TOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPasswordV3("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashUpgrade_V4(t *testing.T) {
+	ph, err := HashPasswordV4("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V4_TOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPasswordV4("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}

cryptext/pronouncablePassword.go

@@ -0,0 +1,263 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"io"
+	"math/big"
+	mathrand "math/rand"
+	"strings"
+)
+
+const (
+	ppStartChar            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppEndChar              = "ABDEFIKMNORSTUXYZ"
+	ppVowel                = "AEIOUY"
+	ppConsonant            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppSegmentLenMin        = 3
+	ppSegmentLenMax        = 7
+	ppMaxRepeatedVowel     = 2
+	ppMaxRepeatedConsonant = 2
+)
+
+var ppContinuation = map[uint8]string{
+	'A': "BCDFGHJKLMNPRSTVWXYZ",
+	'B': "ADFIKLMNORSTUY",
+	'C': "AEIKOUY",
+	'D': "AEILORSUYZ",
+	'E': "BCDFGHJKLMNPRSTVWXYZ",
+	'F': "ADEGIKLOPRTUY",
+	'G': "ABDEFHILMNORSTUY",
+	'H': "AEIOUY",
+	'I': "BCDFGHJKLMNPRSTVWXZ",
+	'J': "AEIOUY",
+	'K': "ADEFHILMNORSTUY",
+	'L': "ADEFGIJKMNOPSTUVWYZ",
+	'M': "ABEFIKOPSTUY",
+	'N': "ABEFIKOPSTUY",
+	'O': "BCDFGHJKLMNPRSTVWXYZ",
+	'P': "AEFIJLORSTUY",
+	'Q': "AEIOUY",
+	'R': "ADEFGHIJKLMNOPSTUVYZ",
+	'S': "ACDEIKLOPTUYZ",
+	'T': "AEHIJOPRSUWY",
+	'U': "BCDFGHJKLMNPRSTVWXZ",
+	'V': "AEIOUY",
+	'W': "AEIOUY",
+	'X': "AEIOUY",
+	'Y': "ABCDFGHKLMNPRSTVXZ",
+	'Z': "AEILOTUY",
+}
+
+var ppLog2Map = map[int]float64{
+	1:  0.00000000,
+	2:  1.00000000,
+	3:  1.58496250,
+	4:  2.00000000,
+	5:  2.32192809,
+	6:  2.58496250,
+	7:  2.80735492,
+	8:  3.00000000,
+	9:  3.16992500,
+	10: 3.32192809,
+	11: 3.45943162,
+	12: 3.58496250,
+	13: 3.70043972,
+	14: 3.80735492,
+	15: 3.90689060,
+	16: 4.00000000,
+	17: 4.08746284,
+	18: 4.16992500,
+	19: 4.24792751,
+	20: 4.32192809,
+	21: 4.39231742,
+	22: 4.45943162,
+	23: 4.52356196,
+	24: 4.58496250,
+	25: 4.64385619,
+	26: 4.70043972,
+	27: 4.75488750,
+	28: 4.80735492,
+	29: 4.85798100,
+	30: 4.90689060,
+	31: 4.95419631,
+	32: 5.00000000,
+}
+
+var (
+	ppVowelMap     = ppMakeSet(ppVowel)
+	ppConsonantMap = ppMakeSet(ppConsonant)
+	ppEndCharMap   = ppMakeSet(ppEndChar)
+)
+
+func ppMakeSet(v string) map[uint8]bool {
+	mp := make(map[uint8]bool, len(v))
+	for _, chr := range v {
+		mp[uint8(chr)] = true
+	}
+	return mp
+}
+
+func ppRandInt(rng io.Reader, max int) int {
+	v, err := rand.Int(rng, big.NewInt(int64(max)))
+	if err != nil {
+		panic(err)
+	}
+	return int(v.Int64())
+}
+
+func ppRand(rng io.Reader, chars string, entropy *float64) uint8 {
+	chr := chars[ppRandInt(rng, len(chars))]
+
+	*entropy = *entropy + ppLog2Map[len(chars)]
+
+	return chr
+}
+
+func ppCharType(chr uint8) (bool, bool) {
+	_, ok1 := ppVowelMap[chr]
+	_, ok2 := ppConsonantMap[chr]
+
+	return ok1, ok2
+}
+
+func ppCharsetRemove(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; !ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func ppCharsetFilter(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func PronouncablePasswordExt(rng io.Reader, pwlen int) (string, float64) {
+
+	// kinda pseudo markov-chain - with a few extra rules and no weights...
+
+	if pwlen <= 0 {
+		return "", 0
+	}
+
+	vowelCount := 0
+	consoCount := 0
+	entropy := float64(0)
+
+	startChar := ppRand(rng, ppStartChar, &entropy)
+
+	result := string(startChar)
+	currentChar := startChar
+
+	isVowel, isConsonant := ppCharType(currentChar)
+	if isVowel {
+		vowelCount = 1
+	}
+	if isConsonant {
+		consoCount = ppMaxRepeatedConsonant
+	}
+
+	segmentLen := 1
+
+	segmentLenTarget := ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+
+	for len(result) < pwlen {
+
+		charset := ppContinuation[currentChar]
+		if vowelCount >= ppMaxRepeatedVowel {
+			charset = ppCharsetRemove(charset, ppVowelMap, false)
+		}
+		if consoCount >= ppMaxRepeatedConsonant {
+			charset = ppCharsetRemove(charset, ppConsonantMap, false)
+		}
+
+		lastOfSegment := false
+		newSegment := false
+
+		if len(result)+1 == pwlen {
+			// last of result
+			charset = ppCharsetFilter(charset, ppEndCharMap, false)
+		} else if segmentLen+1 == segmentLenTarget {
+			// last of segment
+			charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+			if charsetNew != "" {
+				charset = charsetNew
+				lastOfSegment = true
+			}
+		} else if segmentLen >= segmentLenTarget {
+			// (perhaps) start of new segment
+			if _, ok := ppEndCharMap[currentChar]; ok {
+				charset = ppStartChar
+				newSegment = true
+			} else {
+				// continue segment for one more char to (hopefully) find an end-char
+				charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+				if charsetNew != "" {
+					charset = charsetNew
+					lastOfSegment = true
+				}
+			}
+		} else {
+			// normal continuation
+		}
+
+		newChar := ppRand(rng, charset, &entropy)
+		if lastOfSegment {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		} else if newSegment {
+			currentChar = newChar
+			segmentLen = 1
+			result += strings.ToUpper(string(newChar))
+			segmentLenTarget = ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+			vowelCount = 0
+			consoCount = 0
+		} else {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		}
+
+		isVowel, isConsonant := ppCharType(currentChar)
+		if isVowel {
+			vowelCount++
+			consoCount = 0
+		}
+		if isConsonant {
+			vowelCount = 0
+			if newSegment {
+				consoCount = ppMaxRepeatedConsonant
+			} else {
+				consoCount++
+			}
+		}
+	}
+
+	return result, entropy
+}
+
+func PronouncablePassword(len int) string {
+	v, _ := PronouncablePasswordExt(rand.Reader, len)
+	return v
+}
+
+func PronouncablePasswordSeeded(seed int64, len int) string {
+
+	v, _ := PronouncablePasswordExt(mathrand.New(mathrand.NewSource(seed)), len)
+	return v
+}

cryptext/pronouncablePassword_test.go

@@ -0,0 +1,35 @@
+package cryptext
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestPronouncablePasswordExt(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw, entropy := PronouncablePasswordExt(rand.New(rand.NewSource(int64(i))), 16)
+		fmt.Printf("[%.2f] => %s\n", entropy, pw)
+	}
+}
+
+func TestPronouncablePasswordSeeded(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw := PronouncablePasswordSeeded(int64(i), 8)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePassword(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw := PronouncablePassword(i + 1)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePasswordWrongLen(t *testing.T) {
+	PronouncablePassword(0)
+	PronouncablePassword(-1)
+	PronouncablePassword(-2)
+	PronouncablePassword(-3)
+}

cursortoken/direction.go

@@ -0,0 +1,8 @@
+package cursortoken
+
+type SortDirection string //@enum:type
+
+const (
+	SortASC  SortDirection = "ASC"
+	SortDESC SortDirection = "DESC"
+)

cursortoken/filter.go

@@ -0,0 +1,14 @@
+package cursortoken
+
+import (
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type RawFilter interface {
+	FilterQuery() mongo.Pipeline
+}
+
+type Filter interface {
+	FilterQuery() mongo.Pipeline
+	Pagination() (string, SortDirection, string, SortDirection)
+}

cursortoken/token.go

@@ -0,0 +1,184 @@
+package cursortoken
+
+import (
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"strings"
+	"time"
+)
+
+type Mode string
+
+const (
+	CTMStart  Mode = "START"
+	CTMNormal Mode = "NORMAL"
+	CTMEnd    Mode = "END"
+)
+
+type Extra struct {
+	Timestamp *time.Time
+	Id        *string
+	Page      *int
+	PageSize  *int
+}
+
+type CursorToken struct {
+	Mode               Mode
+	ValuePrimary       string
+	ValueSecondary     string
+	Direction          SortDirection
+	DirectionSecondary SortDirection
+	PageSize           int
+	Extra              Extra
+}
+
+type cursorTokenSerialize struct {
+	ValuePrimary       *string        `json:"v1,omitempty"`
+	ValueSecondary     *string        `json:"v2,omitempty"`
+	Direction          *SortDirection `json:"dir,omitempty"`
+	DirectionSecondary *SortDirection `json:"dir2,omitempty"`
+	PageSize           *int           `json:"size,omitempty"`
+
+	ExtraTimestamp *time.Time `json:"ts,omitempty"`
+	ExtraId        *string    `json:"id,omitempty"`
+	ExtraPage      *int       `json:"pg,omitempty"`
+	ExtraPageSize  *int       `json:"sz,omitempty"`
+}
+
+func Start() CursorToken {
+	return CursorToken{
+		Mode:               CTMStart,
+		ValuePrimary:       "",
+		ValueSecondary:     "",
+		Direction:          "",
+		DirectionSecondary: "",
+		PageSize:           0,
+		Extra:              Extra{},
+	}
+}
+
+func End() CursorToken {
+	return CursorToken{
+		Mode:               CTMEnd,
+		ValuePrimary:       "",
+		ValueSecondary:     "",
+		Direction:          "",
+		DirectionSecondary: "",
+		PageSize:           0,
+		Extra:              Extra{},
+	}
+}
+
+func (c *CursorToken) Token() string {
+	if c.Mode == CTMStart {
+		return "@start"
+	}
+	if c.Mode == CTMEnd {
+		return "@end"
+	}
+
+	// We kinda manually implement omitempty for the CursorToken here
+	// because omitempty does not work for time.Time and otherwise we would always
+	// get weird time values when decoding a token that initially didn't have an Timestamp set
+	// For this usecase we treat Unix=0 as an empty timestamp
+
+	sertok := cursorTokenSerialize{}
+
+	if c.ValuePrimary != "" {
+		sertok.ValuePrimary = &c.ValuePrimary
+	}
+	if c.ValueSecondary != "" {
+		sertok.ValueSecondary = &c.ValueSecondary
+	}
+	if c.Direction != "" {
+		sertok.Direction = &c.Direction
+	}
+	if c.DirectionSecondary != "" {
+		sertok.DirectionSecondary = &c.DirectionSecondary
+	}
+	if c.PageSize != 0 {
+		sertok.PageSize = &c.PageSize
+	}
+
+	sertok.ExtraTimestamp = c.Extra.Timestamp
+	sertok.ExtraId = c.Extra.Id
+	sertok.ExtraPage = c.Extra.Page
+	sertok.ExtraPageSize = c.Extra.PageSize
+
+	body, err := json.Marshal(sertok)
+	if err != nil {
+		panic(err)
+	}
+
+	return "tok_" + base32.StdEncoding.EncodeToString(body)
+}
+
+func Decode(tok string) (CursorToken, error) {
+	if tok == "" {
+		return Start(), nil
+	}
+	if strings.ToLower(tok) == "@start" {
+		return Start(), nil
+	}
+	if strings.ToLower(tok) == "@end" {
+		return End(), nil
+	}
+
+	if !strings.HasPrefix(tok, "tok_") {
+		return CursorToken{}, errors.New("could not decode token, missing prefix")
+	}
+
+	body, err := base32.StdEncoding.DecodeString(tok[len("tok_"):])
+	if err != nil {
+		return CursorToken{}, err
+	}
+
+	var tokenDeserialize cursorTokenSerialize
+	err = json.Unmarshal(body, &tokenDeserialize)
+	if err != nil {
+		return CursorToken{}, err
+	}
+
+	token := CursorToken{Mode: CTMNormal}
+
+	if tokenDeserialize.ValuePrimary != nil {
+		token.ValuePrimary = *tokenDeserialize.ValuePrimary
+	}
+	if tokenDeserialize.ValueSecondary != nil {
+		token.ValueSecondary = *tokenDeserialize.ValueSecondary
+	}
+	if tokenDeserialize.Direction != nil {
+		token.Direction = *tokenDeserialize.Direction
+	}
+	if tokenDeserialize.DirectionSecondary != nil {
+		token.DirectionSecondary = *tokenDeserialize.DirectionSecondary
+	}
+	if tokenDeserialize.PageSize != nil {
+		token.PageSize = *tokenDeserialize.PageSize
+	}
+
+	token.Extra.Timestamp = tokenDeserialize.ExtraTimestamp
+	token.Extra.Id = tokenDeserialize.ExtraId
+	token.Extra.Page = tokenDeserialize.ExtraPage
+	token.Extra.PageSize = tokenDeserialize.ExtraPageSize
+
+	return token, nil
+}
+
+func (c *CursorToken) ValuePrimaryObjectId() (primitive.ObjectID, bool) {
+	if oid, err := primitive.ObjectIDFromHex(c.ValuePrimary); err == nil {
+		return oid, true
+	} else {
+		return primitive.ObjectID{}, false
+	}
+}
+
+func (c *CursorToken) ValueSecondaryObjectId() (primitive.ObjectID, bool) {
+	if oid, err := primitive.ObjectIDFromHex(c.ValueSecondary); err == nil {
+		return oid, true
+	} else {
+		return primitive.ObjectID{}, false
+	}
+}

dataext/lruMap_test.go

@@ -12,7 +12,7 @@ func init() {
 }
 
 func TestResultCache1(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key := randomKey()
@@ -50,7 +50,7 @@ func TestResultCache1(t *testing.T) {
 }
 
 func TestResultCache2(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -150,7 +150,7 @@ func TestResultCache2(t *testing.T) {
 }
 
 func TestResultCache3(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -173,7 +173,7 @@ func TestResultCache3(t *testing.T) {
 }
 
 // does a basic consistency check over the internal cache representation
-func verifyLRUList[TData any](cache *LRUMap[TData], t *testing.T) {
+func verifyLRUList[TKey comparable, TData any](cache *LRUMap[TKey, TData], t *testing.T) {
 	size := 0
 
 	tailFound := false

dataext/merge_test.go

@@ -2,6 +2,7 @@ package dataext
 
 import (
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
@@ -43,10 +44,10 @@ func TestObjectMerge(t *testing.T) {
 
 	valueMerge := ObjectMerge(valueA, valueB)
 
-	assertPtrEqual(t, "Field1", valueMerge.Field1, valueB.Field1)
-	assertPtrEqual(t, "Field2", valueMerge.Field2, valueA.Field2)
-	assertPtrEqual(t, "Field3", valueMerge.Field3, valueB.Field3)
-	assertPtrEqual(t, "Field4", valueMerge.Field4, nil)
+	tst.AssertIdentPtrEqual(t, "Field1", valueMerge.Field1, valueB.Field1)
+	tst.AssertIdentPtrEqual(t, "Field2", valueMerge.Field2, valueA.Field2)
+	tst.AssertIdentPtrEqual(t, "Field3", valueMerge.Field3, valueB.Field3)
+	tst.AssertIdentPtrEqual(t, "Field4", valueMerge.Field4, nil)
 
 }
 

dataext/structHash_test.go

@@ -1,8 +1,8 @@
 package dataext
 
 import (
-	"encoding/hex"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
@@ -18,14 +18,14 @@ func noErrStructHash(t *testing.T, dat any, opt ...StructHashOptions) []byte {
 
 func TestStructHashSimple(t *testing.T) {
 
-	assertEqual(t, "209bf774af36cc3a045c152d9f1269ef3684ad819c1359ee73ff0283a308fefa", noErrStructHash(t, "Hello"))
-	assertEqual(t, "c32f3626b981ae2997db656f3acad3f1dc9d30ef6b6d14296c023e391b25f71a", noErrStructHash(t, 0))
-	assertEqual(t, "01b781b03e9586b257d387057dfc70d9f06051e7d3c1e709a57e13cc8daf3e35", noErrStructHash(t, []byte{}))
-	assertEqual(t, "93e1dcd45c732fe0079b0fb3204c7c803f0921835f6bfee2e6ff263e73eed53c", noErrStructHash(t, []int{}))
-	assertEqual(t, "54f637a376aad55b3160d98ebbcae8099b70d91b9400df23fb3709855d59800a", noErrStructHash(t, []int{1, 2, 3}))
-	assertEqual(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", noErrStructHash(t, nil))
-	assertEqual(t, "349a7db91aa78fd30bbaa7c7f9c7bfb2fcfe72869b4861162a96713a852f60d3", noErrStructHash(t, []any{1, "", nil}))
-	assertEqual(t, "ca51aab87808bf0062a4a024de6aac0c2bad54275cc857a4944569f89fd245ad", noErrStructHash(t, struct{}{}))
+	tst.AssertHexEqual(t, "209bf774af36cc3a045c152d9f1269ef3684ad819c1359ee73ff0283a308fefa", noErrStructHash(t, "Hello"))
+	tst.AssertHexEqual(t, "c32f3626b981ae2997db656f3acad3f1dc9d30ef6b6d14296c023e391b25f71a", noErrStructHash(t, 0))
+	tst.AssertHexEqual(t, "01b781b03e9586b257d387057dfc70d9f06051e7d3c1e709a57e13cc8daf3e35", noErrStructHash(t, []byte{}))
+	tst.AssertHexEqual(t, "93e1dcd45c732fe0079b0fb3204c7c803f0921835f6bfee2e6ff263e73eed53c", noErrStructHash(t, []int{}))
+	tst.AssertHexEqual(t, "54f637a376aad55b3160d98ebbcae8099b70d91b9400df23fb3709855d59800a", noErrStructHash(t, []int{1, 2, 3}))
+	tst.AssertHexEqual(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", noErrStructHash(t, nil))
+	tst.AssertHexEqual(t, "349a7db91aa78fd30bbaa7c7f9c7bfb2fcfe72869b4861162a96713a852f60d3", noErrStructHash(t, []any{1, "", nil}))
+	tst.AssertHexEqual(t, "ca51aab87808bf0062a4a024de6aac0c2bad54275cc857a4944569f89fd245ad", noErrStructHash(t, struct{}{}))
 
 }
 
@@ -37,13 +37,13 @@ func TestStructHashSimpleStruct(t *testing.T) {
 		F3 *int
 	}
 
-	assertEqual(t, "a90bff751c70c738bb5cfc9b108e783fa9c19c0bc9273458e0aaee6e74aa1b92", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "a90bff751c70c738bb5cfc9b108e783fa9c19c0bc9273458e0aaee6e74aa1b92", noErrStructHash(t, t0{
 		F1: 10,
 		F2: []string{"1", "2", "3"},
 		F3: nil,
 	}))
 
-	assertEqual(t, "5d09090dc34ac59dd645f197a255f653387723de3afa1b614721ea5a081c675f", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "5d09090dc34ac59dd645f197a255f653387723de3afa1b614721ea5a081c675f", noErrStructHash(t, t0{
 		F1: 10,
 		F2: []string{"1", "2", "3"},
 		F3: langext.Ptr(99),
@@ -64,7 +64,7 @@ func TestStructHashLayeredStruct(t *testing.T) {
 		SV3 t1_1
 	}
 
-	assertEqual(t, "fd4ca071fb40a288fee4b7a3dfdaab577b30cb8f80f81ec511e7afd72dc3b469", noErrStructHash(t, t1_2{
+	tst.AssertHexEqual(t, "fd4ca071fb40a288fee4b7a3dfdaab577b30cb8f80f81ec511e7afd72dc3b469", noErrStructHash(t, t1_2{
 		SV1: nil,
 		SV2: nil,
 		SV3: t1_1{
@@ -73,7 +73,7 @@ func TestStructHashLayeredStruct(t *testing.T) {
 			F15: false,
 		},
 	}))
-	assertEqual(t, "3fbf7c67d8121deda075cc86319a4e32d71744feb2cebf89b43bc682f072a029", noErrStructHash(t, t1_2{
+	tst.AssertHexEqual(t, "3fbf7c67d8121deda075cc86319a4e32d71744feb2cebf89b43bc682f072a029", noErrStructHash(t, t1_2{
 		SV1: nil,
 		SV2: &t1_1{},
 		SV3: t1_1{
@@ -82,7 +82,7 @@ func TestStructHashLayeredStruct(t *testing.T) {
 			F15: true,
 		},
 	}))
-	assertEqual(t, "b1791ccd1b346c3ede5bbffda85555adcd8216b93ffca23f14fe175ec47c5104", noErrStructHash(t, t1_2{
+	tst.AssertHexEqual(t, "b1791ccd1b346c3ede5bbffda85555adcd8216b93ffca23f14fe175ec47c5104", noErrStructHash(t, t1_2{
 		SV1: &t1_1{},
 		SV2: &t1_1{},
 		SV3: t1_1{
@@ -101,7 +101,7 @@ func TestStructHashMap(t *testing.T) {
 		F2 map[string]int
 	}
 
-	assertEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
 		F1: 10,
 		F2: map[string]int{
 			"x": 1,
@@ -110,7 +110,7 @@ func TestStructHashMap(t *testing.T) {
 		},
 	}))
 
-	assertEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
 		F1: 10,
 		F2: map[string]int{
 			"a": 99,
@@ -128,16 +128,9 @@ func TestStructHashMap(t *testing.T) {
 	m3["x"] = 1
 	m3["a"] = 2
 
-	assertEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
 		F1: 10,
 		F2: m3,
 	}))
 
 }
-
-func assertEqual(t *testing.T, expected string, actual []byte) {
-	actualStr := hex.EncodeToString(actual)
-	if actualStr != expected {
-		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actualStr, expected)
-	}
-}

dataext/syncMap.go

@@ -0,0 +1,113 @@
+package dataext
+
+import "sync"
+
+type SyncMap[TKey comparable, TData any] struct {
+	data map[TKey]TData
+	lock sync.Mutex
+}
+
+func (s *SyncMap[TKey, TData]) Set(key TKey, data TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	s.data[key] = data
+}
+
+func (s *SyncMap[TKey, TData]) SetIfNotContains(key TKey, data TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if _, existsInPreState := s.data[key]; existsInPreState {
+		return false
+	}
+
+	s.data[key] = data
+
+	return true
+}
+
+func (s *SyncMap[TKey, TData]) Get(key TKey) (TData, bool) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		return v, true
+	} else {
+		return *new(TData), false
+	}
+}
+
+func (s *SyncMap[TKey, TData]) Delete(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	delete(s.data, key)
+
+	return ok
+}
+
+func (s *SyncMap[TKey, TData]) Contains(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	return ok
+}
+
+func (s *SyncMap[TKey, TData]) GetAllKeys() []TKey {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TKey, 0, len(s.data))
+
+	for k := range s.data {
+		r = append(r, k)
+	}
+
+	return r
+}
+
+func (s *SyncMap[TKey, TData]) GetAllValues() []TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TData, 0, len(s.data))
+
+	for _, v := range s.data {
+		r = append(r, v)
+	}
+
+	return r
+}

dataext/syncSet.go

@@ -7,6 +7,9 @@ type SyncSet[TData comparable] struct {
 	lock sync.Mutex
 }
 
+// Add adds `value` to the set
+// returns true  if the value was actually inserted
+// returns false if the value already existed
 func (s *SyncSet[TData]) Add(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -15,10 +18,10 @@ func (s *SyncSet[TData]) Add(value TData) bool {
 		s.data = make(map[TData]bool)
 	}
 
-	_, ok := s.data[value]
+	_, existsInPreState := s.data[value]
 	s.data[value] = true
 
-	return !ok
+	return !existsInPreState
 }
 
 func (s *SyncSet[TData]) AddAll(values []TData) {

dataext/tuple.go

@@ -0,0 +1,170 @@
+package dataext
+
+type ValueGroup interface {
+	TupleLength() int
+	TupleValues() []any
+}
+
+// ----------------------------------------------------------------------------
+
+type Single[T1 any] struct {
+	V1 T1
+}
+
+func (s Single[T1]) TupleLength() int {
+	return 1
+}
+
+func (s Single[T1]) TupleValues() []any {
+	return []any{s.V1}
+}
+
+// ----------------------------------------------------------------------------
+
+type Tuple[T1 any, T2 any] struct {
+	V1 T1
+	V2 T2
+}
+
+func (t Tuple[T1, T2]) TupleLength() int {
+	return 2
+}
+
+func (t Tuple[T1, T2]) TupleValues() []any {
+	return []any{t.V1, t.V2}
+}
+
+// ----------------------------------------------------------------------------
+
+type Triple[T1 any, T2 any, T3 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+}
+
+func (t Triple[T1, T2, T3]) TupleLength() int {
+	return 3
+}
+
+func (t Triple[T1, T2, T3]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3}
+}
+
+// ----------------------------------------------------------------------------
+
+type Quadruple[T1 any, T2 any, T3 any, T4 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+}
+
+func (t Quadruple[T1, T2, T3, T4]) TupleLength() int {
+	return 4
+}
+
+func (t Quadruple[T1, T2, T3, T4]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4}
+}
+
+// ----------------------------------------------------------------------------
+
+type Quintuple[T1 any, T2 any, T3 any, T4 any, T5 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+}
+
+func (t Quintuple[T1, T2, T3, T4, T5]) TupleLength() int {
+	return 5
+}
+
+func (t Quintuple[T1, T2, T3, T4, T5]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5}
+
+}
+
+// ----------------------------------------------------------------------------
+
+type Sextuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+}
+
+func (t Sextuple[T1, T2, T3, T4, T5, T6]) TupleLength() int {
+	return 6
+}
+
+func (t Sextuple[T1, T2, T3, T4, T5, T6]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6}
+
+}
+
+// ----------------------------------------------------------------------------
+
+type Septuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+	V7 T7
+}
+
+func (t Septuple[T1, T2, T3, T4, T5, T6, T7]) TupleLength() int {
+	return 7
+}
+
+func (t Septuple[T1, T2, T3, T4, T5, T6, T7]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6, t.V7}
+}
+
+// ----------------------------------------------------------------------------
+
+type Octuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+	V7 T7
+	V8 T8
+}
+
+func (t Octuple[T1, T2, T3, T4, T5, T6, T7, T8]) TupleLength() int {
+	return 8
+}
+
+func (t Octuple[T1, T2, T3, T4, T5, T6, T7, T8]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6, t.V7, t.V8}
+}
+
+// ----------------------------------------------------------------------------
+
+type Nonuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any, T9 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+	V7 T7
+	V8 T8
+	V9 T9
+}
+
+func (t Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9]) TupleLength() int {
+	return 9
+}
+
+func (t Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6, t.V7, t.V8, t.V9}
+}

enums/enum.go

@@ -0,0 +1,31 @@
+package enums
+
+type Enum interface {
+	Valid() bool
+	ValuesAny() []any
+	ValuesMeta() []EnumMetaValue
+	VarName() string
+}
+
+type StringEnum interface {
+	Enum
+	String() string
+}
+
+type DescriptionEnum interface {
+	Enum
+	Description() string
+	DescriptionMeta() EnumDescriptionMetaValue
+}
+
+type EnumMetaValue struct {
+	VarName     string  `json:"varName"`
+	Value       Enum    `json:"value"`
+	Description *string `json:"description"`
+}
+
+type EnumDescriptionMetaValue struct {
+	VarName     string `json:"varName"`
+	Value       Enum   `json:"value"`
+	Description string `json:"description"`
+}

exerr/builder.go

@@ -0,0 +1,485 @@
+package exerr
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+	"gogs.mikescher.com/BlackForestBytes/goext/enums"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"net/http"
+	"os"
+	"runtime/debug"
+	"strings"
+	"time"
+)
+
+//
+// ==== USAGE =====
+//
+// If some method returns an error _always wrap it into an exerror:
+//     value, err := do_something(..)
+//     if err != nil {
+//         return nil, exerror.Wrap(err, "do something failed").Build()
+//     }
+//
+// If possible add metadata to the error (eg the id that was not found, ...), the methods are the same as in zerolog
+//     return nil, exerror.Wrap(err, "do something failed").Str("someid", id).Int("count", in.Count).Build()
+//
+// You can change the errortype with `.User()` and `.System()` (User-errors are 400 and System-errors 500)
+// You can also manually set the statuscode with `.WithStatuscode(http.NotFound)`
+// You can set the type with `WithType(..)`
+//
+// New Errors (that don't wrap an existing err object) are created with New
+//     return nil, exerror.New(exerror.TypeInternal, "womethign wen horrible wrong").Build()
+// You can eitehr use an existing ErrorType, the "catch-all" ErrInternal, or add you own ErrType in consts.go
+//
+// All errors should be handled one of the following four ways:
+//   - return the error to the caller and let him handle it:
+//     (also auto-prints the error to the log)
+//     => Wrap/New + Build
+//   - Print the error
+//     (also auto-sends it to the error-service)
+//     This is useful for errors that happen asynchron or are non-fatal for the current request
+//     => Wrap/New + Print
+//   - Return the error to the Rest-API caller
+//     (also auto-prints the error to the log)
+//     (also auto-sends it to the error-service)
+//     => Wrap/New + Output
+//   - Print and stop the service
+//     (also auto-sends it to the error-service)
+//     => Wrap/New + Fatal
+//
+
+var stackSkipLogger zerolog.Logger
+
+func init() {
+	cw := zerolog.ConsoleWriter{
+		Out:        os.Stdout,
+		TimeFormat: "2006-01-02 15:04:05 Z07:00",
+	}
+
+	multi := zerolog.MultiLevelWriter(cw)
+	stackSkipLogger = zerolog.New(multi).With().Timestamp().CallerWithSkipFrameCount(4).Logger()
+}
+
+type Builder struct {
+	wrappedErr      error
+	errorData       *ExErr
+	containsGinData bool
+	noLog           bool
+}
+
+func Get(err error) *Builder {
+	return &Builder{errorData: FromError(err)}
+}
+
+func New(t ErrorType, msg string) *Builder {
+	return &Builder{errorData: newExErr(CatSystem, t, msg)}
+}
+
+func Wrap(err error, msg string) *Builder {
+	if err == nil {
+		return &Builder{errorData: newExErr(CatSystem, TypeInternal, msg)} // prevent NPE if we call Wrap with err==nil
+	}
+
+	if !pkgconfig.RecursiveErrors {
+		v := FromError(err)
+		v.Message = msg
+		return &Builder{wrappedErr: err, errorData: v}
+	}
+	return &Builder{wrappedErr: err, errorData: wrapExErr(FromError(err), msg, CatWrap, 1)}
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) WithType(t ErrorType) *Builder {
+	b.errorData.Type = t
+	return b
+}
+
+func (b *Builder) WithStatuscode(status int) *Builder {
+	b.errorData.StatusCode = &status
+	return b
+}
+
+func (b *Builder) WithMessage(msg string) *Builder {
+	b.errorData.Message = msg
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+// Err changes the Severity to ERROR (default)
+// The error will be:
+//
+//   - On Build():
+//
+//   - Short-Logged as Err
+//
+//   - On Print():
+//
+//   - Logged as Err
+//
+//   - Send to the error-service
+//
+//   - On Output():
+//
+//   - Logged as Err
+//
+//   - Send to the error-service
+func (b *Builder) Err() *Builder {
+	b.errorData.Severity = SevErr
+	return b
+}
+
+// Warn changes the Severity to WARN
+// The error will be:
+//
+//   - On Build():
+//
+//   - -(nothing)-
+//
+//   - On Print():
+//
+//   - Short-Logged as Warn
+//
+//   - On Output():
+//
+//   - Logged as Warn
+func (b *Builder) Warn() *Builder {
+	b.errorData.Severity = SevWarn
+	return b
+}
+
+// Info changes the Severity to INFO
+// The error will be:
+//
+//   - On Build():
+//
+//   - -(nothing)-
+//
+//   - On Print():
+//
+//   - -(nothing)-
+//
+//   - On Output():
+//
+//   - -(nothing)-
+func (b *Builder) Info() *Builder {
+	b.errorData.Severity = SevInfo
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+// User sets the Category to CatUser
+//
+// Errors with category
+func (b *Builder) User() *Builder {
+	b.errorData.Category = CatUser
+	return b
+}
+
+func (b *Builder) System() *Builder {
+	b.errorData.Category = CatSystem
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) NoLog() *Builder {
+	b.noLog = true
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) Id(key string, val fmt.Stringer) *Builder {
+	return b.addMeta(key, MDTID, newIDWrap(val))
+}
+
+func (b *Builder) StrPtr(key string, val *string) *Builder {
+	return b.addMeta(key, MDTStringPtr, val)
+}
+
+func (b *Builder) Str(key string, val string) *Builder {
+	return b.addMeta(key, MDTString, val)
+}
+
+func (b *Builder) Int(key string, val int) *Builder {
+	return b.addMeta(key, MDTInt, val)
+}
+
+func (b *Builder) Int8(key string, val int8) *Builder {
+	return b.addMeta(key, MDTInt8, val)
+}
+
+func (b *Builder) Int16(key string, val int16) *Builder {
+	return b.addMeta(key, MDTInt16, val)
+}
+
+func (b *Builder) Int32(key string, val int32) *Builder {
+	return b.addMeta(key, MDTInt32, val)
+}
+
+func (b *Builder) Int64(key string, val int64) *Builder {
+	return b.addMeta(key, MDTInt64, val)
+}
+
+func (b *Builder) Float32(key string, val float32) *Builder {
+	return b.addMeta(key, MDTFloat32, val)
+}
+
+func (b *Builder) Float64(key string, val float64) *Builder {
+	return b.addMeta(key, MDTFloat64, val)
+}
+
+func (b *Builder) Bool(key string, val bool) *Builder {
+	return b.addMeta(key, MDTBool, val)
+}
+
+func (b *Builder) Bytes(key string, val []byte) *Builder {
+	return b.addMeta(key, MDTBytes, val)
+}
+
+func (b *Builder) ObjectID(key string, val primitive.ObjectID) *Builder {
+	return b.addMeta(key, MDTObjectID, val)
+}
+
+func (b *Builder) Time(key string, val time.Time) *Builder {
+	return b.addMeta(key, MDTTime, val)
+}
+
+func (b *Builder) Dur(key string, val time.Duration) *Builder {
+	return b.addMeta(key, MDTDuration, val)
+}
+
+func (b *Builder) Strs(key string, val []string) *Builder {
+	return b.addMeta(key, MDTStringArray, val)
+}
+
+func (b *Builder) Ints(key string, val []int) *Builder {
+	return b.addMeta(key, MDTIntArray, val)
+}
+
+func (b *Builder) Ints32(key string, val []int32) *Builder {
+	return b.addMeta(key, MDTInt32Array, val)
+}
+
+func (b *Builder) Type(key string, cls interface{}) *Builder {
+	return b.addMeta(key, MDTString, fmt.Sprintf("%T", cls))
+}
+
+func (b *Builder) Interface(key string, val interface{}) *Builder {
+	return b.addMeta(key, MDTAny, newAnyWrap(val))
+}
+
+func (b *Builder) Any(key string, val any) *Builder {
+	return b.addMeta(key, MDTAny, newAnyWrap(val))
+}
+
+func (b *Builder) Stringer(key string, val fmt.Stringer) *Builder {
+	if langext.IsNil(val) {
+		return b.addMeta(key, MDTString, "(!nil)")
+	} else {
+		return b.addMeta(key, MDTString, val.String())
+	}
+}
+
+func (b *Builder) Enum(key string, val enums.Enum) *Builder {
+	return b.addMeta(key, MDTEnum, newEnumWrap(val))
+}
+
+func (b *Builder) Stack() *Builder {
+	return b.addMeta("@Stack", MDTString, string(debug.Stack()))
+}
+
+func (b *Builder) Errs(key string, val []error) *Builder {
+	for i, valerr := range val {
+		b.addMeta(fmt.Sprintf("%v[%v]", key, i), MDTString, Get(valerr).errorData.FormatLog(LogPrintFull))
+	}
+	return b
+}
+
+func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request) *Builder {
+	if v := ctx.Value("start_timestamp"); v != nil {
+		if t, ok := v.(time.Time); ok {
+			b.Time("ctx.startTimestamp", t)
+			b.Time("ctx.endTimestamp", time.Now())
+		}
+	}
+	b.Str("gin.method", req.Method)
+	b.Str("gin.path", g.FullPath())
+	b.Strs("gin.header", extractHeader(g.Request.Header))
+	if req.URL != nil {
+		b.Str("gin.url", req.URL.String())
+	}
+	if ctxVal := g.GetString("apiversion"); ctxVal != "" {
+		b.Str("gin.context.apiversion", ctxVal)
+	}
+	if ctxVal := g.GetString("uid"); ctxVal != "" {
+		b.Str("gin.context.uid", ctxVal)
+	}
+	if ctxVal := g.GetString("fcmId"); ctxVal != "" {
+		b.Str("gin.context.fcmid", ctxVal)
+	}
+	if ctxVal := g.GetString("reqid"); ctxVal != "" {
+		b.Str("gin.context.reqid", ctxVal)
+	}
+	if req.Method != "GET" && req.Body != nil {
+
+		if req.Header.Get("Content-Type") == "application/json" {
+			if brc, ok := req.Body.(dataext.BufferedReadCloser); ok {
+				if bin, err := brc.BufferedAll(); err == nil {
+					if len(bin) < 16*1024 {
+						var prettyJSON bytes.Buffer
+						err = json.Indent(&prettyJSON, bin, "", "  ")
+						if err == nil {
+							b.Str("gin.body", string(prettyJSON.Bytes()))
+						} else {
+							b.Bytes("gin.body", bin)
+						}
+					} else {
+						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+					}
+				}
+			}
+		}
+
+		if req.Header.Get("Content-Type") == "multipart/form-data" || req.Header.Get("Content-Type") == "x-www-form-urlencoded" {
+			if brc, ok := req.Body.(dataext.BufferedReadCloser); ok {
+				if bin, err := brc.BufferedAll(); err == nil {
+					if len(bin) < 16*1024 {
+						b.Bytes("gin.body", bin)
+					} else {
+						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+					}
+				}
+			}
+		}
+
+	}
+
+	b.containsGinData = true
+	return b
+}
+
+func formatHeader(header map[string][]string) string {
+	ml := 1
+	for k, _ := range header {
+		if len(k) > ml {
+			ml = len(k)
+		}
+	}
+	r := ""
+	for k, v := range header {
+		if r != "" {
+			r += "\n"
+		}
+		for _, hval := range v {
+			value := hval
+			value = strings.ReplaceAll(value, "\n", "\\n")
+			value = strings.ReplaceAll(value, "\r", "\\r")
+			value = strings.ReplaceAll(value, "\t", "\\t")
+			r += langext.StrPadRight(k, " ", ml) + " := " + value
+		}
+	}
+	return r
+}
+
+func extractHeader(header map[string][]string) []string {
+	r := make([]string, 0, len(header))
+	for k, v := range header {
+		for _, hval := range v {
+			value := hval
+			value = strings.ReplaceAll(value, "\n", "\\n")
+			value = strings.ReplaceAll(value, "\r", "\\r")
+			value = strings.ReplaceAll(value, "\t", "\\t")
+			r = append(r, k+": "+value)
+		}
+	}
+	return r
+}
+
+// ----------------------------------------------------------------------------
+
+// Build creates a new error, ready to pass up the stack
+// If the errors is not SevWarn or SevInfo it gets also logged (in short form, without stacktrace) onto stdout
+// Can be gloablly configured with ZeroLogErrTraces and ZeroLogAllTraces
+// Can be locally suppressed with Builder.NoLog()
+func (b *Builder) Build() error {
+	warnOnPkgConfigNotInitialized()
+
+	if pkgconfig.DisableErrorWrapping && b.wrappedErr != nil {
+		return b.wrappedErr
+	}
+
+	if pkgconfig.ZeroLogErrTraces && !b.noLog && (b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal) {
+		b.errorData.ShortLog(stackSkipLogger.Error())
+	} else if pkgconfig.ZeroLogAllTraces && !b.noLog {
+		b.errorData.ShortLog(stackSkipLogger.Error())
+	}
+
+	b.errorData.CallListener(MethodBuild)
+
+	return b.errorData
+}
+
+// Output prints the error onto the gin stdout.
+// The error also gets printed to stdout/stderr
+// If the error is SevErr|SevFatal we also send it to the error-service
+func (b *Builder) Output(ctx context.Context, g *gin.Context) {
+	if !b.containsGinData && g.Request != nil {
+		// Auto-Add gin metadata if the caller hasn't already done it
+		b.GinReq(ctx, g, g.Request)
+	}
+
+	b.errorData.Output(g)
+
+	if b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal {
+		b.errorData.Log(stackSkipLogger.Error())
+	} else if b.errorData.Severity == SevWarn {
+		b.errorData.Log(stackSkipLogger.Warn())
+	}
+
+	b.errorData.CallListener(MethodOutput)
+}
+
+// Print prints the error
+// If the error is SevErr we also send it to the error-service
+func (b *Builder) Print() {
+	if b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal {
+		b.errorData.Log(stackSkipLogger.Error())
+	} else if b.errorData.Severity == SevWarn {
+		b.errorData.ShortLog(stackSkipLogger.Warn())
+	}
+
+	b.errorData.CallListener(MethodPrint)
+}
+
+func (b *Builder) Format(level LogPrintLevel) string {
+	return b.errorData.FormatLog(level)
+}
+
+// Fatal prints the error and terminates the program
+// If the error is SevErr we also send it to the error-service
+func (b *Builder) Fatal() {
+	b.errorData.Severity = SevFatal
+	b.errorData.Log(stackSkipLogger.WithLevel(zerolog.FatalLevel))
+
+	b.errorData.CallListener(MethodFatal)
+
+	os.Exit(1)
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) addMeta(key string, mdtype metaDataType, val interface{}) *Builder {
+	b.errorData.Meta.add(key, mdtype, val)
+	return b
+}

exerr/constructor.go

@@ -0,0 +1,204 @@
+package exerr
+
+import (
+	"encoding/json"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"reflect"
+	"time"
+)
+
+var reflectTypeStr = reflect.TypeOf("")
+
+func FromError(err error) *ExErr {
+	if verr, ok := err.(*ExErr); ok {
+		// A simple ExErr
+		return verr
+	}
+
+	// A foreign error (eg a MongoDB exception)
+	return &ExErr{
+		UniqueID:       newID(),
+		Category:       CatForeign,
+		Type:           TypeInternal,
+		Severity:       SevErr,
+		Timestamp:      time.Time{},
+		StatusCode:     nil,
+		Message:        err.Error(),
+		WrappedErrType: fmt.Sprintf("%T", err),
+		WrappedErr:     err,
+		Caller:         "",
+		OriginalError:  nil,
+		Meta:           getForeignMeta(err),
+	}
+}
+
+func newExErr(cat ErrorCategory, errtype ErrorType, msg string) *ExErr {
+	return &ExErr{
+		UniqueID:       newID(),
+		Category:       cat,
+		Type:           errtype,
+		Severity:       SevErr,
+		Timestamp:      time.Now(),
+		StatusCode:     nil,
+		Message:        msg,
+		WrappedErrType: "",
+		WrappedErr:     nil,
+		Caller:         callername(2),
+		OriginalError:  nil,
+		Meta:           make(map[string]MetaValue),
+	}
+}
+
+func wrapExErr(e *ExErr, msg string, cat ErrorCategory, stacktraceskip int) *ExErr {
+	return &ExErr{
+		UniqueID:       newID(),
+		Category:       cat,
+		Type:           TypeWrap,
+		Severity:       SevErr,
+		Timestamp:      time.Now(),
+		StatusCode:     e.StatusCode,
+		Message:        msg,
+		WrappedErrType: "",
+		WrappedErr:     nil,
+		Caller:         callername(1 + stacktraceskip),
+		OriginalError:  e,
+		Meta:           make(map[string]MetaValue),
+	}
+}
+
+func getForeignMeta(err error) (mm MetaMap) {
+	mm = make(map[string]MetaValue)
+
+	defer func() {
+		if panicerr := recover(); panicerr != nil {
+			New(TypePanic, "Panic while trying to get foreign meta").
+				Str("source", err.Error()).
+				Interface("panic-object", panicerr).
+				Stack().
+				Print()
+		}
+	}()
+
+	rval := reflect.ValueOf(err)
+	if rval.Kind() == reflect.Interface || rval.Kind() == reflect.Ptr {
+		rval = reflect.ValueOf(err).Elem()
+	}
+
+	mm.add("foreign.errortype", MDTString, rval.Type().String())
+
+	for k, v := range addMetaPrefix("foreign", getReflectedMetaValues(err, 8)) {
+		mm[k] = v
+	}
+
+	return mm
+}
+
+func getReflectedMetaValues(value interface{}, remainingDepth int) map[string]MetaValue {
+
+	if remainingDepth <= 0 {
+		return map[string]MetaValue{}
+	}
+
+	if langext.IsNil(value) {
+		return map[string]MetaValue{"": {DataType: MDTNil, Value: nil}}
+	}
+
+	rval := reflect.ValueOf(value)
+
+	if rval.Type().Kind() == reflect.Ptr {
+
+		if rval.IsNil() {
+			return map[string]MetaValue{"*": {DataType: MDTNil, Value: nil}}
+		}
+
+		elem := rval.Elem()
+
+		return addMetaPrefix("*", getReflectedMetaValues(elem.Interface(), remainingDepth-1))
+	}
+
+	if !rval.CanInterface() {
+		return map[string]MetaValue{"": {DataType: MDTString, Value: "<<no-interface>>"}}
+	}
+
+	raw := rval.Interface()
+
+	switch ifraw := raw.(type) {
+	case time.Time:
+		return map[string]MetaValue{"": {DataType: MDTTime, Value: ifraw}}
+	case time.Duration:
+		return map[string]MetaValue{"": {DataType: MDTDuration, Value: ifraw}}
+	case int:
+		return map[string]MetaValue{"": {DataType: MDTInt, Value: ifraw}}
+	case int8:
+		return map[string]MetaValue{"": {DataType: MDTInt8, Value: ifraw}}
+	case int16:
+		return map[string]MetaValue{"": {DataType: MDTInt16, Value: ifraw}}
+	case int32:
+		return map[string]MetaValue{"": {DataType: MDTInt32, Value: ifraw}}
+	case int64:
+		return map[string]MetaValue{"": {DataType: MDTInt64, Value: ifraw}}
+	case string:
+		return map[string]MetaValue{"": {DataType: MDTString, Value: ifraw}}
+	case bool:
+		return map[string]MetaValue{"": {DataType: MDTBool, Value: ifraw}}
+	case []byte:
+		return map[string]MetaValue{"": {DataType: MDTBytes, Value: ifraw}}
+	case float32:
+		return map[string]MetaValue{"": {DataType: MDTFloat32, Value: ifraw}}
+	case float64:
+		return map[string]MetaValue{"": {DataType: MDTFloat64, Value: ifraw}}
+	case []int:
+		return map[string]MetaValue{"": {DataType: MDTIntArray, Value: ifraw}}
+	case []int32:
+		return map[string]MetaValue{"": {DataType: MDTInt32Array, Value: ifraw}}
+	case primitive.ObjectID:
+		return map[string]MetaValue{"": {DataType: MDTObjectID, Value: ifraw}}
+	case []string:
+		return map[string]MetaValue{"": {DataType: MDTStringArray, Value: ifraw}}
+	}
+
+	if rval.Type().Kind() == reflect.Struct {
+		m := make(map[string]MetaValue)
+		for i := 0; i < rval.NumField(); i++ {
+			fieldtype := rval.Type().Field(i)
+
+			fieldname := fieldtype.Name
+
+			if fieldtype.IsExported() {
+				for k, v := range addMetaPrefix(fieldname, getReflectedMetaValues(rval.Field(i).Interface(), remainingDepth-1)) {
+					m[k] = v
+				}
+			}
+		}
+		return m
+	}
+
+	if rval.Type().ConvertibleTo(reflectTypeStr) {
+		return map[string]MetaValue{"": {DataType: MDTString, Value: rval.Convert(reflectTypeStr).String()}}
+	}
+
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		panic(err) // gets recovered later up
+	}
+
+	return map[string]MetaValue{"": {DataType: MDTString, Value: string(jsonval)}}
+}
+
+func addMetaPrefix(prefix string, m map[string]MetaValue) map[string]MetaValue {
+	if len(m) == 1 {
+		for k, v := range m {
+			if k == "" {
+				return map[string]MetaValue{prefix: v}
+			}
+		}
+	}
+
+	r := make(map[string]MetaValue, len(m))
+	for k, v := range m {
+		r[prefix+"."+k] = v
+	}
+	return r
+}

exerr/data.go

@@ -0,0 +1,95 @@
+package exerr
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+)
+
+type ErrorCategory struct{ Category string }
+
+var (
+	CatWrap    = ErrorCategory{"Wrap"}    // The error is simply wrapping another error (e.g. when a grpc call returns an error)
+	CatSystem  = ErrorCategory{"System"}  // An internal system error (e.g. connection to db failed)
+	CatUser    = ErrorCategory{"User"}    // The user (the API caller) did something wrong (e.g. he has no permissions to do this)
+	CatForeign = ErrorCategory{"Foreign"} // A foreign error that some component threw (e.g. an unknown mongodb error), happens if we call Wrap(..) on an non-bmerror value
+)
+
+//goland:noinspection GoUnusedGlobalVariable
+var AllCategories = []ErrorCategory{CatWrap, CatSystem, CatUser, CatForeign}
+
+type ErrorSeverity struct{ Severity string }
+
+var (
+	SevTrace = ErrorSeverity{"Trace"}
+	SevDebug = ErrorSeverity{"Debug"}
+	SevInfo  = ErrorSeverity{"Info"}
+	SevWarn  = ErrorSeverity{"Warn"}
+	SevErr   = ErrorSeverity{"Err"}
+	SevFatal = ErrorSeverity{"Fatal"}
+)
+
+//goland:noinspection GoUnusedGlobalVariable
+var AllSeverities = []ErrorSeverity{SevTrace, SevDebug, SevInfo, SevWarn, SevErr, SevFatal}
+
+type ErrorType struct {
+	Key               string
+	DefaultStatusCode *int
+}
+
+//goland:noinspection GoUnusedGlobalVariable
+var (
+	TypeInternal       = NewType("INTERNAL_ERROR", langext.Ptr(500))
+	TypePanic          = NewType("PANIC", langext.Ptr(500))
+	TypeNotImplemented = NewType("NOT_IMPLEMENTED", langext.Ptr(500))
+
+	TypeMongoQuery        = NewType("MONGO_QUERY", langext.Ptr(500))
+	TypeCursorTokenDecode = NewType("CURSOR_TOKEN_DECODE", langext.Ptr(500))
+	TypeMongoFilter       = NewType("MONGO_FILTER", langext.Ptr(500))
+	TypeMongoReflection   = NewType("MONGO_REFLECTION", langext.Ptr(500))
+	TypeMongoInvalidOpt   = NewType("MONGO_INVALIDOPT", langext.Ptr(500))
+
+	TypeSQLQuery  = NewType("SQL_QUERY", langext.Ptr(500))
+	TypeSQLBuild  = NewType("SQL_BUILD", langext.Ptr(500))
+	TypeSQLDecode = NewType("SQL_DECODE", langext.Ptr(500))
+
+	TypeWrap = NewType("Wrap", nil)
+
+	TypeBindFailURI      = NewType("BINDFAIL_URI", langext.Ptr(400))
+	TypeBindFailQuery    = NewType("BINDFAIL_QUERY", langext.Ptr(400))
+	TypeBindFailJSON     = NewType("BINDFAIL_JSON", langext.Ptr(400))
+	TypeBindFailFormData = NewType("BINDFAIL_FORMDATA", langext.Ptr(400))
+	TypeBindFailHeader   = NewType("BINDFAIL_HEADER", langext.Ptr(400))
+
+	TypeMarshalEntityID = NewType("MARSHAL_ENTITY_ID", langext.Ptr(400))
+	TypeInvalidCSID     = NewType("INVALID_CSID", langext.Ptr(400))
+
+	TypeGoogleStatuscode = NewType("GOOGLE_STATUSCODE", langext.Ptr(400))
+	TypeGoogleResponse   = NewType("GOOGLE_RESPONSE", langext.Ptr(400))
+
+	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
+	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))
+
+	// other values come from the downstream application that uses goext
+)
+
+var registeredTypes = dataext.SyncMap[string, ErrorType]{}
+
+func NewType(key string, defStatusCode *int) ErrorType {
+	et := ErrorType{key, defStatusCode}
+
+	registeredTypes.Set(key, et)
+
+	return et
+}
+
+func ListRegisteredTypes() []ErrorType {
+	return registeredTypes.GetAllValues()
+}
+
+type LogPrintLevel string
+
+const (
+	LogPrintFull     LogPrintLevel = "Full"
+	LogPrintOverview LogPrintLevel = "Overview"
+	LogPrintShort    LogPrintLevel = "Short"
+)

exerr/errinit.go

@@ -0,0 +1,88 @@
+package exerr
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+)
+
+type ErrorPackageConfig struct {
+	ZeroLogErrTraces       bool                                             // autom print zerolog logs on .Build()  (for SevErr and SevFatal)
+	ZeroLogAllTraces       bool                                             // autom print zerolog logs on .Build()  (for all Severities)
+	RecursiveErrors        bool                                             // errors contains their Origin-Error
+	ExtendedGinOutput      bool                                             // Log extended data (trace, meta, ...) to gin in err.Output()
+	IncludeMetaInGinOutput bool                                             // Log meta fields ( from e.g. `.Str(key, val).Build()` ) to gin in err.Output()
+	ExtendGinOutput        func(err *ExErr, json map[string]any)            // (Optionally) extend the gin output with more fields
+	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any) // (Optionally) extend the gin `__data` output with more fields
+	DisableErrorWrapping   bool                                             // Disables the exerr.Wrap()...Build() function - will always return the original error
+}
+
+type ErrorPackageConfigInit struct {
+	ZeroLogErrTraces       *bool
+	ZeroLogAllTraces       *bool
+	RecursiveErrors        *bool
+	ExtendedGinOutput      *bool
+	IncludeMetaInGinOutput *bool
+	ExtendGinOutput        func(err *ExErr, json map[string]any)
+	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any)
+	DisableErrorWrapping   *bool
+}
+
+var initialized = false
+
+var pkgconfig = ErrorPackageConfig{
+	ZeroLogErrTraces:       true,
+	ZeroLogAllTraces:       false,
+	RecursiveErrors:        true,
+	ExtendedGinOutput:      false,
+	IncludeMetaInGinOutput: true,
+	ExtendGinOutput:        func(err *ExErr, json map[string]any) {},
+	ExtendGinDataOutput:    func(err *ExErr, depth int, json map[string]any) {},
+	DisableErrorWrapping:   false,
+}
+
+// Init initializes the exerr packages
+// Must be called at the program start, before (!) any errors
+// Is not thread-safe
+func Init(cfg ErrorPackageConfigInit) {
+	if initialized {
+		panic("Cannot re-init error package")
+	}
+
+	ego := func(err *ExErr, json map[string]any) {}
+	egdo := func(err *ExErr, depth int, json map[string]any) {}
+
+	if cfg.ExtendGinOutput != nil {
+		ego = cfg.ExtendGinOutput
+	}
+	if cfg.ExtendGinDataOutput != nil {
+		egdo = cfg.ExtendGinDataOutput
+	}
+
+	pkgconfig = ErrorPackageConfig{
+		ZeroLogErrTraces:       langext.Coalesce(cfg.ZeroLogErrTraces, pkgconfig.ZeroLogErrTraces),
+		ZeroLogAllTraces:       langext.Coalesce(cfg.ZeroLogAllTraces, pkgconfig.ZeroLogAllTraces),
+		RecursiveErrors:        langext.Coalesce(cfg.RecursiveErrors, pkgconfig.RecursiveErrors),
+		ExtendedGinOutput:      langext.Coalesce(cfg.ExtendedGinOutput, pkgconfig.ExtendedGinOutput),
+		IncludeMetaInGinOutput: langext.Coalesce(cfg.IncludeMetaInGinOutput, pkgconfig.IncludeMetaInGinOutput),
+		ExtendGinOutput:        ego,
+		ExtendGinDataOutput:    egdo,
+		DisableErrorWrapping:   langext.Coalesce(cfg.DisableErrorWrapping, pkgconfig.DisableErrorWrapping),
+	}
+
+	initialized = true
+}
+
+func Initialized() bool {
+	return initialized
+}
+
+func warnOnPkgConfigNotInitialized() {
+	if !initialized {
+		fmt.Printf("\n")
+		fmt.Printf("%s\n", langext.StrRepeat("=", 80))
+		fmt.Printf("%s\n", "[WARNING] exerr package used without initializiation")
+		fmt.Printf("%s\n", "          call exerr.Init() in your main() function")
+		fmt.Printf("%s\n", langext.StrRepeat("=", 80))
+		fmt.Printf("\n")
+	}
+}

exerr/exerr.go

@@ -0,0 +1,383 @@
+package exerr
+
+import (
+	"github.com/rs/xid"
+	"github.com/rs/zerolog"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"reflect"
+	"strings"
+	"time"
+)
+
+type ExErr struct {
+	UniqueID string `json:"uniqueID"`
+
+	Timestamp time.Time     `json:"timestamp"`
+	Category  ErrorCategory `json:"category"`
+	Severity  ErrorSeverity `json:"severity"`
+	Type      ErrorType     `json:"type"`
+
+	StatusCode *int `json:"statusCode"`
+
+	Message        string `json:"message"`
+	WrappedErrType string `json:"wrappedErrType"`
+	WrappedErr     any    `json:"-"`
+	Caller         string `json:"caller"`
+
+	OriginalError *ExErr `json:"originalError"`
+
+	Meta MetaMap `json:"meta"`
+}
+
+func (ee *ExErr) Error() string {
+	return ee.RecursiveMessage()
+}
+
+// Unwrap must be implemented so that some error.XXX methods work
+func (ee *ExErr) Unwrap() error {
+	if ee.OriginalError == nil {
+		return nil // this is neccessary - otherwise we return a wrapped nil and the `x == nil` comparison fails (= panic in errors.Is and other failures)
+	}
+	return ee.OriginalError
+}
+
+// Is must be implemented so that error.Is(x) works
+func (ee *ExErr) Is(e error) bool {
+	return IsFrom(ee, e)
+}
+
+// As must be implemented so that error.As(x) works
+//
+//goland:noinspection GoTypeAssertionOnErrors
+func (ee *ExErr) As(target any) bool {
+	if dstErr, ok := target.(*ExErr); ok {
+
+		if dst0, ok := ee.contains(dstErr); ok {
+			dstErr = dst0
+			return true
+		} else {
+			return false
+		}
+
+	} else {
+
+		val := reflect.ValueOf(target)
+
+		typStr := val.Type().Elem().String()
+
+		for curr := ee; curr != nil; curr = curr.OriginalError {
+			if curr.Category == CatForeign && curr.WrappedErrType == typStr && curr.WrappedErr != nil {
+				val.Elem().Set(reflect.ValueOf(curr.WrappedErr))
+				return true
+			}
+		}
+
+		return false
+	}
+}
+
+func (ee *ExErr) Log(evt *zerolog.Event) {
+	evt.Msg(ee.FormatLog(LogPrintFull))
+}
+
+func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
+	if lvl == LogPrintShort {
+
+		msg := ee.Message
+		if ee.OriginalError != nil && ee.OriginalError.Category == CatForeign {
+			msg = msg + " (" + strings.ReplaceAll(ee.OriginalError.Message, "\n", " ") + ")"
+		}
+
+		if ee.Type != TypeWrap {
+			return "[" + ee.Type.Key + "] " + msg
+		} else {
+			return msg
+		}
+
+	} else if lvl == LogPrintOverview {
+
+		str := "[" + ee.RecursiveType().Key + "] <" + ee.UniqueID + "> " + strings.ReplaceAll(ee.RecursiveMessage(), "\n", " ") + "\n"
+
+		indent := ""
+		for curr := ee; curr != nil; curr = curr.OriginalError {
+			indent += "  "
+
+			str += indent
+			str += "-> "
+			strmsg := strings.Trim(curr.Message, " \r\n\t")
+			if lbidx := strings.Index(curr.Message, "\n"); lbidx >= 0 {
+				strmsg = strmsg[0:lbidx]
+			}
+			strmsg = langext.StrLimit(strmsg, 61, "...")
+			str += strmsg
+			str += "\n"
+
+		}
+		return str
+
+	} else if lvl == LogPrintFull {
+
+		str := "[" + ee.RecursiveType().Key + "] <" + ee.UniqueID + "> " + strings.ReplaceAll(ee.RecursiveMessage(), "\n", " ") + "\n"
+
+		indent := ""
+		for curr := ee; curr != nil; curr = curr.OriginalError {
+			indent += "  "
+
+			etype := ee.Type.Key
+			if ee.Type == TypeWrap {
+				etype = "~"
+			}
+
+			str += indent
+			str += "-> ["
+			str += etype
+			if curr.Category == CatForeign {
+				str += "|Foreign"
+			}
+			str += "] "
+			str += strings.ReplaceAll(curr.Message, "\n", " ")
+			if curr.Caller != "" {
+				str += " (@ "
+				str += curr.Caller
+				str += ")"
+			}
+			str += "\n"
+
+			if curr.Meta.Any() {
+				meta := indent + "   {" + curr.Meta.FormatOneLine(240) + "}"
+				if len(meta) < 200 {
+					str += meta
+					str += "\n"
+				} else {
+					str += curr.Meta.FormatMultiLine(indent+"   ", "  ", 1024)
+					str += "\n"
+				}
+			}
+		}
+		return str
+
+	} else {
+
+		return "[?[" + ee.UniqueID + "]?]"
+
+	}
+}
+
+func (ee *ExErr) ShortLog(evt *zerolog.Event) {
+	ee.Meta.Apply(evt, langext.Ptr(240)).Msg(ee.FormatLog(LogPrintShort))
+}
+
+// RecursiveMessage returns the message to show
+// = first error (top-down) that is not wrapping/foreign/empty
+// = lowest level error (that is not empty)
+// = fallback to self.message
+func (ee *ExErr) RecursiveMessage() string {
+
+	// ==== [1] ==== first error (top-down) that is not wrapping/foreign/empty
+
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Message != "" && curr.Category != CatWrap && curr.Category != CatForeign {
+			return curr.Message
+		}
+	}
+
+	// ==== [2] ==== lowest level error (that is not empty)
+
+	deepestMsg := ""
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Message != "" {
+			deepestMsg = curr.Message
+		}
+	}
+	if deepestMsg != "" {
+		return deepestMsg
+	}
+
+	// ==== [3] ==== fallback to self.message
+
+	return ee.Message
+}
+
+// RecursiveType returns the statuscode to use
+// = first error (top-down) that is not wrapping/empty
+func (ee *ExErr) RecursiveType() ErrorType {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Type != TypeWrap {
+			return curr.Type
+		}
+
+	}
+
+	// fallback to self
+	return ee.Type
+}
+
+// RecursiveStatuscode returns the HTTP Statuscode to use
+// = first error (top-down) that has a statuscode set
+func (ee *ExErr) RecursiveStatuscode() *int {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.StatusCode != nil {
+			return langext.Ptr(*curr.StatusCode)
+		}
+	}
+
+	return nil
+}
+
+// RecursiveCategory returns the ErrorCategory to use
+// = first error (top-down) that has a statuscode set
+func (ee *ExErr) RecursiveCategory() ErrorCategory {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Category != CatWrap {
+			return curr.Category
+		}
+	}
+
+	// fallback to <empty>
+	return ee.Category
+}
+
+// RecursiveMeta searches (top-down) for teh first error that has a meta value with teh specified key
+// and returns its value (or nil)
+func (ee *ExErr) RecursiveMeta(key string) *MetaValue {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if metaval, ok := curr.Meta[key]; ok {
+			return langext.Ptr(metaval)
+		}
+	}
+
+	return nil
+}
+
+// Depth returns the depth of recursively contained errors
+func (ee *ExErr) Depth() int {
+	if ee.OriginalError == nil {
+		return 1
+	} else {
+		return ee.OriginalError.Depth() + 1
+	}
+}
+
+// GetMeta returns the meta value with the specified key
+// this method recurses through all wrapped errors and returns the first matching meta value
+func (ee *ExErr) GetMeta(key string) (any, bool) {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if v, ok := curr.Meta[key]; ok {
+			return v.Value, true
+		}
+	}
+
+	return nil, false
+}
+
+// GetMetaString functions the same as GetMeta, but returns false if the type does not match
+func (ee *ExErr) GetMetaString(key string) (string, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(string); ok {
+			return v2, true
+		}
+	}
+	return "", false
+}
+
+func (ee *ExErr) GetMetaBool(key string) (bool, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(bool); ok {
+			return v2, true
+		}
+	}
+	return false, false
+}
+
+func (ee *ExErr) GetMetaInt(key string) (int, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(int); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaFloat32(key string) (float32, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(float32); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaFloat64(key string) (float64, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(float64); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaTime(key string) (time.Time, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(time.Time); ok {
+			return v2, true
+		}
+	}
+	return time.Time{}, false
+}
+
+// contains test if the supplied error is contained in this error (anywhere in the chain)
+func (ee *ExErr) contains(original *ExErr) (*ExErr, bool) {
+	if original == nil {
+		return nil, false
+	}
+
+	if ee == original {
+		return ee, true
+	}
+
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.equalsDirectProperties(curr) {
+			return curr, true
+		}
+	}
+
+	return nil, false
+}
+
+// equalsDirectProperties tests if ee and other are equals, but only looks at primary properties (not `OriginalError` or `Meta`)
+func (ee *ExErr) equalsDirectProperties(other *ExErr) bool {
+
+	if ee.UniqueID != other.UniqueID {
+		return false
+	}
+	if ee.Timestamp != other.Timestamp {
+		return false
+	}
+	if ee.Category != other.Category {
+		return false
+	}
+	if ee.Severity != other.Severity {
+		return false
+	}
+	if ee.Type != other.Type {
+		return false
+	}
+	if ee.StatusCode != other.StatusCode {
+		return false
+	}
+	if ee.Message != other.Message {
+		return false
+	}
+	if ee.WrappedErrType != other.WrappedErrType {
+		return false
+	}
+	if ee.Caller != other.Caller {
+		return false
+	}
+
+	return true
+}
+
+func newID() string {
+	return xid.New().String()
+}

exerr/exerr_test.go

@@ -0,0 +1,93 @@
+package exerr
+
+import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+type golangErr struct {
+	Message string
+}
+
+func (g golangErr) Error() string {
+	return g.Message
+}
+
+type golangErr2 struct {
+	Message string
+}
+
+func (g golangErr2) Error() string {
+	return g.Message
+}
+
+type simpleError struct {
+}
+
+func (g simpleError) Error() string {
+	return "Something simple went wroong"
+}
+
+type simpleError2 struct {
+}
+
+func (g simpleError2) Error() string {
+	return "Something simple went wroong"
+}
+
+func TestExErrIs1(t *testing.T) {
+	e0 := simpleError{}
+
+	wrap := Wrap(e0, "something went wrong").Str("test", "123").Build()
+
+	tst.AssertTrue(t, errors.Is(wrap, simpleError{}))
+	tst.AssertFalse(t, errors.Is(wrap, golangErr{}))
+	tst.AssertFalse(t, errors.Is(wrap, golangErr{"error1"}))
+}
+
+func TestExErrIs2(t *testing.T) {
+	e0 := golangErr{"error1"}
+
+	wrap := Wrap(e0, "something went wrong").Str("test", "123").Build()
+
+	tst.AssertTrue(t, errors.Is(wrap, e0))
+	tst.AssertTrue(t, errors.Is(wrap, golangErr{"error1"}))
+	tst.AssertFalse(t, errors.Is(wrap, golangErr{"error2"}))
+	tst.AssertFalse(t, errors.Is(wrap, simpleError{}))
+}
+
+func TestExErrAs(t *testing.T) {
+
+	e0 := golangErr{"error1"}
+
+	w0 := Wrap(e0, "something went wrong").Str("test", "123").Build()
+
+	{
+		out := golangErr{}
+		ok := errors.As(w0, &out)
+		tst.AssertTrue(t, ok)
+		tst.AssertEqual(t, out.Message, "error1")
+	}
+
+	w1 := Wrap(w0, "outher error").Build()
+
+	{
+		out := golangErr{}
+		ok := errors.As(w1, &out)
+		tst.AssertTrue(t, ok)
+		tst.AssertEqual(t, out.Message, "error1")
+	}
+
+	{
+		out := golangErr2{}
+		ok := errors.As(w1, &out)
+		tst.AssertFalse(t, ok)
+	}
+
+	{
+		out := simpleError2{}
+		ok := errors.As(w1, &out)
+		tst.AssertFalse(t, ok)
+	}
+}

exerr/gin.go

@@ -0,0 +1,125 @@
+package exerr
+
+import (
+	"github.com/gin-gonic/gin"
+	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"net/http"
+	"time"
+)
+
+func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) langext.H {
+	ginJson := langext.H{}
+
+	if ee.UniqueID != "" {
+		ginJson["id"] = ee.UniqueID
+	}
+	if ee.Category != CatWrap {
+		ginJson["category"] = ee.Category
+	}
+	if ee.Type != TypeWrap {
+		ginJson["type"] = ee.Type
+	}
+	if ee.StatusCode != nil {
+		ginJson["statuscode"] = ee.StatusCode
+	}
+	if ee.Message != "" {
+		ginJson["message"] = ee.Message
+	}
+	if ee.Caller != "" {
+		ginJson["caller"] = ee.Caller
+	}
+	if ee.Severity != SevErr {
+		ginJson["severity"] = ee.Severity
+	}
+	if ee.Timestamp != (time.Time{}) {
+		ginJson["time"] = ee.Timestamp.Format(time.RFC3339)
+	}
+	if ee.WrappedErrType != "" {
+		ginJson["wrappedErrType"] = ee.WrappedErrType
+	}
+	if ee.OriginalError != nil {
+		ginJson["original"] = ee.OriginalError.toJson(depth+1, applyExtendListener, outputMeta)
+	}
+
+	if outputMeta {
+		metaJson := langext.H{}
+		for metaKey, metaVal := range ee.Meta {
+			metaJson[metaKey] = metaVal.rawValueForJson()
+		}
+		ginJson["meta"] = metaJson
+	}
+
+	if applyExtendListener {
+		pkgconfig.ExtendGinDataOutput(ee, depth, ginJson)
+	}
+
+	return ginJson
+}
+
+func (ee *ExErr) ToDefaultAPIJson() (string, error) {
+
+	gjr := json.GoJsonRender{Data: ee.ToAPIJson(true, pkgconfig.ExtendedGinOutput, pkgconfig.IncludeMetaInGinOutput), NilSafeSlices: true, NilSafeMaps: true}
+
+	r, err := gjr.RenderString()
+
+	if err != nil {
+		return "", err
+	}
+
+	return r, nil
+}
+
+// ToAPIJson converts the ExError to a json object
+// (the same object as used in the Output(gin) method)
+//
+// Parameters:
+//   - [applyExtendListener]:  if false the pkgconfig.ExtendGinOutput / pkgconfig.ExtendGinDataOutput will not be applied
+//   - [includeWrappedErrors]: if false we do not include the recursive/wrapped errors in `__data`
+//   - [includeMetaFields]:    if true  we also include meta-values (aka from `.Str(key, value).Build()`), needs includeWrappedErrors=true
+func (ee *ExErr) ToAPIJson(applyExtendListener bool, includeWrappedErrors bool, includeMetaFields bool) langext.H {
+
+	apiOutput := langext.H{
+		"errorid":   ee.UniqueID,
+		"message":   ee.RecursiveMessage(),
+		"errorcode": ee.RecursiveType().Key,
+		"category":  ee.RecursiveCategory().Category,
+	}
+
+	if includeWrappedErrors {
+		apiOutput["__data"] = ee.toJson(0, applyExtendListener, includeMetaFields)
+	}
+
+	if applyExtendListener {
+		pkgconfig.ExtendGinOutput(ee, apiOutput)
+	}
+
+	return apiOutput
+}
+
+func (ee *ExErr) Output(g *gin.Context) {
+
+	warnOnPkgConfigNotInitialized()
+
+	var statuscode = http.StatusInternalServerError
+
+	var baseCat = ee.RecursiveCategory()
+	var baseType = ee.RecursiveType()
+	var baseStatuscode = ee.RecursiveStatuscode()
+
+	if baseCat == CatUser {
+		statuscode = http.StatusBadRequest
+	} else if baseCat == CatSystem {
+		statuscode = http.StatusInternalServerError
+	}
+
+	if baseStatuscode != nil {
+		statuscode = *ee.StatusCode
+	} else if baseType.DefaultStatusCode != nil {
+		statuscode = *baseType.DefaultStatusCode
+	}
+
+	ginOutput := ee.ToAPIJson(true, pkgconfig.ExtendedGinOutput, pkgconfig.IncludeMetaInGinOutput)
+
+	g.Render(statuscode, json.GoJsonRender{Data: ginOutput, NilSafeSlices: true, NilSafeMaps: true})
+}

exerr/helper.go

@@ -0,0 +1,88 @@
+package exerr
+
+import "fmt"
+
+// IsType test if the supplied error is of the specified ErrorType.
+func IsType(err error, errType ErrorType) bool {
+	if err == nil {
+		return false
+	}
+
+	bmerr := FromError(err)
+	for bmerr != nil {
+		if bmerr.Type == errType {
+			return true
+		}
+		bmerr = bmerr.OriginalError
+	}
+
+	return false
+}
+
+// IsFrom test if the supplied error stems originally from original
+func IsFrom(e error, original error) bool {
+	if e == nil {
+		return false
+	}
+
+	//goland:noinspection GoDirectComparisonOfErrors
+	if e == original {
+		return true
+	}
+
+	bmerr := FromError(e)
+	for bmerr == nil {
+		return false
+	}
+
+	for curr := bmerr; curr != nil; curr = curr.OriginalError {
+		if curr.Category == CatForeign && curr.Message == original.Error() && curr.WrappedErrType == fmt.Sprintf("%T", original) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// HasSourceMessage tests if the supplied error stems originally from an error with the message msg
+func HasSourceMessage(e error, msg string) bool {
+	if e == nil {
+		return false
+	}
+
+	bmerr := FromError(e)
+	for bmerr == nil {
+		return false
+	}
+
+	for curr := bmerr; curr != nil; curr = curr.OriginalError {
+		if curr.OriginalError == nil && curr.Message == msg {
+			return true
+		}
+	}
+
+	return false
+}
+
+func MessageMatch(e error, matcher func(string) bool) bool {
+	if e == nil {
+		return false
+	}
+
+	if matcher(e.Error()) {
+		return true
+	}
+
+	bmerr := FromError(e)
+	for bmerr == nil {
+		return false
+	}
+
+	for curr := bmerr; curr != nil; curr = curr.OriginalError {
+		if matcher(curr.Message) {
+			return true
+		}
+	}
+
+	return false
+}

exerr/listener.go

@@ -0,0 +1,35 @@
+package exerr
+
+import (
+	"sync"
+)
+
+type Method string
+
+const (
+	MethodOutput Method = "OUTPUT"
+	MethodPrint  Method = "PRINT"
+	MethodBuild  Method = "BUILD"
+	MethodFatal  Method = "FATAL"
+)
+
+type Listener = func(method Method, v *ExErr)
+
+var listenerLock = sync.Mutex{}
+var listener = make([]Listener, 0)
+
+func RegisterListener(l Listener) {
+	listenerLock.Lock()
+	defer listenerLock.Unlock()
+
+	listener = append(listener, l)
+}
+
+func (ee *ExErr) CallListener(m Method) {
+	listenerLock.Lock()
+	defer listenerLock.Unlock()
+
+	for _, v := range listener {
+		v(m, ee)
+	}
+}

exerr/meta.go

@@ -0,0 +1,759 @@
+package exerr
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/rs/zerolog"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// This is a buffed up map[string]any
+// we also save type information of the map-values
+// which allows us to deserialize them back into te correct types later
+
+type MetaMap map[string]MetaValue
+
+type metaDataType string
+
+const (
+	MDTString      metaDataType = "String"
+	MDTStringPtr   metaDataType = "StringPtr"
+	MDTInt         metaDataType = "Int"
+	MDTInt8        metaDataType = "Int8"
+	MDTInt16       metaDataType = "Int16"
+	MDTInt32       metaDataType = "Int32"
+	MDTInt64       metaDataType = "Int64"
+	MDTFloat32     metaDataType = "Float32"
+	MDTFloat64     metaDataType = "Float64"
+	MDTBool        metaDataType = "Bool"
+	MDTBytes       metaDataType = "Bytes"
+	MDTObjectID    metaDataType = "ObjectID"
+	MDTTime        metaDataType = "Time"
+	MDTDuration    metaDataType = "Duration"
+	MDTStringArray metaDataType = "StringArr"
+	MDTIntArray    metaDataType = "IntArr"
+	MDTInt32Array  metaDataType = "Int32Arr"
+	MDTID          metaDataType = "ID"
+	MDTAny         metaDataType = "Interface"
+	MDTNil         metaDataType = "Nil"
+	MDTEnum        metaDataType = "Enum"
+)
+
+type MetaValue struct {
+	DataType metaDataType `json:"dataType"`
+	Value    interface{}  `json:"value"`
+}
+
+type metaValueSerialization struct {
+	DataType metaDataType `bson:"dataType"`
+	Value    string       `bson:"value"`
+	Raw      interface{}  `bson:"raw"`
+}
+
+func (v MetaValue) SerializeValue() (string, error) {
+	switch v.DataType {
+	case MDTString:
+		return v.Value.(string), nil
+	case MDTID:
+		return v.Value.(IDWrap).Serialize(), nil
+	case MDTAny:
+		return v.Value.(AnyWrap).Serialize(), nil
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r := v.Value.(*string)
+		if r != nil {
+			return "*" + *r, nil
+		} else {
+			return "#", nil
+		}
+	case MDTInt:
+		return strconv.Itoa(v.Value.(int)), nil
+	case MDTInt8:
+		return strconv.FormatInt(int64(v.Value.(int8)), 10), nil
+	case MDTInt16:
+		return strconv.FormatInt(int64(v.Value.(int16)), 10), nil
+	case MDTInt32:
+		return strconv.FormatInt(int64(v.Value.(int32)), 10), nil
+	case MDTInt64:
+		return strconv.FormatInt(v.Value.(int64), 10), nil
+	case MDTFloat32:
+		return strconv.FormatFloat(float64(v.Value.(float32)), 'X', -1, 32), nil
+	case MDTFloat64:
+		return strconv.FormatFloat(v.Value.(float64), 'X', -1, 64), nil
+	case MDTBool:
+		if v.Value.(bool) {
+			return "true", nil
+		} else {
+			return "false", nil
+		}
+	case MDTBytes:
+		return hex.EncodeToString(v.Value.([]byte)), nil
+	case MDTObjectID:
+		return v.Value.(primitive.ObjectID).Hex(), nil
+	case MDTTime:
+		return strconv.FormatInt(v.Value.(time.Time).Unix(), 10) + "|" + strconv.FormatInt(int64(v.Value.(time.Time).Nanosecond()), 10), nil
+	case MDTDuration:
+		return v.Value.(time.Duration).String(), nil
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r, err := json.Marshal(v.Value.([]string))
+		if err != nil {
+			return "", err
+		}
+		return string(r), nil
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r, err := json.Marshal(v.Value.([]int))
+		if err != nil {
+			return "", err
+		}
+		return string(r), nil
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r, err := json.Marshal(v.Value.([]int32))
+		if err != nil {
+			return "", err
+		}
+		return string(r), nil
+	case MDTNil:
+		return "", nil
+	case MDTEnum:
+		return v.Value.(EnumWrap).Serialize(), nil
+	}
+	return "", errors.New("Unknown type: " + string(v.DataType))
+}
+
+func (v MetaValue) ShortString(lim int) string {
+	switch v.DataType {
+	case MDTString:
+		r := strings.ReplaceAll(v.Value.(string), "\r", "")
+		r = strings.ReplaceAll(r, "\n", "\\n")
+		r = strings.ReplaceAll(r, "\t", "\\t")
+		return langext.StrLimit(r, lim, "...")
+	case MDTID:
+		return v.Value.(IDWrap).String()
+	case MDTAny:
+		return v.Value.(AnyWrap).String()
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r := langext.CoalesceString(v.Value.(*string), "<<null>>")
+		r = strings.ReplaceAll(r, "\r", "")
+		r = strings.ReplaceAll(r, "\n", "\\n")
+		r = strings.ReplaceAll(r, "\t", "\\t")
+		return langext.StrLimit(r, lim, "...")
+	case MDTInt:
+		return strconv.Itoa(v.Value.(int))
+	case MDTInt8:
+		return strconv.FormatInt(int64(v.Value.(int8)), 10)
+	case MDTInt16:
+		return strconv.FormatInt(int64(v.Value.(int16)), 10)
+	case MDTInt32:
+		return strconv.FormatInt(int64(v.Value.(int32)), 10)
+	case MDTInt64:
+		return strconv.FormatInt(v.Value.(int64), 10)
+	case MDTFloat32:
+		return strconv.FormatFloat(float64(v.Value.(float32)), 'g', 4, 32)
+	case MDTFloat64:
+		return strconv.FormatFloat(v.Value.(float64), 'g', 4, 64)
+	case MDTBool:
+		return fmt.Sprintf("%v", v.Value.(bool))
+	case MDTBytes:
+		return langext.StrLimit(hex.EncodeToString(v.Value.([]byte)), lim, "...")
+	case MDTObjectID:
+		return v.Value.(primitive.ObjectID).Hex()
+	case MDTTime:
+		return v.Value.(time.Time).Format(time.RFC3339)
+	case MDTDuration:
+		return v.Value.(time.Duration).String()
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.Marshal(v.Value.([]string))
+		if err != nil {
+			return "(err)"
+		}
+		return langext.StrLimit(string(r), lim, "...")
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.Marshal(v.Value.([]int))
+		if err != nil {
+			return "(err)"
+		}
+		return langext.StrLimit(string(r), lim, "...")
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.Marshal(v.Value.([]int32))
+		if err != nil {
+			return "(err)"
+		}
+		return langext.StrLimit(string(r), lim, "...")
+	case MDTNil:
+		return "<<null>>"
+	case MDTEnum:
+		return v.Value.(EnumWrap).String()
+	}
+	return "(err)"
+}
+
+func (v MetaValue) Apply(key string, evt *zerolog.Event, limitLen *int) *zerolog.Event {
+	switch v.DataType {
+	case MDTString:
+		if limitLen == nil {
+			return evt.Str(key, v.Value.(string))
+		} else {
+			return evt.Str(key, langext.StrLimit(v.Value.(string), *limitLen, "..."))
+		}
+	case MDTID:
+		return evt.Str(key, v.Value.(IDWrap).Value)
+	case MDTAny:
+		if v.Value.(AnyWrap).IsError {
+			return evt.Str(key, "(err)")
+		} else {
+			if limitLen == nil {
+				return evt.Str(key, v.Value.(AnyWrap).Json)
+			} else {
+				return evt.Str(key, langext.StrLimit(v.Value.(AnyWrap).Json, *limitLen, "..."))
+			}
+		}
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return evt.Str(key, "<<null>>")
+		}
+		if limitLen == nil {
+			return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+		} else {
+			return evt.Str(key, langext.StrLimit(langext.CoalesceString(v.Value.(*string), "<<null>>"), *limitLen, "..."))
+		}
+	case MDTInt:
+		return evt.Int(key, v.Value.(int))
+	case MDTInt8:
+		return evt.Int8(key, v.Value.(int8))
+	case MDTInt16:
+		return evt.Int16(key, v.Value.(int16))
+	case MDTInt32:
+		return evt.Int32(key, v.Value.(int32))
+	case MDTInt64:
+		return evt.Int64(key, v.Value.(int64))
+	case MDTFloat32:
+		return evt.Float32(key, v.Value.(float32))
+	case MDTFloat64:
+		return evt.Float64(key, v.Value.(float64))
+	case MDTBool:
+		return evt.Bool(key, v.Value.(bool))
+	case MDTBytes:
+		return evt.Bytes(key, v.Value.([]byte))
+	case MDTObjectID:
+		return evt.Str(key, v.Value.(primitive.ObjectID).Hex())
+	case MDTTime:
+		return evt.Time(key, v.Value.(time.Time))
+	case MDTDuration:
+		return evt.Dur(key, v.Value.(time.Duration))
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return evt.Strs(key, nil)
+		}
+		return evt.Strs(key, v.Value.([]string))
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return evt.Ints(key, nil)
+		}
+		return evt.Ints(key, v.Value.([]int))
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return evt.Ints32(key, nil)
+		}
+		return evt.Ints32(key, v.Value.([]int32))
+	case MDTNil:
+		return evt.Str(key, "<<null>>")
+	case MDTEnum:
+		if v.Value.(EnumWrap).IsNil {
+			return evt.Any(key, nil)
+		} else if v.Value.(EnumWrap).ValueRaw != nil {
+			return evt.Any(key, v.Value.(EnumWrap).ValueRaw)
+		} else {
+			return evt.Str(key, v.Value.(EnumWrap).ValueString)
+		}
+	}
+	return evt.Str(key, "(err)")
+}
+
+func (v MetaValue) MarshalJSON() ([]byte, error) {
+	str, err := v.SerializeValue()
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(string(v.DataType) + ":" + str)
+}
+
+func (v *MetaValue) UnmarshalJSON(data []byte) error {
+	var str = ""
+	err := json.Unmarshal(data, &str)
+	if err != nil {
+		return err
+	}
+
+	split := strings.SplitN(str, ":", 2)
+	if len(split) != 2 {
+		return errors.New("failed to decode MetaValue: '" + str + "'")
+	}
+
+	return v.Deserialize(split[1], metaDataType(split[0]))
+}
+
+func (v MetaValue) MarshalBSON() ([]byte, error) {
+	serval, err := v.SerializeValue()
+	if err != nil {
+		return nil, Wrap(err, "failed to bson-marshal MetaValue (serialize)").Build()
+	}
+
+	// this is an kinda ugly hack - but serialization to mongodb and back can loose the correct type information....
+	bin, err := bson.Marshal(metaValueSerialization{
+		DataType: v.DataType,
+		Value:    serval,
+		Raw:      v.Value,
+	})
+	if err != nil {
+		return nil, Wrap(err, "failed to bson-marshal MetaValue (marshal)").Build()
+	}
+
+	return bin, nil
+}
+
+func (v *MetaValue) UnmarshalBSON(bytes []byte) error {
+	var serval metaValueSerialization
+	err := bson.Unmarshal(bytes, &serval)
+	if err != nil {
+		return Wrap(err, "failed to bson-unmarshal MetaValue (unmarshal)").Build()
+	}
+
+	err = v.Deserialize(serval.Value, serval.DataType)
+	if err != nil {
+		return Wrap(err, "failed to deserialize MetaValue from bson").Str("raw", serval.Value).Build()
+	}
+
+	return nil
+}
+
+func (v *MetaValue) Deserialize(value string, datatype metaDataType) error {
+	switch datatype {
+	case MDTString:
+		v.Value = value
+		v.DataType = datatype
+		return nil
+	case MDTID:
+		v.Value = deserializeIDWrap(value)
+		v.DataType = datatype
+		return nil
+	case MDTAny:
+		v.Value = deserializeAnyWrap(value)
+		v.DataType = datatype
+		return nil
+	case MDTStringPtr:
+		if len(value) <= 0 || (value[0] != '*' && value[0] != '#') {
+			return errors.New("Invalid StringPtr: " + value)
+		} else if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		} else {
+			v.Value = langext.Ptr(value[1:])
+			v.DataType = datatype
+			return nil
+		}
+	case MDTInt:
+		pv, err := strconv.ParseInt(value, 10, 0)
+		if err != nil {
+			return err
+		}
+		v.Value = int(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt8:
+		pv, err := strconv.ParseInt(value, 10, 8)
+		if err != nil {
+			return err
+		}
+		v.Value = int8(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt16:
+		pv, err := strconv.ParseInt(value, 10, 16)
+		if err != nil {
+			return err
+		}
+		v.Value = int16(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt32:
+		pv, err := strconv.ParseInt(value, 10, 32)
+		if err != nil {
+			return err
+		}
+		v.Value = int32(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt64:
+		pv, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			return err
+		}
+		v.Value = pv
+		v.DataType = datatype
+		return nil
+	case MDTFloat32:
+		pv, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			return err
+		}
+		v.Value = float32(pv)
+		v.DataType = datatype
+		return nil
+	case MDTFloat64:
+		pv, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			return err
+		}
+		v.Value = pv
+		v.DataType = datatype
+		return nil
+	case MDTBool:
+		if value == "true" {
+			v.Value = true
+			v.DataType = datatype
+			return nil
+		}
+		if value == "false" {
+			v.Value = false
+			v.DataType = datatype
+			return nil
+		}
+		return errors.New("invalid bool value: " + value)
+	case MDTBytes:
+		r, err := hex.DecodeString(value)
+		if err != nil {
+			return err
+		}
+		v.Value = r
+		v.DataType = datatype
+		return nil
+	case MDTObjectID:
+		r, err := primitive.ObjectIDFromHex(value)
+		if err != nil {
+			return err
+		}
+		v.Value = r
+		v.DataType = datatype
+		return nil
+	case MDTTime:
+		ps := strings.Split(value, "|")
+		if len(ps) != 2 {
+			return errors.New("invalid time.time: " + value)
+		}
+		p1, err := strconv.ParseInt(ps[0], 10, 64)
+		if err != nil {
+			return err
+		}
+		p2, err := strconv.ParseInt(ps[1], 10, 32)
+		if err != nil {
+			return err
+		}
+		v.Value = time.Unix(p1, p2)
+		v.DataType = datatype
+		return nil
+	case MDTDuration:
+		r, err := time.ParseDuration(value)
+		if err != nil {
+			return err
+		}
+		v.Value = r
+		v.DataType = datatype
+		return nil
+	case MDTStringArray:
+		if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		}
+		pj := make([]string, 0)
+		err := json.Unmarshal([]byte(value), &pj)
+		if err != nil {
+			return err
+		}
+		v.Value = pj
+		v.DataType = datatype
+		return nil
+	case MDTIntArray:
+		if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		}
+		pj := make([]int, 0)
+		err := json.Unmarshal([]byte(value), &pj)
+		if err != nil {
+			return err
+		}
+		v.Value = pj
+		v.DataType = datatype
+		return nil
+	case MDTInt32Array:
+		if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		}
+		pj := make([]int32, 0)
+		err := json.Unmarshal([]byte(value), &pj)
+		if err != nil {
+			return err
+		}
+		v.Value = pj
+		v.DataType = datatype
+		return nil
+	case MDTNil:
+		v.Value = nil
+		v.DataType = datatype
+		return nil
+	case MDTEnum:
+		v.Value = deserializeEnumWrap(value)
+		v.DataType = datatype
+		return nil
+	}
+	return errors.New("Unknown type: " + string(datatype))
+}
+
+func (v MetaValue) ValueString() string {
+	switch v.DataType {
+	case MDTString:
+		return v.Value.(string)
+	case MDTID:
+		return v.Value.(IDWrap).String()
+	case MDTAny:
+		return v.Value.(AnyWrap).String()
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		return langext.CoalesceString(v.Value.(*string), "<<null>>")
+	case MDTInt:
+		return strconv.Itoa(v.Value.(int))
+	case MDTInt8:
+		return strconv.FormatInt(int64(v.Value.(int8)), 10)
+	case MDTInt16:
+		return strconv.FormatInt(int64(v.Value.(int16)), 10)
+	case MDTInt32:
+		return strconv.FormatInt(int64(v.Value.(int32)), 10)
+	case MDTInt64:
+		return strconv.FormatInt(v.Value.(int64), 10)
+	case MDTFloat32:
+		return strconv.FormatFloat(float64(v.Value.(float32)), 'g', 4, 32)
+	case MDTFloat64:
+		return strconv.FormatFloat(v.Value.(float64), 'g', 4, 64)
+	case MDTBool:
+		return fmt.Sprintf("%v", v.Value.(bool))
+	case MDTBytes:
+		return hex.EncodeToString(v.Value.([]byte))
+	case MDTObjectID:
+		return v.Value.(primitive.ObjectID).Hex()
+	case MDTTime:
+		return v.Value.(time.Time).Format(time.RFC3339Nano)
+	case MDTDuration:
+		return v.Value.(time.Duration).String()
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.MarshalIndent(v.Value.([]string), "", "  ")
+		if err != nil {
+			return "(err)"
+		}
+		return string(r)
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.MarshalIndent(v.Value.([]int), "", "  ")
+		if err != nil {
+			return "(err)"
+		}
+		return string(r)
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.MarshalIndent(v.Value.([]int32), "", "  ")
+		if err != nil {
+			return "(err)"
+		}
+		return string(r)
+	case MDTNil:
+		return "<<null>>"
+	case MDTEnum:
+		return v.Value.(EnumWrap).String()
+	}
+	return "(err)"
+}
+
+// rawValueForJson returns most-of-the-time the `Value` field
+// but for some datatyes we do special processing
+// all, so we can pluck the output value in json.Marshal without any suprises
+func (v MetaValue) rawValueForJson() any {
+	if v.DataType == MDTAny {
+		if v.Value.(AnyWrap).IsNil {
+			return nil
+		}
+		if v.Value.(AnyWrap).IsError {
+			return bson.M{"@error": true}
+		}
+		jsonobj := primitive.M{}
+		jsonarr := primitive.A{}
+		if err := json.Unmarshal([]byte(v.Value.(AnyWrap).Json), &jsonobj); err == nil {
+			return jsonobj
+		} else if err := json.Unmarshal([]byte(v.Value.(AnyWrap).Json), &jsonarr); err == nil {
+			return jsonarr
+		} else {
+			return bson.M{"type": v.Value.(AnyWrap).Type, "data": v.Value.(AnyWrap).Json}
+		}
+	}
+	if v.DataType == MDTID {
+		if v.Value.(IDWrap).IsNil {
+			return nil
+		}
+		return v.Value.(IDWrap).Value
+	}
+	if v.DataType == MDTBytes {
+		return hex.EncodeToString(v.Value.([]byte))
+	}
+	if v.DataType == MDTDuration {
+		return v.Value.(time.Duration).String()
+	}
+	if v.DataType == MDTTime {
+		return v.Value.(time.Time).Format(time.RFC3339Nano)
+	}
+	if v.DataType == MDTObjectID {
+		return v.Value.(primitive.ObjectID).Hex()
+	}
+	if v.DataType == MDTNil {
+		return nil
+	}
+	if v.DataType == MDTEnum {
+		if v.Value.(EnumWrap).IsNil {
+			return nil
+		}
+		if v.Value.(EnumWrap).ValueRaw != nil {
+			return v.Value.(EnumWrap).ValueRaw
+		}
+		return v.Value.(EnumWrap).ValueString
+	}
+	if v.DataType == MDTFloat32 {
+		if math.IsNaN(float64(v.Value.(float32))) {
+			return "float64::NaN"
+		} else if math.IsInf(float64(v.Value.(float32)), +1) {
+			return "float64::+inf"
+		} else if math.IsInf(float64(v.Value.(float32)), -1) {
+			return "float64::-inf"
+		} else {
+			return v.Value
+		}
+	}
+	if v.DataType == MDTFloat64 {
+		if math.IsNaN(v.Value.(float64)) {
+			return "float64::NaN"
+		} else if math.IsInf(v.Value.(float64), +1) {
+			return "float64::+inf"
+		} else if math.IsInf(v.Value.(float64), -1) {
+			return "float64::-inf"
+		} else {
+			return v.Value
+		}
+	}
+	return v.Value
+}
+
+func (mm MetaMap) FormatOneLine(singleMaxLen int) string {
+	r := ""
+
+	i := 0
+	for key, val := range mm {
+		if i > 0 {
+			r += ", "
+		}
+
+		r += "\"" + key + "\""
+		r += ": "
+		r += "\"" + val.ShortString(singleMaxLen) + "\""
+
+		i++
+	}
+
+	return r
+}
+
+func (mm MetaMap) FormatMultiLine(indentFront string, indentKeys string, maxLenValue int) string {
+	r := ""
+
+	r += indentFront + "{" + "\n"
+	for key, val := range mm {
+		if key == "gin.body" {
+			continue
+		}
+
+		r += indentFront
+		r += indentKeys
+		r += "\"" + key + "\""
+		r += ": "
+		r += "\"" + val.ShortString(maxLenValue) + "\""
+		r += ",\n"
+	}
+	r += indentFront + "}"
+
+	return r
+}
+
+func (mm MetaMap) Any() bool {
+	return len(mm) > 0
+}
+
+func (mm MetaMap) Apply(evt *zerolog.Event, limitLen *int) *zerolog.Event {
+	for key, val := range mm {
+		evt = val.Apply(key, evt, limitLen)
+	}
+	return evt
+}
+
+func (mm MetaMap) add(key string, mdtype metaDataType, val interface{}) {
+	if _, ok := mm[key]; !ok {
+		mm[key] = MetaValue{DataType: mdtype, Value: val}
+		return
+	}
+	for i := 2; ; i++ {
+		realkey := key + "-" + strconv.Itoa(i)
+		if _, ok := mm[realkey]; !ok {
+			mm[realkey] = MetaValue{DataType: mdtype, Value: val}
+			return
+		}
+	}
+}

exerr/stacktrace.go

@@ -0,0 +1,14 @@
+package exerr
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func callername(skip int) string {
+	pc := make([]uintptr, 15)
+	n := runtime.Callers(skip+2, pc)
+	frames := runtime.CallersFrames(pc[:n])
+	frame, _ := frames.Next()
+	return fmt.Sprintf("%s:%d %s", frame.File, frame.Line, frame.Function)
+}

exerr/typeWrapper.go

@@ -0,0 +1,189 @@
+package exerr
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/rs/zerolog/log"
+	"gogs.mikescher.com/BlackForestBytes/goext/enums"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"strings"
+)
+
+//
+// These are wrapper objects, because for some metadata-types we need to serialize a bit more complex data
+// (eg thy actual type for ID objects, or the json representation for any types)
+//
+
+type IDWrap struct {
+	Type  string
+	Value string
+	IsNil bool
+}
+
+func newIDWrap(val fmt.Stringer) IDWrap {
+	t := fmt.Sprintf("%T", val)
+	arr := strings.Split(t, ".")
+	if len(arr) > 0 {
+		t = arr[len(arr)-1]
+	}
+
+	if langext.IsNil(val) {
+		return IDWrap{Type: t, Value: "", IsNil: true}
+	}
+
+	v := val.String()
+	return IDWrap{Type: t, Value: v, IsNil: false}
+}
+
+func (w IDWrap) Serialize() string {
+	if w.IsNil {
+		return "!nil" + ":" + w.Type
+	}
+	return w.Type + ":" + w.Value
+}
+
+func (w IDWrap) String() string {
+	if w.IsNil {
+		return w.Type + "<<nil>>"
+	}
+	return w.Type + "(" + w.Value + ")"
+}
+
+func deserializeIDWrap(v string) IDWrap {
+	r := strings.SplitN(v, ":", 2)
+
+	if len(r) == 2 && r[0] == "!nil" {
+		return IDWrap{Type: r[1], Value: v, IsNil: true}
+	}
+
+	if len(r) == 0 {
+		return IDWrap{}
+	} else if len(r) == 1 {
+		return IDWrap{Type: "", Value: v, IsNil: false}
+	} else {
+		return IDWrap{Type: r[0], Value: r[1], IsNil: false}
+	}
+}
+
+type AnyWrap struct {
+	Type    string
+	Json    string
+	IsError bool
+	IsNil   bool
+}
+
+func newAnyWrap(val any) (result AnyWrap) {
+	result = AnyWrap{Type: "", Json: "", IsError: true, IsNil: false} // ensure a return in case of recover()
+
+	defer func() {
+		if err := recover(); err != nil {
+			// send error should never crash our program
+			log.Error().Interface("err", err).Msg("Panic while trying to marshal anywrap ( bmerror.Interface )")
+		}
+	}()
+
+	t := fmt.Sprintf("%T", val)
+
+	if langext.IsNil(val) {
+		return AnyWrap{Type: t, Json: "", IsError: false, IsNil: true}
+	}
+
+	j, err := json.Marshal(val)
+	if err == nil {
+		return AnyWrap{Type: t, Json: string(j), IsError: false, IsNil: false}
+	} else {
+		return AnyWrap{Type: t, Json: "", IsError: true, IsNil: false}
+	}
+}
+
+func (w AnyWrap) Serialize() string {
+	if w.IsError {
+		return "ERR" + ":" + w.Type + ":" + w.Json
+	} else if w.IsNil {
+		return "NIL" + ":" + w.Type + ":" + w.Json
+	} else {
+		return "OK" + ":" + w.Type + ":" + w.Json
+	}
+}
+
+func (w AnyWrap) String() string {
+	if w.IsError {
+		return "(error)"
+	} else if w.IsNil {
+		return "(nil)"
+	} else {
+		return w.Json
+	}
+}
+
+func deserializeAnyWrap(v string) AnyWrap {
+	r := strings.SplitN(v, ":", 3)
+	if len(r) != 3 {
+		return AnyWrap{IsError: true, Type: "", Json: "", IsNil: false}
+	} else {
+		if r[0] == "OK" {
+			return AnyWrap{IsError: false, Type: r[1], Json: r[2], IsNil: false}
+		} else if r[0] == "ERR" {
+			return AnyWrap{IsError: true, Type: r[1], Json: r[2], IsNil: false}
+		} else if r[0] == "NIL" {
+			return AnyWrap{IsError: false, Type: r[1], Json: "", IsNil: true}
+		} else {
+			return AnyWrap{IsError: true, Type: "", Json: "", IsNil: false}
+		}
+	}
+}
+
+type EnumWrap struct {
+	Type        string
+	ValueString string
+	ValueRaw    enums.Enum // `ValueRaw` is lost during serialization roundtrip
+	IsNil       bool
+}
+
+func newEnumWrap(val enums.Enum) EnumWrap {
+	t := fmt.Sprintf("%T", val)
+	arr := strings.Split(t, ".")
+	if len(arr) > 0 {
+		t = arr[len(arr)-1]
+	}
+
+	if langext.IsNil(val) {
+		return EnumWrap{Type: t, ValueString: "", ValueRaw: val, IsNil: true}
+	}
+
+	if enumstr, ok := val.(enums.StringEnum); ok {
+		return EnumWrap{Type: t, ValueString: enumstr.String(), ValueRaw: val, IsNil: false}
+	}
+
+	return EnumWrap{Type: t, ValueString: fmt.Sprintf("%v", val), ValueRaw: val, IsNil: false}
+}
+
+func (w EnumWrap) Serialize() string {
+	if w.IsNil {
+		return "!nil" + ":" + w.Type
+	}
+	return w.Type + ":" + w.ValueString
+}
+
+func (w EnumWrap) String() string {
+	if w.IsNil {
+		return w.Type + "<<nil>>"
+	}
+	return "[" + w.Type + "] " + w.ValueString
+}
+
+func deserializeEnumWrap(v string) EnumWrap {
+	r := strings.SplitN(v, ":", 2)
+
+	if len(r) == 2 && r[0] == "!nil" {
+		return EnumWrap{Type: r[1], ValueString: v, ValueRaw: nil, IsNil: true}
+	}
+
+	if len(r) == 0 {
+		return EnumWrap{}
+	} else if len(r) == 1 {
+		return EnumWrap{Type: "", ValueString: v, ValueRaw: nil, IsNil: false}
+	} else {
+		return EnumWrap{Type: r[0], ValueString: r[1], ValueRaw: nil, IsNil: false}
+	}
+}

fsext/exists.go

@@ -0,0 +1,36 @@
+package fsext
+
+import "os"
+
+func PathExists(fp string) (bool, error) {
+	_, err := os.Stat(fp)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func FileExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return !stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func DirectoryExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}

ginext/appContext.go

@@ -0,0 +1,59 @@
+package ginext
+
+import (
+	"context"
+	"github.com/gin-gonic/gin"
+	"time"
+)
+
+type AppContext struct {
+	inner      context.Context
+	cancelFunc context.CancelFunc
+	cancelled  bool
+	GinContext *gin.Context
+}
+
+func CreateAppContext(g *gin.Context, innerCtx context.Context, cancelFn context.CancelFunc) *AppContext {
+	for key, value := range g.Keys {
+		innerCtx = context.WithValue(innerCtx, key, value)
+	}
+	return &AppContext{
+		inner:      innerCtx,
+		cancelFunc: cancelFn,
+		cancelled:  false,
+		GinContext: g,
+	}
+}
+
+func (ac *AppContext) Deadline() (deadline time.Time, ok bool) {
+	return ac.inner.Deadline()
+}
+
+func (ac *AppContext) Done() <-chan struct{} {
+	return ac.inner.Done()
+}
+
+func (ac *AppContext) Err() error {
+	return ac.inner.Err()
+}
+
+func (ac *AppContext) Value(key any) any {
+	return ac.inner.Value(key)
+}
+
+func (ac *AppContext) Set(key, value any) {
+	ac.inner = context.WithValue(ac.inner, key, value)
+}
+
+func (ac *AppContext) Cancel() {
+	ac.cancelled = true
+	ac.cancelFunc()
+}
+
+func (ac *AppContext) RequestURI() string {
+	if ac.GinContext != nil && ac.GinContext.Request != nil {
+		return ac.GinContext.Request.Method + " :: " + ac.GinContext.Request.RequestURI
+	} else {
+		return ""
+	}
+}

ginext/commonHandler.go

@@ -0,0 +1,23 @@
+package ginext
+
+import (
+	"net/http"
+)
+
+func RedirectFound(newuri string) WHandlerFunc {
+	return func(pctx PreContext) HTTPResponse {
+		return Redirect(http.StatusFound, newuri)
+	}
+}
+
+func RedirectTemporary(newuri string) WHandlerFunc {
+	return func(pctx PreContext) HTTPResponse {
+		return Redirect(http.StatusTemporaryRedirect, newuri)
+	}
+}
+
+func RedirectPermanent(newuri string) WHandlerFunc {
+	return func(pctx PreContext) HTTPResponse {
+		return Redirect(http.StatusPermanentRedirect, newuri)
+	}
+}

ginext/commonMiddlewares.go

@@ -0,0 +1,12 @@
+package ginext
+
+import (
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+)
+
+func BodyBuffer(g *gin.Context) {
+	if g.Request.Body != nil {
+		g.Request.Body = dataext.NewBufferedReadCloser(g.Request.Body)
+	}
+}

ginext/cors.go

@@ -0,0 +1,21 @@
+package ginext
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func CorsMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "OPTIONS, GET, POST, PUT, PATCH, DELETE, COUNT")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(http.StatusOK)
+		} else {
+			c.Next()
+		}
+	}
+}

ginext/engine.go

@@ -0,0 +1,195 @@
+package ginext
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type GinWrapper struct {
+	engine          *gin.Engine
+	suppressGinLogs bool
+
+	allowCors             bool
+	ginDebug              bool
+	bufferBody            bool
+	requestTimeout        time.Duration
+	listenerBeforeRequest []func(g *gin.Context)
+	listenerAfterRequest  []func(g *gin.Context, resp HTTPResponse)
+
+	routeSpecs []ginRouteSpec
+}
+
+type ginRouteSpec struct {
+	Method      string
+	URL         string
+	Middlewares []string
+	Handler     string
+}
+
+type Options struct {
+	AllowCors             *bool                                     // Add cors handler to allow all CORS requests on the default http methods
+	GinDebug              *bool                                     // Set gin.debug to true (adds more logs)
+	BufferBody            *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
+	Timeout               *time.Duration                            // The default handler timeout
+	ListenerBeforeRequest []func(g *gin.Context)                    // Register listener that are called before the handler method
+	ListenerAfterRequest  []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+}
+
+// NewEngine creates a new (wrapped) ginEngine
+func NewEngine(opt Options) *GinWrapper {
+	engine := gin.New()
+
+	wrapper := &GinWrapper{
+		engine:                engine,
+		suppressGinLogs:       false,
+		allowCors:             langext.Coalesce(opt.AllowCors, false),
+		ginDebug:              langext.Coalesce(opt.GinDebug, true),
+		bufferBody:            langext.Coalesce(opt.BufferBody, false),
+		requestTimeout:        langext.Coalesce(opt.Timeout, 24*time.Hour),
+		listenerBeforeRequest: opt.ListenerBeforeRequest,
+		listenerAfterRequest:  opt.ListenerAfterRequest,
+	}
+
+	engine.RedirectFixedPath = false
+	engine.RedirectTrailingSlash = false
+
+	if wrapper.allowCors {
+		engine.Use(CorsMiddleware())
+	}
+
+	// do not debug-print routes
+	gin.DebugPrintRouteFunc = func(_, _, _ string, _ int) {}
+
+	if !wrapper.ginDebug {
+		gin.SetMode(gin.ReleaseMode)
+
+		ginlogger := gin.Logger()
+		engine.Use(func(context *gin.Context) {
+			if !wrapper.suppressGinLogs {
+				ginlogger(context)
+			}
+		})
+	} else {
+		gin.SetMode(gin.DebugMode)
+	}
+
+	return wrapper
+}
+
+func (w *GinWrapper) ListenAndServeHTTP(addr string, postInit func(port string)) (chan error, *http.Server) {
+
+	w.DebugPrintRoutes()
+
+	httpserver := &http.Server{
+		Addr:    addr,
+		Handler: w.engine,
+	}
+
+	errChan := make(chan error)
+
+	go func() {
+
+		ln, err := net.Listen("tcp", httpserver.Addr)
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		_, port, err := net.SplitHostPort(ln.Addr().String())
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		log.Info().Str("address", httpserver.Addr).Msg("HTTP-Server started on http://localhost:" + port)
+
+		if postInit != nil {
+			postInit(port) // the net.Listener a few lines above is at this point actually already buffering requests
+		}
+
+		errChan <- httpserver.Serve(ln)
+	}()
+
+	return errChan, httpserver
+}
+
+func (w *GinWrapper) DebugPrintRoutes() {
+	if !w.ginDebug {
+		return
+	}
+
+	lines := make([][4]string, 0)
+
+	pad := [4]int{0, 0, 0, 0}
+
+	for _, spec := range w.routeSpecs {
+
+		line := [4]string{
+			spec.Method,
+			spec.URL,
+			strings.Join(langext.ArrMap(spec.Middlewares, w.cleanMiddlewareName), " -> "),
+			w.cleanMiddlewareName(spec.Handler),
+		}
+
+		lines = append(lines, line)
+
+		pad[0] = mathext.Max(pad[0], len(line[0]))
+		pad[1] = mathext.Max(pad[1], len(line[1]))
+		pad[2] = mathext.Max(pad[2], len(line[2]))
+		pad[3] = mathext.Max(pad[3], len(line[3]))
+	}
+
+	fmt.Printf("Gin-Routes:\n")
+	fmt.Printf("{\n")
+	for _, line := range lines {
+
+		fmt.Printf(" %s  %s  -->  %s  -->  %s\n",
+			langext.StrPadRight("["+line[0]+"]", " ", pad[0]+2),
+			langext.StrPadRight(line[1], " ", pad[1]),
+			langext.StrPadRight(line[2], " ", pad[2]),
+			langext.StrPadRight(line[3], " ", pad[3]))
+	}
+	fmt.Printf("}\n")
+}
+
+func (w *GinWrapper) cleanMiddlewareName(fname string) string {
+
+	funcSuffix := rext.W(regexp.MustCompile(`\.func[0-9]+(?:\.[0-9]+)*$`))
+	if match, ok := funcSuffix.MatchFirst(fname); ok {
+		fname = fname[:len(fname)-match.FullMatch().Length()]
+	}
+
+	if strings.HasSuffix(fname, ".(*GinRoutesWrapper).WithJSONFilter") {
+		fname = "[JSONFilter]"
+	}
+
+	if fname == "ginext.BodyBuffer" {
+		fname = "[BodyBuffer]"
+	}
+
+	skipPrefixes := []string{"api.(*Handler).", "api.", "ginext.", "handler.", "admin-app.", "employee-app.", "employer-app."}
+	for _, pfx := range skipPrefixes {
+		if strings.HasPrefix(fname, pfx) {
+			fname = fname[len(pfx):]
+		}
+	}
+
+	return fname
+}
+
+// ServeHTTP only used for unit tests
+func (w *GinWrapper) ServeHTTP(req *http.Request) *httptest.ResponseRecorder {
+	respRec := httptest.NewRecorder()
+	w.engine.ServeHTTP(respRec, req)
+	return respRec
+}

ginext/funcWrapper.go

@@ -0,0 +1,60 @@
+package ginext
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+)
+
+type WHandlerFunc func(PreContext) HTTPResponse
+
+func Wrap(w *GinWrapper, fn WHandlerFunc) gin.HandlerFunc {
+
+	return func(g *gin.Context) {
+
+		reqctx := g.Request.Context()
+
+		pctx := PreContext{
+			wrapper:        w,
+			ginCtx:         g,
+			persistantData: &preContextData{},
+		}
+
+		for _, lstr := range w.listenerBeforeRequest {
+			lstr(g)
+		}
+
+		wrap, stackTrace, panicObj := callPanicSafe(fn, pctx)
+		if panicObj != nil {
+
+			fmt.Printf("\n======== ======== STACKTRACE ======== ========\n%s\n======== ======== ======== ========\n\n", stackTrace)
+
+			err := exerr.
+				New(exerr.TypePanic, "Panic occured (in gin handler)").
+				Any("panicObj", panicObj).
+				Str("trace", stackTrace).
+				Build()
+
+			wrap = Error(err)
+		}
+
+		if g.Writer.Written() {
+			panic("Writing in WrapperFunc is not supported")
+		}
+
+		if pctx.persistantData.sessionObj != nil {
+			err := pctx.persistantData.sessionObj.Finish(reqctx, wrap)
+			if err != nil {
+				wrap = Error(exerr.Wrap(err, "Failed to finish session").Any("originalResponse", wrap).Build())
+			}
+		}
+
+		for _, lstr := range w.listenerAfterRequest {
+			lstr(g, wrap)
+		}
+
+		if reqctx.Err() == nil {
+			wrap.Write(g)
+		}
+	}
+}

ginext/preContext.go

@@ -0,0 +1,190 @@
+package ginext
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/gin-gonic/gin/binding"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"io"
+	"runtime/debug"
+	"time"
+)
+
+type PreContext struct {
+	ginCtx         *gin.Context
+	wrapper        *GinWrapper
+	uri            any
+	query          any
+	body           any
+	rawbody        *[]byte
+	form           any
+	header         any
+	timeout        *time.Duration
+	persistantData *preContextData // must be a ptr, so that we can get the values back in out Wrap func
+}
+
+type preContextData struct {
+	sessionObj SessionObject
+}
+
+func (pctx *PreContext) URI(uri any) *PreContext {
+	pctx.uri = uri
+	return pctx
+}
+
+func (pctx *PreContext) Query(query any) *PreContext {
+	pctx.query = query
+	return pctx
+}
+
+func (pctx *PreContext) Body(body any) *PreContext {
+	pctx.body = body
+	return pctx
+}
+
+func (pctx *PreContext) RawBody(rawbody *[]byte) *PreContext {
+	pctx.rawbody = rawbody
+	return pctx
+}
+
+func (pctx *PreContext) Form(form any) *PreContext {
+	pctx.form = form
+	return pctx
+}
+
+func (pctx *PreContext) Header(header any) *PreContext {
+	pctx.header = header
+	return pctx
+}
+
+func (pctx *PreContext) WithTimeout(to time.Duration) *PreContext {
+	pctx.timeout = &to
+	return pctx
+}
+
+func (pctx *PreContext) WithSession(sessionObj SessionObject) *PreContext {
+	pctx.persistantData.sessionObj = sessionObj
+	return pctx
+}
+
+func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
+	if pctx.uri != nil {
+		if err := pctx.ginCtx.ShouldBindUri(pctx.uri); err != nil {
+			err = exerr.Wrap(err, "Failed to read uri").
+				WithType(exerr.TypeBindFailURI).
+				Str("struct_type", fmt.Sprintf("%T", pctx.uri)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.query != nil {
+		if err := pctx.ginCtx.ShouldBindQuery(pctx.query); err != nil {
+			err = exerr.Wrap(err, "Failed to read query").
+				WithType(exerr.TypeBindFailQuery).
+				Str("struct_type", fmt.Sprintf("%T", pctx.query)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.body != nil {
+		if pctx.ginCtx.ContentType() == "application/json" {
+			if err := pctx.ginCtx.ShouldBindJSON(pctx.body); err != nil {
+				err = exerr.Wrap(err, "Failed to read json-body").
+					WithType(exerr.TypeBindFailJSON).
+					Str("struct_type", fmt.Sprintf("%T", pctx.body)).
+					Build()
+				return nil, nil, langext.Ptr(Error(err))
+			}
+		} else {
+			err := exerr.New(exerr.TypeBindFailJSON, "missing JSON body").
+				Str("struct_type", fmt.Sprintf("%T", pctx.body)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.rawbody != nil {
+		if brc, ok := pctx.ginCtx.Request.Body.(dataext.BufferedReadCloser); ok {
+			v, err := brc.BufferedAll()
+			if err != nil {
+				return nil, nil, langext.Ptr(Error(err))
+			}
+			*pctx.rawbody = v
+		} else {
+			buf := &bytes.Buffer{}
+			_, err := io.Copy(buf, pctx.ginCtx.Request.Body)
+			if err != nil {
+				return nil, nil, langext.Ptr(Error(err))
+			}
+			*pctx.rawbody = buf.Bytes()
+		}
+	}
+
+	if pctx.form != nil {
+		if pctx.ginCtx.ContentType() == "multipart/form-data" {
+			if err := pctx.ginCtx.ShouldBindWith(pctx.form, binding.Form); err != nil {
+				err = exerr.Wrap(err, "Failed to read multipart-form").
+					WithType(exerr.TypeBindFailFormData).
+					Str("struct_type", fmt.Sprintf("%T", pctx.form)).
+					Build()
+				return nil, nil, langext.Ptr(Error(err))
+			}
+		} else if pctx.ginCtx.ContentType() == "application/x-www-form-urlencoded" {
+			if err := pctx.ginCtx.ShouldBindWith(pctx.form, binding.Form); err != nil {
+				err = exerr.Wrap(err, "Failed to read urlencoded-form").
+					WithType(exerr.TypeBindFailFormData).
+					Str("struct_type", fmt.Sprintf("%T", pctx.form)).
+					Build()
+				return nil, nil, langext.Ptr(Error(err))
+			}
+		} else {
+			err := exerr.New(exerr.TypeBindFailFormData, "missing form body").
+				Str("struct_type", fmt.Sprintf("%T", pctx.form)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.header != nil {
+		if err := pctx.ginCtx.ShouldBindHeader(pctx.header); err != nil {
+			err = exerr.Wrap(err, "Failed to read header").
+				WithType(exerr.TypeBindFailHeader).
+				Str("struct_type", fmt.Sprintf("%T", pctx.query)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	ictx, cancel := context.WithTimeout(context.Background(), langext.Coalesce(pctx.timeout, pctx.wrapper.requestTimeout))
+
+	actx := CreateAppContext(pctx.ginCtx, ictx, cancel)
+
+	if pctx.persistantData.sessionObj != nil {
+		err := pctx.persistantData.sessionObj.Init(pctx.ginCtx, actx)
+		if err != nil {
+			actx.Cancel()
+			return nil, nil, langext.Ptr(Error(exerr.Wrap(err, "Failed to init session").Build()))
+		}
+	}
+
+	return actx, pctx.ginCtx, nil
+}
+
+func callPanicSafe(fn WHandlerFunc, pctx PreContext) (res HTTPResponse, stackTrace string, panicObj any) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			res = nil
+			stackTrace = string(debug.Stack())
+			panicObj = rec
+		}
+	}()
+
+	res = fn(pctx)
+	return res, "", nil
+}

ginext/response.go

@@ -0,0 +1,502 @@
+package ginext
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"os"
+)
+
+type cookieval struct {
+	name     string
+	value    string
+	maxAge   int
+	path     string
+	domain   string
+	secure   bool
+	httpOnly bool
+}
+
+type headerval struct {
+	Key string
+	Val string
+}
+
+type HTTPResponse interface {
+	Write(g *gin.Context)
+	WithHeader(k string, v string) HTTPResponse
+	WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse
+	IsSuccess() bool
+}
+
+type InspectableHTTPResponse interface {
+	HTTPResponse
+
+	Statuscode() int
+	BodyString(g *gin.Context) *string
+	ContentType() string
+	Headers() []string
+}
+
+type jsonHTTPResponse struct {
+	statusCode int
+	data       any
+	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j jsonHTTPResponse) jsonRenderer(g *gin.Context) json.GoJsonRender {
+	var f *string
+	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
+		f = &jsonfilter
+	}
+	return json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f}
+}
+
+func (j jsonHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.Render(j.statusCode, j.jsonRenderer(g))
+}
+
+func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j jsonHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j jsonHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j jsonHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j jsonHTTPResponse) BodyString(g *gin.Context) *string {
+	if str, err := j.jsonRenderer(g).RenderString(); err == nil {
+		return &str
+	} else {
+		return nil
+	}
+}
+
+func (j jsonHTTPResponse) ContentType() string {
+	return "application/json"
+}
+
+func (j jsonHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type emptyHTTPResponse struct {
+	statusCode int
+	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j emptyHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.Status(j.statusCode)
+}
+
+func (j emptyHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j emptyHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j emptyHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j emptyHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j emptyHTTPResponse) BodyString(*gin.Context) *string {
+	return nil
+}
+
+func (j emptyHTTPResponse) ContentType() string {
+	return ""
+}
+
+func (j emptyHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type textHTTPResponse struct {
+	statusCode int
+	data       string
+	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j textHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.String(j.statusCode, "%s", j.data)
+}
+
+func (j textHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j textHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j textHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j textHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j textHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(j.data)
+}
+
+func (j textHTTPResponse) ContentType() string {
+	return "text/plain"
+}
+
+func (j textHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type dataHTTPResponse struct {
+	statusCode  int
+	data        []byte
+	contentType string
+	headers     []headerval
+	cookies     []cookieval
+}
+
+func (j dataHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.Data(j.statusCode, j.contentType, j.data)
+}
+
+func (j dataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j dataHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j dataHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j dataHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j dataHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(string(j.data))
+}
+
+func (j dataHTTPResponse) ContentType() string {
+	return j.contentType
+}
+
+func (j dataHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type fileHTTPResponse struct {
+	mimetype string
+	filepath string
+	filename *string
+	headers  []headerval
+	cookies  []cookieval
+}
+
+func (j fileHTTPResponse) Write(g *gin.Context) {
+	g.Header("Content-Type", j.mimetype) // if we don't set it here gin does weird file-sniffing later...
+	if j.filename != nil {
+		g.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", *j.filename))
+
+	}
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.File(j.filepath)
+}
+
+func (j fileHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j fileHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j fileHTTPResponse) IsSuccess() bool {
+	return true
+}
+
+func (j fileHTTPResponse) Statuscode() int {
+	return 200
+}
+
+func (j fileHTTPResponse) BodyString(*gin.Context) *string {
+	data, err := os.ReadFile(j.filepath)
+	if err != nil {
+		return nil
+	}
+	return langext.Ptr(string(data))
+}
+
+func (j fileHTTPResponse) ContentType() string {
+	return j.mimetype
+}
+
+func (j fileHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type downloadDataHTTPResponse struct {
+	statusCode int
+	mimetype   string
+	data       []byte
+	filename   *string
+	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j downloadDataHTTPResponse) Write(g *gin.Context) {
+	g.Header("Content-Type", j.mimetype) // if we don't set it here gin does weird file-sniffing later...
+	if j.filename != nil {
+		g.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", *j.filename))
+	}
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.Data(j.statusCode, j.mimetype, j.data)
+}
+
+func (j downloadDataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j downloadDataHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j downloadDataHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j downloadDataHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j downloadDataHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(string(j.data))
+}
+
+func (j downloadDataHTTPResponse) ContentType() string {
+	return j.mimetype
+}
+
+func (j downloadDataHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type redirectHTTPResponse struct {
+	statusCode int
+	url        string
+	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j redirectHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	g.Redirect(j.statusCode, j.url)
+}
+
+func (j redirectHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j redirectHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j redirectHTTPResponse) IsSuccess() bool {
+	return j.statusCode >= 200 && j.statusCode <= 399
+}
+
+func (j redirectHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j redirectHTTPResponse) BodyString(*gin.Context) *string {
+	return nil
+}
+
+func (j redirectHTTPResponse) ContentType() string {
+	return ""
+}
+
+func (j redirectHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+type jsonAPIErrResponse struct {
+	err     *exerr.ExErr
+	headers []headerval
+	cookies []cookieval
+}
+
+func (j jsonAPIErrResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
+	j.err.Output(g)
+
+	j.err.CallListener(exerr.MethodOutput)
+}
+
+func (j jsonAPIErrResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func (j jsonAPIErrResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
+func (j jsonAPIErrResponse) IsSuccess() bool {
+	return false
+}
+
+func (j jsonAPIErrResponse) Statuscode() int {
+	return langext.Coalesce(j.err.RecursiveStatuscode(), 0)
+}
+
+func (j jsonAPIErrResponse) BodyString(*gin.Context) *string {
+	if str, err := j.err.ToDefaultAPIJson(); err == nil {
+		return &str
+	} else {
+		return nil
+	}
+}
+
+func (j jsonAPIErrResponse) ContentType() string {
+	return "application/json"
+}
+
+func (j jsonAPIErrResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+func (j jsonAPIErrResponse) Unwrap() error {
+	return j.err
+}
+
+func Status(sc int) HTTPResponse {
+	return &emptyHTTPResponse{statusCode: sc}
+}
+
+func JSON(sc int, data any) HTTPResponse {
+	return &jsonHTTPResponse{statusCode: sc, data: data}
+}
+
+func Data(sc int, contentType string, data []byte) HTTPResponse {
+	return &dataHTTPResponse{statusCode: sc, contentType: contentType, data: data}
+}
+
+func Text(sc int, data string) HTTPResponse {
+	return &textHTTPResponse{statusCode: sc, data: data}
+}
+
+func File(mimetype string, filepath string) HTTPResponse {
+	return &fileHTTPResponse{mimetype: mimetype, filepath: filepath}
+}
+
+func Download(mimetype string, filepath string, filename string) HTTPResponse {
+	return &fileHTTPResponse{mimetype: mimetype, filepath: filepath, filename: &filename}
+}
+
+func DownloadData(status int, mimetype string, filename string, data []byte) HTTPResponse {
+	return &downloadDataHTTPResponse{statusCode: status, mimetype: mimetype, data: data, filename: &filename}
+}
+
+func Redirect(sc int, newURL string) HTTPResponse {
+	return &redirectHTTPResponse{statusCode: sc, url: newURL}
+}
+
+func Error(e error) HTTPResponse {
+	return &jsonAPIErrResponse{
+		err: exerr.FromError(e),
+	}
+}
+
+func ErrWrap(e error, errorType exerr.ErrorType, msg string) HTTPResponse {
+	return &jsonAPIErrResponse{
+		err: exerr.FromError(exerr.Wrap(e, msg).WithType(errorType).Build()),
+	}
+}
+
+func NotImplemented() HTTPResponse {
+	return Error(exerr.New(exerr.TypeNotImplemented, "").Build())
+}

ginext/routes.go

@@ -0,0 +1,218 @@
+package ginext
+
+import (
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"net/http"
+	"path"
+	"reflect"
+	"runtime"
+	"strings"
+)
+
+var anyMethods = []string{
+	http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch,
+	http.MethodHead, http.MethodOptions, http.MethodDelete, http.MethodConnect,
+	http.MethodTrace,
+}
+
+type GinRoutesWrapper struct {
+	wrapper        *GinWrapper
+	routes         gin.IRouter
+	absPath        string
+	defaultHandler []gin.HandlerFunc
+}
+
+type GinRouteBuilder struct {
+	routes *GinRoutesWrapper
+
+	method   string
+	relPath  string
+	absPath  string
+	handlers []gin.HandlerFunc
+}
+
+func (w *GinWrapper) Routes() *GinRoutesWrapper {
+	return &GinRoutesWrapper{
+		wrapper:        w,
+		routes:         w.engine,
+		absPath:        "",
+		defaultHandler: make([]gin.HandlerFunc, 0),
+	}
+}
+
+func (w *GinRoutesWrapper) Group(relativePath string) *GinRoutesWrapper {
+	return &GinRoutesWrapper{
+		wrapper:        w.wrapper,
+		routes:         w.routes.Group(relativePath),
+		defaultHandler: langext.ArrCopy(w.defaultHandler),
+		absPath:        joinPaths(w.absPath, relativePath),
+	}
+}
+
+func (w *GinRoutesWrapper) Use(middleware ...gin.HandlerFunc) *GinRoutesWrapper {
+	defHandler := langext.ArrCopy(w.defaultHandler)
+	defHandler = append(defHandler, middleware...)
+	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+}
+
+func (w *GinRoutesWrapper) WithJSONFilter(filter string) *GinRoutesWrapper {
+	defHandler := langext.ArrCopy(w.defaultHandler)
+	defHandler = append(defHandler, func(g *gin.Context) {
+		g.Set("goext.jsonfilter", filter)
+	})
+	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+}
+
+func (w *GinRoutesWrapper) GET(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodGet, relativePath)
+}
+
+func (w *GinRoutesWrapper) POST(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodPost, relativePath)
+}
+
+func (w *GinRoutesWrapper) DELETE(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodDelete, relativePath)
+}
+
+func (w *GinRoutesWrapper) PATCH(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodPatch, relativePath)
+}
+
+func (w *GinRoutesWrapper) PUT(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodPut, relativePath)
+}
+
+func (w *GinRoutesWrapper) OPTIONS(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodOptions, relativePath)
+}
+
+func (w *GinRoutesWrapper) HEAD(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodHead, relativePath)
+}
+
+func (w *GinRoutesWrapper) COUNT(relativePath string) *GinRouteBuilder {
+	return w._route("COUNT", relativePath)
+}
+
+func (w *GinRoutesWrapper) Any(relativePath string) *GinRouteBuilder {
+	return w._route("*", relativePath)
+}
+
+func (w *GinRoutesWrapper) _route(method string, relativePath string) *GinRouteBuilder {
+	return &GinRouteBuilder{
+		routes:   w,
+		method:   method,
+		relPath:  relativePath,
+		absPath:  joinPaths(w.absPath, relativePath),
+		handlers: langext.ArrCopy(w.defaultHandler),
+	}
+}
+
+func (w *GinRouteBuilder) Use(middleware ...gin.HandlerFunc) *GinRouteBuilder {
+	w.handlers = append(w.handlers, middleware...)
+	return w
+}
+
+func (w *GinRouteBuilder) WithJSONFilter(filter string) *GinRouteBuilder {
+	w.handlers = append(w.handlers, func(g *gin.Context) {
+		g.Set("goext.jsonfilter", filter)
+	})
+	return w
+}
+
+func (w *GinRouteBuilder) Handle(handler WHandlerFunc) {
+
+	if w.routes.wrapper.bufferBody {
+		arr := make([]gin.HandlerFunc, 0, len(w.handlers)+1)
+		arr = append(arr, BodyBuffer)
+		arr = append(arr, w.handlers...)
+		w.handlers = arr
+	}
+
+	middlewareNames := langext.ArrMap(w.handlers, func(v gin.HandlerFunc) string { return nameOfFunction(v) })
+	handlerName := nameOfFunction(handler)
+
+	w.handlers = append(w.handlers, Wrap(w.routes.wrapper, handler))
+
+	methodName := w.method
+
+	if w.method == "*" {
+		methodName = "ANY"
+		for _, method := range anyMethods {
+			w.routes.routes.Handle(method, w.relPath, w.handlers...)
+		}
+	} else {
+		w.routes.routes.Handle(w.method, w.relPath, w.handlers...)
+	}
+
+	w.routes.wrapper.routeSpecs = append(w.routes.wrapper.routeSpecs, ginRouteSpec{
+		Method:      methodName,
+		URL:         w.absPath,
+		Middlewares: middlewareNames,
+		Handler:     handlerName,
+	})
+}
+
+func (w *GinWrapper) NoRoute(handler WHandlerFunc) {
+
+	handlers := make([]gin.HandlerFunc, 0)
+
+	if w.bufferBody {
+		handlers = append(handlers, BodyBuffer)
+	}
+
+	middlewareNames := langext.ArrMap(handlers, func(v gin.HandlerFunc) string { return nameOfFunction(v) })
+	handlerName := nameOfFunction(handler)
+
+	handlers = append(handlers, Wrap(w, handler))
+
+	w.engine.NoRoute(handlers...)
+
+	w.routeSpecs = append(w.routeSpecs, ginRouteSpec{
+		Method:      "ANY",
+		URL:         "[NO_ROUTE]",
+		Middlewares: middlewareNames,
+		Handler:     handlerName,
+	})
+}
+
+func nameOfFunction(f any) string {
+
+	fname := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+
+	split := strings.Split(fname, "/")
+	if len(split) == 0 {
+		return ""
+	}
+
+	fname = split[len(split)-1]
+
+	// https://stackoverflow.com/a/32925345/1761622
+	if strings.HasSuffix(fname, "-fm") {
+		fname = fname[:len(fname)-len("-fm")]
+	}
+
+	return fname
+}
+
+// joinPaths is copied verbatim from gin@v1.9.1/gin.go
+func joinPaths(absolutePath, relativePath string) string {
+	if relativePath == "" {
+		return absolutePath
+	}
+
+	finalPath := path.Join(absolutePath, relativePath)
+	if lastChar(relativePath) == '/' && lastChar(finalPath) != '/' {
+		return finalPath + "/"
+	}
+	return finalPath
+}
+
+func lastChar(str string) uint8 {
+	if str == "" {
+		panic("The length of the string can't be 0")
+	}
+	return str[len(str)-1]
+}

ginext/session.go

@@ -0,0 +1,11 @@
+package ginext
+
+import (
+	"context"
+	"github.com/gin-gonic/gin"
+)
+
+type SessionObject interface {
+	Init(g *gin.Context, ctx *AppContext) error
+	Finish(ctx context.Context, resp HTTPResponse) error
+}

go.mod

@@ -1,13 +1,57 @@
 module gogs.mikescher.com/BlackForestBytes/goext
 
-go 1.19
+go 1.21
 
 require (
-	golang.org/x/sys v0.3.0
-	golang.org/x/term v0.3.0
+	github.com/gin-gonic/gin v1.9.1
+	github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.-
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/rs/xid v1.5.0
+	github.com/rs/zerolog v1.32.0
+	go.mongodb.org/mongo-driver v1.13.1
+	golang.org/x/crypto v0.19.0
+	golang.org/x/sys v0.17.0
+	golang.org/x/term v0.17.0
 )
 
 require (
-	github.com/jmoiron/sqlx v1.3.5 // indirect
-	golang.org/x/crypto v0.4.0 // indirect
+	github.com/bytedance/sonic v1.10.2 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
+	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.17.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.37.6 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/sqlite v1.28.0 // indirect
 )

go.sum

@@ -1,15 +1,210 @@
+github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
+github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
+github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
+github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
+github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
+github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
+github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
+github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
+github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
+github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
+github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74=
+github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
+github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
-golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
+github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
+github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
+go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
+golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw=
+modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
+modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

goextVersion.go

@@ -0,0 +1,5 @@
+package goext
+
+const GoextVersion = "0.0.386"
+
+const GoextVersionTimestamp = "2024-02-09T15:58:21+0100"

gojson/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

gojson/README.md

@@ -0,0 +1,12 @@
+
+
+JSON serializer which serializes nil-Arrays as `[]` and nil-maps als `{}`.
+
+Idea from: https://github.com/homelight/json
+
+Forked from https://github.com/golang/go/tree/547e8e22fe565d65d1fd4d6e71436a5a855447b0/src/encoding/json ( tag go1.20.2 )
+
+Added:
+
+ - `MarshalSafeCollections()` method
+ - `Encoder.nilSafeSlices` and `Encoder.nilSafeMaps` fields

gojson/example_marshaling_test.go

@@ -0,0 +1,73 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+)
+
+type Animal int
+
+const (
+	Unknown Animal = iota
+	Gopher
+	Zebra
+)
+
+func (a *Animal) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	switch strings.ToLower(s) {
+	default:
+		*a = Unknown
+	case "gopher":
+		*a = Gopher
+	case "zebra":
+		*a = Zebra
+	}
+
+	return nil
+}
+
+func (a Animal) MarshalJSON() ([]byte, error) {
+	var s string
+	switch a {
+	default:
+		s = "unknown"
+	case Gopher:
+		s = "gopher"
+	case Zebra:
+		s = "zebra"
+	}
+
+	return json.Marshal(s)
+}
+
+func Example_customMarshalJSON() {
+	blob := `["gopher","armadillo","zebra","unknown","gopher","bee","gopher","zebra"]`
+	var zoo []Animal
+	if err := json.Unmarshal([]byte(blob), &zoo); err != nil {
+		log.Fatal(err)
+	}
+
+	census := make(map[Animal]int)
+	for _, animal := range zoo {
+		census[animal] += 1
+	}
+
+	fmt.Printf("Zoo Census:\n* Gophers: %d\n* Zebras:  %d\n* Unknown: %d\n",
+		census[Gopher], census[Zebra], census[Unknown])
+
+	// Output:
+	// Zoo Census:
+	// * Gophers: 3
+	// * Zebras:  2
+	// * Unknown: 3
+}

gojson/example_test.go

@@ -0,0 +1,310 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func ExampleMarshal() {
+	type ColorGroup struct {
+		ID     int
+		Name   string
+		Colors []string
+	}
+	group := ColorGroup{
+		ID:     1,
+		Name:   "Reds",
+		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
+	}
+	b, err := json.Marshal(group)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+	// Output:
+	// {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]}
+}
+
+func ExampleUnmarshal() {
+	var jsonBlob = []byte(`[
+	{"Name": "Platypus", "Order": "Monotremata"},
+	{"Name": "Quoll",    "Order": "Dasyuromorphia"}
+]`)
+	type Animal struct {
+		Name  string
+		Order string
+	}
+	var animals []Animal
+	err := json.Unmarshal(jsonBlob, &animals)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	fmt.Printf("%+v", animals)
+	// Output:
+	// [{Name:Platypus Order:Monotremata} {Name:Quoll Order:Dasyuromorphia}]
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder() {
+	const jsonStream = `
+	{"Name": "Ed", "Text": "Knock knock."}
+	{"Name": "Sam", "Text": "Who's there?"}
+	{"Name": "Ed", "Text": "Go fmt."}
+	{"Name": "Sam", "Text": "Go fmt who?"}
+	{"Name": "Ed", "Text": "Go fmt yourself!"}
+`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		var m Message
+		if err := dec.Decode(&m); err == io.EOF {
+			break
+		} else if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%s: %s\n", m.Name, m.Text)
+	}
+	// Output:
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder_Token() {
+	const jsonStream = `
+	{"Message": "Hello", "Array": [1, 2, 3], "Null": null, "Number": 1.234}
+`
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		t, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%T: %v", t, t)
+		if dec.More() {
+			fmt.Printf(" (more)")
+		}
+		fmt.Printf("\n")
+	}
+	// Output:
+	// json.Delim: { (more)
+	// string: Message (more)
+	// string: Hello (more)
+	// string: Array (more)
+	// json.Delim: [ (more)
+	// float64: 1 (more)
+	// float64: 2 (more)
+	// float64: 3
+	// json.Delim: ] (more)
+	// string: Null (more)
+	// <nil>: <nil> (more)
+	// string: Number (more)
+	// float64: 1.234
+	// json.Delim: }
+}
+
+// This example uses a Decoder to decode a streaming array of JSON objects.
+func ExampleDecoder_Decode_stream() {
+	const jsonStream = `
+	[
+		{"Name": "Ed", "Text": "Knock knock."},
+		{"Name": "Sam", "Text": "Who's there?"},
+		{"Name": "Ed", "Text": "Go fmt."},
+		{"Name": "Sam", "Text": "Go fmt who?"},
+		{"Name": "Ed", "Text": "Go fmt yourself!"}
+	]
+`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+
+	// read open bracket
+	t, err := dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// while the array contains values
+	for dec.More() {
+		var m Message
+		// decode an array value (Message)
+		err := dec.Decode(&m)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("%v: %v\n", m.Name, m.Text)
+	}
+
+	// read closing bracket
+	t, err = dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// Output:
+	// json.Delim: [
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+	// json.Delim: ]
+}
+
+// This example uses RawMessage to delay parsing part of a JSON message.
+func ExampleRawMessage_unmarshal() {
+	type Color struct {
+		Space string
+		Point json.RawMessage // delay parsing until we know the color space
+	}
+	type RGB struct {
+		R uint8
+		G uint8
+		B uint8
+	}
+	type YCbCr struct {
+		Y  uint8
+		Cb int8
+		Cr int8
+	}
+
+	var j = []byte(`[
+	{"Space": "YCbCr", "Point": {"Y": 255, "Cb": 0, "Cr": -10}},
+	{"Space": "RGB",   "Point": {"R": 98, "G": 218, "B": 255}}
+]`)
+	var colors []Color
+	err := json.Unmarshal(j, &colors)
+	if err != nil {
+		log.Fatalln("error:", err)
+	}
+
+	for _, c := range colors {
+		var dst any
+		switch c.Space {
+		case "RGB":
+			dst = new(RGB)
+		case "YCbCr":
+			dst = new(YCbCr)
+		}
+		err := json.Unmarshal(c.Point, dst)
+		if err != nil {
+			log.Fatalln("error:", err)
+		}
+		fmt.Println(c.Space, dst)
+	}
+	// Output:
+	// YCbCr &{255 0 -10}
+	// RGB &{98 218 255}
+}
+
+// This example uses RawMessage to use a precomputed JSON during marshal.
+func ExampleRawMessage_marshal() {
+	h := json.RawMessage(`{"precomputed": true}`)
+
+	c := struct {
+		Header *json.RawMessage `json:"header"`
+		Body   string           `json:"body"`
+	}{Header: &h, Body: "Hello Gophers!"}
+
+	b, err := json.MarshalIndent(&c, "", "\t")
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+
+	// Output:
+	// {
+	// 	"header": {
+	// 		"precomputed": true
+	// 	},
+	// 	"body": "Hello Gophers!"
+	// }
+}
+
+func ExampleIndent() {
+	type Road struct {
+		Name   string
+		Number int
+	}
+	roads := []Road{
+		{"Diamond Fork", 29},
+		{"Sheep Creek", 51},
+	}
+
+	b, err := json.Marshal(roads)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var out bytes.Buffer
+	json.Indent(&out, b, "=", "\t")
+	out.WriteTo(os.Stdout)
+	// Output:
+	// [
+	// =	{
+	// =		"Name": "Diamond Fork",
+	// =		"Number": 29
+	// =	},
+	// =	{
+	// =		"Name": "Sheep Creek",
+	// =		"Number": 51
+	// =	}
+	// =]
+}
+
+func ExampleMarshalIndent() {
+	data := map[string]int{
+		"a": 1,
+		"b": 2,
+	}
+
+	b, err := json.MarshalIndent(data, "<prefix>", "<indent>")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(b))
+	// Output:
+	// {
+	// <prefix><indent>"a": 1,
+	// <prefix><indent>"b": 2
+	// <prefix>}
+}
+
+func ExampleValid() {
+	goodJSON := `{"example": 1}`
+	badJSON := `{"example":2:]}}`
+
+	fmt.Println(json.Valid([]byte(goodJSON)), json.Valid([]byte(badJSON)))
+	// Output:
+	// true false
+}
+
+func ExampleHTMLEscape() {
+	var out bytes.Buffer
+	json.HTMLEscape(&out, []byte(`{"Name":"<b>HTML content</b>"}`))
+	out.WriteTo(os.Stdout)
+	// Output:
+	//{"Name":"\u003cb\u003eHTML content\u003c/b\u003e"}
+}

gojson/example_text_marshaling_test.go

@@ -0,0 +1,67 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+)
+
+type Size int
+
+const (
+	Unrecognized Size = iota
+	Small
+	Large
+)
+
+func (s *Size) UnmarshalText(text []byte) error {
+	switch strings.ToLower(string(text)) {
+	default:
+		*s = Unrecognized
+	case "small":
+		*s = Small
+	case "large":
+		*s = Large
+	}
+	return nil
+}
+
+func (s Size) MarshalText() ([]byte, error) {
+	var name string
+	switch s {
+	default:
+		name = "unrecognized"
+	case Small:
+		name = "small"
+	case Large:
+		name = "large"
+	}
+	return []byte(name), nil
+}
+
+func Example_textMarshalJSON() {
+	blob := `["small","regular","large","unrecognized","small","normal","small","large"]`
+	var inventory []Size
+	if err := json.Unmarshal([]byte(blob), &inventory); err != nil {
+		log.Fatal(err)
+	}
+
+	counts := make(map[Size]int)
+	for _, size := range inventory {
+		counts[size] += 1
+	}
+
+	fmt.Printf("Inventory Counts:\n* Small:        %d\n* Large:        %d\n* Unrecognized: %d\n",
+		counts[Small], counts[Large], counts[Unrecognized])
+
+	// Output:
+	// Inventory Counts:
+	// * Small:        3
+	// * Large:        2
+	// * Unrecognized: 3
+}

gojson/fold.go

@@ -0,0 +1,144 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+const (
+	caseMask     = ^byte(0x20) // Mask to ignore case in ASCII.
+	kelvin       = '\u212a'
+	smallLongEss = '\u017f'
+)
+
+// foldFunc returns one of four different case folding equivalence
+// functions, from most general (and slow) to fastest:
+//
+// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8
+// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S')
+// 3) asciiEqualFold, no special, but includes non-letters (including _)
+// 4) simpleLetterEqualFold, no specials, no non-letters.
+//
+// The letters S and K are special because they map to 3 runes, not just 2:
+//   - S maps to s and to U+017F 'ſ' Latin small letter long s
+//   - k maps to K and to U+212A 'K' Kelvin sign
+//
+// See https://play.golang.org/p/tTxjOc0OGo
+//
+// The returned function is specialized for matching against s and
+// should only be given s. It's not curried for performance reasons.
+func foldFunc(s []byte) func(s, t []byte) bool {
+	nonLetter := false
+	special := false // special letter
+	for _, b := range s {
+		if b >= utf8.RuneSelf {
+			return bytes.EqualFold
+		}
+		upper := b & caseMask
+		if upper < 'A' || upper > 'Z' {
+			nonLetter = true
+		} else if upper == 'K' || upper == 'S' {
+			// See above for why these letters are special.
+			special = true
+		}
+	}
+	if special {
+		return equalFoldRight
+	}
+	if nonLetter {
+		return asciiEqualFold
+	}
+	return simpleLetterEqualFold
+}
+
+// equalFoldRight is a specialization of bytes.EqualFold when s is
+// known to be all ASCII (including punctuation), but contains an 's',
+// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t.
+// See comments on foldFunc.
+func equalFoldRight(s, t []byte) bool {
+	for _, sb := range s {
+		if len(t) == 0 {
+			return false
+		}
+		tb := t[0]
+		if tb < utf8.RuneSelf {
+			if sb != tb {
+				sbUpper := sb & caseMask
+				if 'A' <= sbUpper && sbUpper <= 'Z' {
+					if sbUpper != tb&caseMask {
+						return false
+					}
+				} else {
+					return false
+				}
+			}
+			t = t[1:]
+			continue
+		}
+		// sb is ASCII and t is not. t must be either kelvin
+		// sign or long s; sb must be s, S, k, or K.
+		tr, size := utf8.DecodeRune(t)
+		switch sb {
+		case 's', 'S':
+			if tr != smallLongEss {
+				return false
+			}
+		case 'k', 'K':
+			if tr != kelvin {
+				return false
+			}
+		default:
+			return false
+		}
+		t = t[size:]
+
+	}
+	if len(t) > 0 {
+		return false
+	}
+	return true
+}
+
+// asciiEqualFold is a specialization of bytes.EqualFold for use when
+// s is all ASCII (but may contain non-letters) and contains no
+// special-folding letters.
+// See comments on foldFunc.
+func asciiEqualFold(s, t []byte) bool {
+	if len(s) != len(t) {
+		return false
+	}
+	for i, sb := range s {
+		tb := t[i]
+		if sb == tb {
+			continue
+		}
+		if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') {
+			if sb&caseMask != tb&caseMask {
+				return false
+			}
+		} else {
+			return false
+		}
+	}
+	return true
+}
+
+// simpleLetterEqualFold is a specialization of bytes.EqualFold for
+// use when s is all ASCII letters (no underscores, etc) and also
+// doesn't contain 'k', 'K', 's', or 'S'.
+// See comments on foldFunc.
+func simpleLetterEqualFold(s, t []byte) bool {
+	if len(s) != len(t) {
+		return false
+	}
+	for i, b := range s {
+		if b&caseMask != t[i]&caseMask {
+			return false
+		}
+	}
+	return true
+}

gojson/fold_test.go

@@ -0,0 +1,116 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+var foldTests = []struct {
+	fn   func(s, t []byte) bool
+	s, t string
+	want bool
+}{
+	{equalFoldRight, "", "", true},
+	{equalFoldRight, "a", "a", true},
+	{equalFoldRight, "", "a", false},
+	{equalFoldRight, "a", "", false},
+	{equalFoldRight, "a", "A", true},
+	{equalFoldRight, "AB", "ab", true},
+	{equalFoldRight, "AB", "ac", false},
+	{equalFoldRight, "sbkKc", "ſbKKc", true},
+	{equalFoldRight, "SbKkc", "ſbKKc", true},
+	{equalFoldRight, "SbKkc", "ſbKK", false},
+	{equalFoldRight, "e", "é", false},
+	{equalFoldRight, "s", "S", true},
+
+	{simpleLetterEqualFold, "", "", true},
+	{simpleLetterEqualFold, "abc", "abc", true},
+	{simpleLetterEqualFold, "abc", "ABC", true},
+	{simpleLetterEqualFold, "abc", "ABCD", false},
+	{simpleLetterEqualFold, "abc", "xxx", false},
+
+	{asciiEqualFold, "a_B", "A_b", true},
+	{asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent
+}
+
+func TestFold(t *testing.T) {
+	for i, tt := range foldTests {
+		if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want {
+			t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want)
+		}
+		truth := strings.EqualFold(tt.s, tt.t)
+		if truth != tt.want {
+			t.Errorf("strings.EqualFold doesn't agree with case %d", i)
+		}
+	}
+}
+
+func TestFoldAgainstUnicode(t *testing.T) {
+	const bufSize = 5
+	buf1 := make([]byte, 0, bufSize)
+	buf2 := make([]byte, 0, bufSize)
+	var runes []rune
+	for i := 0x20; i <= 0x7f; i++ {
+		runes = append(runes, rune(i))
+	}
+	runes = append(runes, kelvin, smallLongEss)
+
+	funcs := []struct {
+		name   string
+		fold   func(s, t []byte) bool
+		letter bool // must be ASCII letter
+		simple bool // must be simple ASCII letter (not 'S' or 'K')
+	}{
+		{
+			name: "equalFoldRight",
+			fold: equalFoldRight,
+		},
+		{
+			name:   "asciiEqualFold",
+			fold:   asciiEqualFold,
+			simple: true,
+		},
+		{
+			name:   "simpleLetterEqualFold",
+			fold:   simpleLetterEqualFold,
+			simple: true,
+			letter: true,
+		},
+	}
+
+	for _, ff := range funcs {
+		for _, r := range runes {
+			if r >= utf8.RuneSelf {
+				continue
+			}
+			if ff.letter && !isASCIILetter(byte(r)) {
+				continue
+			}
+			if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') {
+				continue
+			}
+			for _, r2 := range runes {
+				buf1 := append(buf1[:0], 'x')
+				buf2 := append(buf2[:0], 'x')
+				buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
+				buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
+				buf1 = append(buf1, 'x')
+				buf2 = append(buf2, 'x')
+				want := bytes.EqualFold(buf1, buf2)
+				if got := ff.fold(buf1, buf2); got != want {
+					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)
+				}
+			}
+		}
+	}
+}
+
+func isASCIILetter(b byte) bool {
+	return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z')
+}

gojson/fuzz.go

@@ -0,0 +1,42 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build gofuzz
+
+package json
+
+import (
+	"fmt"
+)
+
+func Fuzz(data []byte) (score int) {
+	for _, ctor := range []func() any{
+		func() any { return new(any) },
+		func() any { return new(map[string]any) },
+		func() any { return new([]any) },
+	} {
+		v := ctor()
+		err := Unmarshal(data, v)
+		if err != nil {
+			continue
+		}
+		score = 1
+
+		m, err := Marshal(v)
+		if err != nil {
+			fmt.Printf("v=%#v\n", v)
+			panic(err)
+		}
+
+		u := ctor()
+		err = Unmarshal(m, u)
+		if err != nil {
+			fmt.Printf("v=%#v\n", v)
+			fmt.Printf("m=%s\n", m)
+			panic(err)
+		}
+	}
+
+	return
+}

gojson/fuzz_test.go

@@ -0,0 +1,83 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func FuzzUnmarshalJSON(f *testing.F) {
+	f.Add([]byte(`{
+"object": {
+	"slice": [
+		1,
+		2.0,
+		"3",
+		[4],
+		{5: {}}
+	]
+},
+"slice": [[]],
+"string": ":)",
+"int": 1e5,
+"float": 3e-9"
+}`))
+
+	f.Fuzz(func(t *testing.T, b []byte) {
+		for _, typ := range []func() interface{}{
+			func() interface{} { return new(interface{}) },
+			func() interface{} { return new(map[string]interface{}) },
+			func() interface{} { return new([]interface{}) },
+		} {
+			i := typ()
+			if err := Unmarshal(b, i); err != nil {
+				return
+			}
+
+			encoded, err := Marshal(i)
+			if err != nil {
+				t.Fatalf("failed to marshal: %s", err)
+			}
+
+			if err := Unmarshal(encoded, i); err != nil {
+				t.Fatalf("failed to roundtrip: %s", err)
+			}
+		}
+	})
+}
+
+func FuzzDecoderToken(f *testing.F) {
+	f.Add([]byte(`{
+"object": {
+	"slice": [
+		1,
+		2.0,
+		"3",
+		[4],
+		{5: {}}
+	]
+},
+"slice": [[]],
+"string": ":)",
+"int": 1e5,
+"float": 3e-9"
+}`))
+
+	f.Fuzz(func(t *testing.T, b []byte) {
+		r := bytes.NewReader(b)
+		d := NewDecoder(r)
+		for {
+			_, err := d.Token()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return
+			}
+		}
+	})
+}

gojson/gionic.go

@@ -0,0 +1,53 @@
+package json
+
+import (
+	"net/http"
+)
+
+// Render interface is copied from github.com/gin-gonic/gin@v1.8.1/render/render.go
+type Render interface {
+	// Render writes data with custom ContentType.
+	Render(http.ResponseWriter) error
+	// WriteContentType writes custom ContentType.
+	WriteContentType(w http.ResponseWriter)
+}
+
+type GoJsonRender struct {
+	Data          any
+	NilSafeSlices bool
+	NilSafeMaps   bool
+	Indent        *IndentOpt
+	Filter        *string
+}
+
+func (r GoJsonRender) Render(w http.ResponseWriter) error {
+	header := w.Header()
+	if val := header["Content-Type"]; len(val) == 0 {
+		header["Content-Type"] = []string{"application/json; charset=utf-8"}
+	}
+
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
+	if err != nil {
+		panic(err)
+	}
+	_, err = w.Write(jsonBytes)
+	if err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+func (r GoJsonRender) RenderString() (string, error) {
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
+	if err != nil {
+		panic(err)
+	}
+	return string(jsonBytes), nil
+}
+
+func (r GoJsonRender) WriteContentType(w http.ResponseWriter) {
+	header := w.Header()
+	if val := header["Content-Type"]; len(val) == 0 {
+		header["Content-Type"] = []string{"application/json; charset=utf-8"}
+	}
+}

gojson/indent.go

@@ -0,0 +1,143 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+)
+
+// Compact appends to dst the JSON-encoded src with
+// insignificant space characters elided.
+func Compact(dst *bytes.Buffer, src []byte) error {
+	return compact(dst, src, false)
+}
+
+func compact(dst *bytes.Buffer, src []byte, escape bool) error {
+	origLen := dst.Len()
+	scan := newScanner()
+	defer freeScanner(scan)
+	start := 0
+	for i, c := range src {
+		if escape && (c == '<' || c == '>' || c == '&') {
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			dst.WriteString(`\u00`)
+			dst.WriteByte(hex[c>>4])
+			dst.WriteByte(hex[c&0xF])
+			start = i + 1
+		}
+		// Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9).
+		if escape && c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 {
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			dst.WriteString(`\u202`)
+			dst.WriteByte(hex[src[i+2]&0xF])
+			start = i + 3
+		}
+		v := scan.step(scan, c)
+		if v >= scanSkipSpace {
+			if v == scanError {
+				break
+			}
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			start = i + 1
+		}
+	}
+	if scan.eof() == scanError {
+		dst.Truncate(origLen)
+		return scan.err
+	}
+	if start < len(src) {
+		dst.Write(src[start:])
+	}
+	return nil
+}
+
+func newline(dst *bytes.Buffer, prefix, indent string, depth int) {
+	dst.WriteByte('\n')
+	dst.WriteString(prefix)
+	for i := 0; i < depth; i++ {
+		dst.WriteString(indent)
+	}
+}
+
+// Indent appends to dst an indented form of the JSON-encoded src.
+// Each element in a JSON object or array begins on a new,
+// indented line beginning with prefix followed by one or more
+// copies of indent according to the indentation nesting.
+// The data appended to dst does not begin with the prefix nor
+// any indentation, to make it easier to embed inside other formatted JSON data.
+// Although leading space characters (space, tab, carriage return, newline)
+// at the beginning of src are dropped, trailing space characters
+// at the end of src are preserved and copied to dst.
+// For example, if src has no trailing spaces, neither will dst;
+// if src ends in a trailing newline, so will dst.
+func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
+	origLen := dst.Len()
+	scan := newScanner()
+	defer freeScanner(scan)
+	needIndent := false
+	depth := 0
+	for _, c := range src {
+		scan.bytes++
+		v := scan.step(scan, c)
+		if v == scanSkipSpace {
+			continue
+		}
+		if v == scanError {
+			break
+		}
+		if needIndent && v != scanEndObject && v != scanEndArray {
+			needIndent = false
+			depth++
+			newline(dst, prefix, indent, depth)
+		}
+
+		// Emit semantically uninteresting bytes
+		// (in particular, punctuation in strings) unmodified.
+		if v == scanContinue {
+			dst.WriteByte(c)
+			continue
+		}
+
+		// Add spacing around real punctuation.
+		switch c {
+		case '{', '[':
+			// delay indent so that empty object and array are formatted as {} and [].
+			needIndent = true
+			dst.WriteByte(c)
+
+		case ',':
+			dst.WriteByte(c)
+			newline(dst, prefix, indent, depth)
+
+		case ':':
+			dst.WriteByte(c)
+			dst.WriteByte(' ')
+
+		case '}', ']':
+			if needIndent {
+				// suppress indent in empty object/array
+				needIndent = false
+			} else {
+				depth--
+				newline(dst, prefix, indent, depth)
+			}
+			dst.WriteByte(c)
+
+		default:
+			dst.WriteByte(c)
+		}
+	}
+	if scan.eof() == scanError {
+		dst.Truncate(origLen)
+		return scan.err
+	}
+	return nil
+}

gojson/number_test.go

@@ -0,0 +1,133 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"regexp"
+	"testing"
+)
+
+func TestNumberIsValid(t *testing.T) {
+	// From: https://stackoverflow.com/a/13340826
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+
+	validTests := []string{
+		"0",
+		"-0",
+		"1",
+		"-1",
+		"0.1",
+		"-0.1",
+		"1234",
+		"-1234",
+		"12.34",
+		"-12.34",
+		"12E0",
+		"12E1",
+		"12e34",
+		"12E-0",
+		"12e+1",
+		"12e-34",
+		"-12E0",
+		"-12E1",
+		"-12e34",
+		"-12E-0",
+		"-12e+1",
+		"-12e-34",
+		"1.2E0",
+		"1.2E1",
+		"1.2e34",
+		"1.2E-0",
+		"1.2e+1",
+		"1.2e-34",
+		"-1.2E0",
+		"-1.2E1",
+		"-1.2e34",
+		"-1.2E-0",
+		"-1.2e+1",
+		"-1.2e-34",
+		"0E0",
+		"0E1",
+		"0e34",
+		"0E-0",
+		"0e+1",
+		"0e-34",
+		"-0E0",
+		"-0E1",
+		"-0e34",
+		"-0E-0",
+		"-0e+1",
+		"-0e-34",
+	}
+
+	for _, test := range validTests {
+		if !isValidNumber(test) {
+			t.Errorf("%s should be valid", test)
+		}
+
+		var f float64
+		if err := Unmarshal([]byte(test), &f); err != nil {
+			t.Errorf("%s should be valid but Unmarshal failed: %v", test, err)
+		}
+
+		if !jsonNumberRegexp.MatchString(test) {
+			t.Errorf("%s should be valid but regexp does not match", test)
+		}
+	}
+
+	invalidTests := []string{
+		"",
+		"invalid",
+		"1.0.1",
+		"1..1",
+		"-1-2",
+		"012a42",
+		"01.2",
+		"012",
+		"12E12.12",
+		"1e2e3",
+		"1e+-2",
+		"1e--23",
+		"1e",
+		"e1",
+		"1e+",
+		"1ea",
+		"1a",
+		"1.a",
+		"1.",
+		"01",
+		"1.e1",
+	}
+
+	for _, test := range invalidTests {
+		if isValidNumber(test) {
+			t.Errorf("%s should be invalid", test)
+		}
+
+		var f float64
+		if err := Unmarshal([]byte(test), &f); err == nil {
+			t.Errorf("%s should be invalid but unmarshal wrote %v", test, f)
+		}
+
+		if jsonNumberRegexp.MatchString(test) {
+			t.Errorf("%s should be invalid but matches regexp", test)
+		}
+	}
+}
+
+func BenchmarkNumberIsValid(b *testing.B) {
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		isValidNumber(s)
+	}
+}
+
+func BenchmarkNumberIsValidRegexp(b *testing.B) {
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+	s := "-61657.61667E+61673"
+	for i := 0; i < b.N; i++ {
+		jsonNumberRegexp.MatchString(s)
+	}
+}

gojson/scanner.go

@@ -0,0 +1,610 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+// JSON value parser state machine.
+// Just about at the limit of what is reasonable to write by hand.
+// Some parts are a bit tedious, but overall it nicely factors out the
+// otherwise common code from the multiple scanning functions
+// in this package (Compact, Indent, checkValid, etc).
+//
+// This file starts with two simple examples using the scanner
+// before diving into the scanner itself.
+
+import (
+	"strconv"
+	"sync"
+)
+
+// Valid reports whether data is a valid JSON encoding.
+func Valid(data []byte) bool {
+	scan := newScanner()
+	defer freeScanner(scan)
+	return checkValid(data, scan) == nil
+}
+
+// checkValid verifies that data is valid JSON-encoded data.
+// scan is passed in for use by checkValid to avoid an allocation.
+// checkValid returns nil or a SyntaxError.
+func checkValid(data []byte, scan *scanner) error {
+	scan.reset()
+	for _, c := range data {
+		scan.bytes++
+		if scan.step(scan, c) == scanError {
+			return scan.err
+		}
+	}
+	if scan.eof() == scanError {
+		return scan.err
+	}
+	return nil
+}
+
+// A SyntaxError is a description of a JSON syntax error.
+// Unmarshal will return a SyntaxError if the JSON can't be parsed.
+type SyntaxError struct {
+	msg    string // description of error
+	Offset int64  // error occurred after reading Offset bytes
+}
+
+func (e *SyntaxError) Error() string { return e.msg }
+
+// A scanner is a JSON scanning state machine.
+// Callers call scan.reset and then pass bytes in one at a time
+// by calling scan.step(&scan, c) for each byte.
+// The return value, referred to as an opcode, tells the
+// caller about significant parsing events like beginning
+// and ending literals, objects, and arrays, so that the
+// caller can follow along if it wishes.
+// The return value scanEnd indicates that a single top-level
+// JSON value has been completed, *before* the byte that
+// just got passed in.  (The indication must be delayed in order
+// to recognize the end of numbers: is 123 a whole value or
+// the beginning of 12345e+6?).
+type scanner struct {
+	// The step is a func to be called to execute the next transition.
+	// Also tried using an integer constant and a single func
+	// with a switch, but using the func directly was 10% faster
+	// on a 64-bit Mac Mini, and it's nicer to read.
+	step func(*scanner, byte) int
+
+	// Reached end of top-level value.
+	endTop bool
+
+	// Stack of what we're in the middle of - array values, object keys, object values.
+	parseState []int
+
+	// Error that happened, if any.
+	err error
+
+	// total bytes consumed, updated by decoder.Decode (and deliberately
+	// not set to zero by scan.reset)
+	bytes int64
+}
+
+var scannerPool = sync.Pool{
+	New: func() any {
+		return &scanner{}
+	},
+}
+
+func newScanner() *scanner {
+	scan := scannerPool.Get().(*scanner)
+	// scan.reset by design doesn't set bytes to zero
+	scan.bytes = 0
+	scan.reset()
+	return scan
+}
+
+func freeScanner(scan *scanner) {
+	// Avoid hanging on to too much memory in extreme cases.
+	if len(scan.parseState) > 1024 {
+		scan.parseState = nil
+	}
+	scannerPool.Put(scan)
+}
+
+// These values are returned by the state transition functions
+// assigned to scanner.state and the method scanner.eof.
+// They give details about the current state of the scan that
+// callers might be interested to know about.
+// It is okay to ignore the return value of any particular
+// call to scanner.state: if one call returns scanError,
+// every subsequent call will return scanError too.
+const (
+	// Continue.
+	scanContinue     = iota // uninteresting byte
+	scanBeginLiteral        // end implied by next result != scanContinue
+	scanBeginObject         // begin object
+	scanObjectKey           // just finished object key (string)
+	scanObjectValue         // just finished non-last object value
+	scanEndObject           // end object (implies scanObjectValue if possible)
+	scanBeginArray          // begin array
+	scanArrayValue          // just finished array value
+	scanEndArray            // end array (implies scanArrayValue if possible)
+	scanSkipSpace           // space byte; can skip; known to be last "continue" result
+
+	// Stop.
+	scanEnd   // top-level value ended *before* this byte; known to be first "stop" result
+	scanError // hit an error, scanner.err.
+)
+
+// These values are stored in the parseState stack.
+// They give the current state of a composite value
+// being scanned. If the parser is inside a nested value
+// the parseState describes the nested state, outermost at entry 0.
+const (
+	parseObjectKey   = iota // parsing object key (before colon)
+	parseObjectValue        // parsing object value (after colon)
+	parseArrayValue         // parsing array value
+)
+
+// This limits the max nesting depth to prevent stack overflow.
+// This is permitted by https://tools.ietf.org/html/rfc7159#section-9
+const maxNestingDepth = 10000
+
+// reset prepares the scanner for use.
+// It must be called before calling s.step.
+func (s *scanner) reset() {
+	s.step = stateBeginValue
+	s.parseState = s.parseState[0:0]
+	s.err = nil
+	s.endTop = false
+}
+
+// eof tells the scanner that the end of input has been reached.
+// It returns a scan status just as s.step does.
+func (s *scanner) eof() int {
+	if s.err != nil {
+		return scanError
+	}
+	if s.endTop {
+		return scanEnd
+	}
+	s.step(s, ' ')
+	if s.endTop {
+		return scanEnd
+	}
+	if s.err == nil {
+		s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
+	}
+	return scanError
+}
+
+// pushParseState pushes a new parse state p onto the parse stack.
+// an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned.
+func (s *scanner) pushParseState(c byte, newParseState int, successState int) int {
+	s.parseState = append(s.parseState, newParseState)
+	if len(s.parseState) <= maxNestingDepth {
+		return successState
+	}
+	return s.error(c, "exceeded max depth")
+}
+
+// popParseState pops a parse state (already obtained) off the stack
+// and updates s.step accordingly.
+func (s *scanner) popParseState() {
+	n := len(s.parseState) - 1
+	s.parseState = s.parseState[0:n]
+	if n == 0 {
+		s.step = stateEndTop
+		s.endTop = true
+	} else {
+		s.step = stateEndValue
+	}
+}
+
+func isSpace(c byte) bool {
+	return c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n')
+}
+
+// stateBeginValueOrEmpty is the state after reading `[`.
+func stateBeginValueOrEmpty(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == ']' {
+		return stateEndValue(s, c)
+	}
+	return stateBeginValue(s, c)
+}
+
+// stateBeginValue is the state at the beginning of the input.
+func stateBeginValue(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	switch c {
+	case '{':
+		s.step = stateBeginStringOrEmpty
+		return s.pushParseState(c, parseObjectKey, scanBeginObject)
+	case '[':
+		s.step = stateBeginValueOrEmpty
+		return s.pushParseState(c, parseArrayValue, scanBeginArray)
+	case '"':
+		s.step = stateInString
+		return scanBeginLiteral
+	case '-':
+		s.step = stateNeg
+		return scanBeginLiteral
+	case '0': // beginning of 0.123
+		s.step = state0
+		return scanBeginLiteral
+	case 't': // beginning of true
+		s.step = stateT
+		return scanBeginLiteral
+	case 'f': // beginning of false
+		s.step = stateF
+		return scanBeginLiteral
+	case 'n': // beginning of null
+		s.step = stateN
+		return scanBeginLiteral
+	}
+	if '1' <= c && c <= '9' { // beginning of 1234.5
+		s.step = state1
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of value")
+}
+
+// stateBeginStringOrEmpty is the state after reading `{`.
+func stateBeginStringOrEmpty(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '}' {
+		n := len(s.parseState)
+		s.parseState[n-1] = parseObjectValue
+		return stateEndValue(s, c)
+	}
+	return stateBeginString(s, c)
+}
+
+// stateBeginString is the state after reading `{"key": value,`.
+func stateBeginString(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '"' {
+		s.step = stateInString
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of object key string")
+}
+
+// stateEndValue is the state after completing a value,
+// such as after reading `{}` or `true` or `["x"`.
+func stateEndValue(s *scanner, c byte) int {
+	n := len(s.parseState)
+	if n == 0 {
+		// Completed top-level before the current byte.
+		s.step = stateEndTop
+		s.endTop = true
+		return stateEndTop(s, c)
+	}
+	if isSpace(c) {
+		s.step = stateEndValue
+		return scanSkipSpace
+	}
+	ps := s.parseState[n-1]
+	switch ps {
+	case parseObjectKey:
+		if c == ':' {
+			s.parseState[n-1] = parseObjectValue
+			s.step = stateBeginValue
+			return scanObjectKey
+		}
+		return s.error(c, "after object key")
+	case parseObjectValue:
+		if c == ',' {
+			s.parseState[n-1] = parseObjectKey
+			s.step = stateBeginString
+			return scanObjectValue
+		}
+		if c == '}' {
+			s.popParseState()
+			return scanEndObject
+		}
+		return s.error(c, "after object key:value pair")
+	case parseArrayValue:
+		if c == ',' {
+			s.step = stateBeginValue
+			return scanArrayValue
+		}
+		if c == ']' {
+			s.popParseState()
+			return scanEndArray
+		}
+		return s.error(c, "after array element")
+	}
+	return s.error(c, "")
+}
+
+// stateEndTop is the state after finishing the top-level value,
+// such as after reading `{}` or `[1,2,3]`.
+// Only space characters should be seen now.
+func stateEndTop(s *scanner, c byte) int {
+	if !isSpace(c) {
+		// Complain about non-space byte on next call.
+		s.error(c, "after top-level value")
+	}
+	return scanEnd
+}
+
+// stateInString is the state after reading `"`.
+func stateInString(s *scanner, c byte) int {
+	if c == '"' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	if c == '\\' {
+		s.step = stateInStringEsc
+		return scanContinue
+	}
+	if c < 0x20 {
+		return s.error(c, "in string literal")
+	}
+	return scanContinue
+}
+
+// stateInStringEsc is the state after reading `"\` during a quoted string.
+func stateInStringEsc(s *scanner, c byte) int {
+	switch c {
+	case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
+		s.step = stateInString
+		return scanContinue
+	case 'u':
+		s.step = stateInStringEscU
+		return scanContinue
+	}
+	return s.error(c, "in string escape code")
+}
+
+// stateInStringEscU is the state after reading `"\u` during a quoted string.
+func stateInStringEscU(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU1
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
+func stateInStringEscU1(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU12
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
+func stateInStringEscU12(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU123
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
+func stateInStringEscU123(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInString
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateNeg is the state after reading `-` during a number.
+func stateNeg(s *scanner, c byte) int {
+	if c == '0' {
+		s.step = state0
+		return scanContinue
+	}
+	if '1' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return s.error(c, "in numeric literal")
+}
+
+// state1 is the state after reading a non-zero integer during a number,
+// such as after reading `1` or `100` but not `0`.
+func state1(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return state0(s, c)
+}
+
+// state0 is the state after reading `0` during a number.
+func state0(s *scanner, c byte) int {
+	if c == '.' {
+		s.step = stateDot
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateDot is the state after reading the integer and decimal point in a number,
+// such as after reading `1.`.
+func stateDot(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateDot0
+		return scanContinue
+	}
+	return s.error(c, "after decimal point in numeric literal")
+}
+
+// stateDot0 is the state after reading the integer, decimal point, and subsequent
+// digits of a number, such as after reading `3.14`.
+func stateDot0(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateE is the state after reading the mantissa and e in a number,
+// such as after reading `314e` or `0.314e`.
+func stateE(s *scanner, c byte) int {
+	if c == '+' || c == '-' {
+		s.step = stateESign
+		return scanContinue
+	}
+	return stateESign(s, c)
+}
+
+// stateESign is the state after reading the mantissa, e, and sign in a number,
+// such as after reading `314e-` or `0.314e+`.
+func stateESign(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateE0
+		return scanContinue
+	}
+	return s.error(c, "in exponent of numeric literal")
+}
+
+// stateE0 is the state after reading the mantissa, e, optional sign,
+// and at least one digit of the exponent in a number,
+// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
+func stateE0(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateT is the state after reading `t`.
+func stateT(s *scanner, c byte) int {
+	if c == 'r' {
+		s.step = stateTr
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'r')")
+}
+
+// stateTr is the state after reading `tr`.
+func stateTr(s *scanner, c byte) int {
+	if c == 'u' {
+		s.step = stateTru
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'u')")
+}
+
+// stateTru is the state after reading `tru`.
+func stateTru(s *scanner, c byte) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'e')")
+}
+
+// stateF is the state after reading `f`.
+func stateF(s *scanner, c byte) int {
+	if c == 'a' {
+		s.step = stateFa
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'a')")
+}
+
+// stateFa is the state after reading `fa`.
+func stateFa(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateFal
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'l')")
+}
+
+// stateFal is the state after reading `fal`.
+func stateFal(s *scanner, c byte) int {
+	if c == 's' {
+		s.step = stateFals
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 's')")
+}
+
+// stateFals is the state after reading `fals`.
+func stateFals(s *scanner, c byte) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'e')")
+}
+
+// stateN is the state after reading `n`.
+func stateN(s *scanner, c byte) int {
+	if c == 'u' {
+		s.step = stateNu
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'u')")
+}
+
+// stateNu is the state after reading `nu`.
+func stateNu(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateNul
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateNul is the state after reading `nul`.
+func stateNul(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateError is the state after reaching a syntax error,
+// such as after reading `[1}` or `5.1.2`.
+func stateError(s *scanner, c byte) int {
+	return scanError
+}
+
+// error records an error and switches to the error state.
+func (s *scanner) error(c byte, context string) int {
+	s.step = stateError
+	s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
+	return scanError
+}
+
+// quoteChar formats c as a quoted character literal
+func quoteChar(c byte) string {
+	// special cases - different from quoted strings
+	if c == '\'' {
+		return `'\''`
+	}
+	if c == '"' {
+		return `'"'`
+	}
+
+	// use quoted string with different quotation marks
+	s := strconv.Quote(string(c))
+	return "'" + s[1:len(s)-1] + "'"
+}

gojson/scanner_test.go

@@ -0,0 +1,301 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"math"
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+var validTests = []struct {
+	data string
+	ok   bool
+}{
+	{`foo`, false},
+	{`}{`, false},
+	{`{]`, false},
+	{`{}`, true},
+	{`{"foo":"bar"}`, true},
+	{`{"foo":"bar","bar":{"baz":["qux"]}}`, true},
+}
+
+func TestValid(t *testing.T) {
+	for _, tt := range validTests {
+		if ok := Valid([]byte(tt.data)); ok != tt.ok {
+			t.Errorf("Valid(%#q) = %v, want %v", tt.data, ok, tt.ok)
+		}
+	}
+}
+
+// Tests of simple examples.
+
+type example struct {
+	compact string
+	indent  string
+}
+
+var examples = []example{
+	{`1`, `1`},
+	{`{}`, `{}`},
+	{`[]`, `[]`},
+	{`{"":2}`, "{\n\t\"\": 2\n}"},
+	{`[3]`, "[\n\t3\n]"},
+	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
+	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
+	{ex1, ex1i},
+	{"{\"\":\"<>&\u2028\u2029\"}", "{\n\t\"\": \"<>&\u2028\u2029\"\n}"}, // See golang.org/issue/34070
+}
+
+var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`
+
+var ex1i = `[
+	true,
+	false,
+	null,
+	"x",
+	1,
+	1.5,
+	0,
+	-5e+2
+]`
+
+func TestCompact(t *testing.T) {
+	var buf bytes.Buffer
+	for _, tt := range examples {
+		buf.Reset()
+		if err := Compact(&buf, []byte(tt.compact)); err != nil {
+			t.Errorf("Compact(%#q): %v", tt.compact, err)
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
+		}
+
+		buf.Reset()
+		if err := Compact(&buf, []byte(tt.indent)); err != nil {
+			t.Errorf("Compact(%#q): %v", tt.indent, err)
+			continue
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
+		}
+	}
+}
+
+func TestCompactSeparators(t *testing.T) {
+	// U+2028 and U+2029 should be escaped inside strings.
+	// They should not appear outside strings.
+	tests := []struct {
+		in, compact string
+	}{
+		{"{\"\u2028\": 1}", "{\"\u2028\":1}"},
+		{"{\"\u2029\" :2}", "{\"\u2029\":2}"},
+	}
+	for _, tt := range tests {
+		var buf bytes.Buffer
+		if err := Compact(&buf, []byte(tt.in)); err != nil {
+			t.Errorf("Compact(%q): %v", tt.in, err)
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%q) = %q, want %q", tt.in, s, tt.compact)
+		}
+	}
+}
+
+func TestIndent(t *testing.T) {
+	var buf bytes.Buffer
+	for _, tt := range examples {
+		buf.Reset()
+		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
+			t.Errorf("Indent(%#q): %v", tt.indent, err)
+		} else if s := buf.String(); s != tt.indent {
+			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
+		}
+
+		buf.Reset()
+		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
+			t.Errorf("Indent(%#q): %v", tt.compact, err)
+			continue
+		} else if s := buf.String(); s != tt.indent {
+			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
+		}
+	}
+}
+
+// Tests of a large random structure.
+
+func TestCompactBig(t *testing.T) {
+	initBig()
+	var buf bytes.Buffer
+	if err := Compact(&buf, jsonBig); err != nil {
+		t.Fatalf("Compact: %v", err)
+	}
+	b := buf.Bytes()
+	if !bytes.Equal(b, jsonBig) {
+		t.Error("Compact(jsonBig) != jsonBig")
+		diff(t, b, jsonBig)
+		return
+	}
+}
+
+func TestIndentBig(t *testing.T) {
+	t.Parallel()
+	initBig()
+	var buf bytes.Buffer
+	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
+		t.Fatalf("Indent1: %v", err)
+	}
+	b := buf.Bytes()
+	if len(b) == len(jsonBig) {
+		// jsonBig is compact (no unnecessary spaces);
+		// indenting should make it bigger
+		t.Fatalf("Indent(jsonBig) did not get bigger")
+	}
+
+	// should be idempotent
+	var buf1 bytes.Buffer
+	if err := Indent(&buf1, b, "", "\t"); err != nil {
+		t.Fatalf("Indent2: %v", err)
+	}
+	b1 := buf1.Bytes()
+	if !bytes.Equal(b1, b) {
+		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
+		diff(t, b1, b)
+		return
+	}
+
+	// should get back to original
+	buf1.Reset()
+	if err := Compact(&buf1, b); err != nil {
+		t.Fatalf("Compact: %v", err)
+	}
+	b1 = buf1.Bytes()
+	if !bytes.Equal(b1, jsonBig) {
+		t.Error("Compact(Indent(jsonBig)) != jsonBig")
+		diff(t, b1, jsonBig)
+		return
+	}
+}
+
+type indentErrorTest struct {
+	in  string
+	err error
+}
+
+var indentErrorTests = []indentErrorTest{
+	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
+	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
+}
+
+func TestIndentErrors(t *testing.T) {
+	for i, tt := range indentErrorTests {
+		slice := make([]uint8, 0)
+		buf := bytes.NewBuffer(slice)
+		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
+			if !reflect.DeepEqual(err, tt.err) {
+				t.Errorf("#%d: Indent: %#v", i, err)
+				continue
+			}
+		}
+	}
+}
+
+func diff(t *testing.T, a, b []byte) {
+	for i := 0; ; i++ {
+		if i >= len(a) || i >= len(b) || a[i] != b[i] {
+			j := i - 10
+			if j < 0 {
+				j = 0
+			}
+			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
+			return
+		}
+	}
+}
+
+func trim(b []byte) []byte {
+	if len(b) > 20 {
+		return b[0:20]
+	}
+	return b
+}
+
+// Generate a random JSON object.
+
+var jsonBig []byte
+
+func initBig() {
+	n := 10000
+	if testing.Short() {
+		n = 100
+	}
+	b, err := Marshal(genValue(n))
+	if err != nil {
+		panic(err)
+	}
+	jsonBig = b
+}
+
+func genValue(n int) any {
+	if n > 1 {
+		switch rand.Intn(2) {
+		case 0:
+			return genArray(n)
+		case 1:
+			return genMap(n)
+		}
+	}
+	switch rand.Intn(3) {
+	case 0:
+		return rand.Intn(2) == 0
+	case 1:
+		return rand.NormFloat64()
+	case 2:
+		return genString(30)
+	}
+	panic("unreachable")
+}
+
+func genString(stddev float64) string {
+	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
+	c := make([]rune, n)
+	for i := range c {
+		f := math.Abs(rand.NormFloat64()*64 + 32)
+		if f > 0x10ffff {
+			f = 0x10ffff
+		}
+		c[i] = rune(f)
+	}
+	return string(c)
+}
+
+func genArray(n int) []any {
+	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
+	if f > n {
+		f = n
+	}
+	if f < 1 {
+		f = 1
+	}
+	x := make([]any, f)
+	for i := range x {
+		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
+	}
+	return x
+}
+
+func genMap(n int) map[string]any {
+	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
+	if f > n {
+		f = n
+	}
+	if n > 0 && f == 0 {
+		f = 1
+	}
+	x := make(map[string]any)
+	for i := 0; i < f; i++ {
+		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
+	}
+	return x
+}

gojson/stream.go

@@ -0,0 +1,513 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+// A Decoder reads and decodes JSON values from an input stream.
+type Decoder struct {
+	r       io.Reader
+	buf     []byte
+	d       decodeState
+	scanp   int   // start of unread data in buf
+	scanned int64 // amount of data already scanned
+	scan    scanner
+	err     error
+
+	tokenState int
+	tokenStack []int
+}
+
+// NewDecoder returns a new decoder that reads from r.
+//
+// The decoder introduces its own buffering and may
+// read data from r beyond the JSON values requested.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{r: r}
+}
+
+// UseNumber causes the Decoder to unmarshal a number into an interface{} as a
+// Number instead of as a float64.
+func (dec *Decoder) UseNumber() { dec.d.useNumber = true }
+
+// DisallowUnknownFields causes the Decoder to return an error when the destination
+// is a struct and the input contains object keys which do not match any
+// non-ignored, exported fields in the destination.
+func (dec *Decoder) DisallowUnknownFields() { dec.d.disallowUnknownFields = true }
+
+// Decode reads the next JSON-encoded value from its
+// input and stores it in the value pointed to by v.
+//
+// See the documentation for Unmarshal for details about
+// the conversion of JSON into a Go value.
+func (dec *Decoder) Decode(v any) error {
+	if dec.err != nil {
+		return dec.err
+	}
+
+	if err := dec.tokenPrepareForDecode(); err != nil {
+		return err
+	}
+
+	if !dec.tokenValueAllowed() {
+		return &SyntaxError{msg: "not at beginning of value", Offset: dec.InputOffset()}
+	}
+
+	// Read whole value into buffer.
+	n, err := dec.readValue()
+	if err != nil {
+		return err
+	}
+	dec.d.init(dec.buf[dec.scanp : dec.scanp+n])
+	dec.scanp += n
+
+	// Don't save err from unmarshal into dec.err:
+	// the connection is still usable since we read a complete JSON
+	// object from it before the error happened.
+	err = dec.d.unmarshal(v)
+
+	// fixup token streaming state
+	dec.tokenValueEnd()
+
+	return err
+}
+
+// Buffered returns a reader of the data remaining in the Decoder's
+// buffer. The reader is valid until the next call to Decode.
+func (dec *Decoder) Buffered() io.Reader {
+	return bytes.NewReader(dec.buf[dec.scanp:])
+}
+
+// readValue reads a JSON value into dec.buf.
+// It returns the length of the encoding.
+func (dec *Decoder) readValue() (int, error) {
+	dec.scan.reset()
+
+	scanp := dec.scanp
+	var err error
+Input:
+	// help the compiler see that scanp is never negative, so it can remove
+	// some bounds checks below.
+	for scanp >= 0 {
+
+		// Look in the buffer for a new value.
+		for ; scanp < len(dec.buf); scanp++ {
+			c := dec.buf[scanp]
+			dec.scan.bytes++
+			switch dec.scan.step(&dec.scan, c) {
+			case scanEnd:
+				// scanEnd is delayed one byte so we decrement
+				// the scanner bytes count by 1 to ensure that
+				// this value is correct in the next call of Decode.
+				dec.scan.bytes--
+				break Input
+			case scanEndObject, scanEndArray:
+				// scanEnd is delayed one byte.
+				// We might block trying to get that byte from src,
+				// so instead invent a space byte.
+				if stateEndValue(&dec.scan, ' ') == scanEnd {
+					scanp++
+					break Input
+				}
+			case scanError:
+				dec.err = dec.scan.err
+				return 0, dec.scan.err
+			}
+		}
+
+		// Did the last read have an error?
+		// Delayed until now to allow buffer scan.
+		if err != nil {
+			if err == io.EOF {
+				if dec.scan.step(&dec.scan, ' ') == scanEnd {
+					break Input
+				}
+				if nonSpace(dec.buf) {
+					err = io.ErrUnexpectedEOF
+				}
+			}
+			dec.err = err
+			return 0, err
+		}
+
+		n := scanp - dec.scanp
+		err = dec.refill()
+		scanp = dec.scanp + n
+	}
+	return scanp - dec.scanp, nil
+}
+
+func (dec *Decoder) refill() error {
+	// Make room to read more into the buffer.
+	// First slide down data already consumed.
+	if dec.scanp > 0 {
+		dec.scanned += int64(dec.scanp)
+		n := copy(dec.buf, dec.buf[dec.scanp:])
+		dec.buf = dec.buf[:n]
+		dec.scanp = 0
+	}
+
+	// Grow buffer if not large enough.
+	const minRead = 512
+	if cap(dec.buf)-len(dec.buf) < minRead {
+		newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead)
+		copy(newBuf, dec.buf)
+		dec.buf = newBuf
+	}
+
+	// Read. Delay error for next iteration (after scan).
+	n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
+	dec.buf = dec.buf[0 : len(dec.buf)+n]
+
+	return err
+}
+
+func nonSpace(b []byte) bool {
+	for _, c := range b {
+		if !isSpace(c) {
+			return true
+		}
+	}
+	return false
+}
+
+// An Encoder writes JSON values to an output stream.
+type Encoder struct {
+	w          io.Writer
+	err        error
+	escapeHTML bool
+
+	indentBuf    *bytes.Buffer
+	indentPrefix string
+	indentValue  string
+}
+
+// NewEncoder returns a new encoder that writes to w.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{w: w, escapeHTML: true}
+}
+
+// Encode writes the JSON encoding of v to the stream,
+// followed by a newline character.
+//
+// See the documentation for Marshal for details about the
+// conversion of Go values to JSON.
+func (enc *Encoder) Encode(v any) error {
+	if enc.err != nil {
+		return enc.err
+	}
+	e := newEncodeState()
+	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
+	if err != nil {
+		return err
+	}
+
+	// Terminate each value with a newline.
+	// This makes the output look a little nicer
+	// when debugging, and some kind of space
+	// is required if the encoded value was a number,
+	// so that the reader knows there aren't more
+	// digits coming.
+	e.WriteByte('\n')
+
+	b := e.Bytes()
+	if enc.indentPrefix != "" || enc.indentValue != "" {
+		if enc.indentBuf == nil {
+			enc.indentBuf = new(bytes.Buffer)
+		}
+		enc.indentBuf.Reset()
+		err = Indent(enc.indentBuf, b, enc.indentPrefix, enc.indentValue)
+		if err != nil {
+			return err
+		}
+		b = enc.indentBuf.Bytes()
+	}
+	if _, err = enc.w.Write(b); err != nil {
+		enc.err = err
+	}
+	encodeStatePool.Put(e)
+	return err
+}
+
+// SetIndent instructs the encoder to format each subsequent encoded
+// value as if indented by the package-level function Indent(dst, src, prefix, indent).
+// Calling SetIndent("", "") disables indentation.
+func (enc *Encoder) SetIndent(prefix, indent string) {
+	enc.indentPrefix = prefix
+	enc.indentValue = indent
+}
+
+// SetEscapeHTML specifies whether problematic HTML characters
+// should be escaped inside JSON quoted strings.
+// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e
+// to avoid certain safety problems that can arise when embedding JSON in HTML.
+//
+// In non-HTML settings where the escaping interferes with the readability
+// of the output, SetEscapeHTML(false) disables this behavior.
+func (enc *Encoder) SetEscapeHTML(on bool) {
+	enc.escapeHTML = on
+}
+
+// RawMessage is a raw encoded JSON value.
+// It implements Marshaler and Unmarshaler and can
+// be used to delay JSON decoding or precompute a JSON encoding.
+type RawMessage []byte
+
+// MarshalJSON returns m as the JSON encoding of m.
+func (m RawMessage) MarshalJSON() ([]byte, error) {
+	if m == nil {
+		return []byte("null"), nil
+	}
+	return m, nil
+}
+
+// UnmarshalJSON sets *m to a copy of data.
+func (m *RawMessage) UnmarshalJSON(data []byte) error {
+	if m == nil {
+		return errors.New("json.RawMessage: UnmarshalJSON on nil pointer")
+	}
+	*m = append((*m)[0:0], data...)
+	return nil
+}
+
+var _ Marshaler = (*RawMessage)(nil)
+var _ Unmarshaler = (*RawMessage)(nil)
+
+// A Token holds a value of one of these types:
+//
+//	Delim, for the four JSON delimiters [ ] { }
+//	bool, for JSON booleans
+//	float64, for JSON numbers
+//	Number, for JSON numbers
+//	string, for JSON string literals
+//	nil, for JSON null
+type Token any
+
+const (
+	tokenTopValue = iota
+	tokenArrayStart
+	tokenArrayValue
+	tokenArrayComma
+	tokenObjectStart
+	tokenObjectKey
+	tokenObjectColon
+	tokenObjectValue
+	tokenObjectComma
+)
+
+// advance tokenstate from a separator state to a value state
+func (dec *Decoder) tokenPrepareForDecode() error {
+	// Note: Not calling peek before switch, to avoid
+	// putting peek into the standard Decode path.
+	// peek is only called when using the Token API.
+	switch dec.tokenState {
+	case tokenArrayComma:
+		c, err := dec.peek()
+		if err != nil {
+			return err
+		}
+		if c != ',' {
+			return &SyntaxError{"expected comma after array element", dec.InputOffset()}
+		}
+		dec.scanp++
+		dec.tokenState = tokenArrayValue
+	case tokenObjectColon:
+		c, err := dec.peek()
+		if err != nil {
+			return err
+		}
+		if c != ':' {
+			return &SyntaxError{"expected colon after object key", dec.InputOffset()}
+		}
+		dec.scanp++
+		dec.tokenState = tokenObjectValue
+	}
+	return nil
+}
+
+func (dec *Decoder) tokenValueAllowed() bool {
+	switch dec.tokenState {
+	case tokenTopValue, tokenArrayStart, tokenArrayValue, tokenObjectValue:
+		return true
+	}
+	return false
+}
+
+func (dec *Decoder) tokenValueEnd() {
+	switch dec.tokenState {
+	case tokenArrayStart, tokenArrayValue:
+		dec.tokenState = tokenArrayComma
+	case tokenObjectValue:
+		dec.tokenState = tokenObjectComma
+	}
+}
+
+// A Delim is a JSON array or object delimiter, one of [ ] { or }.
+type Delim rune
+
+func (d Delim) String() string {
+	return string(d)
+}
+
+// Token returns the next JSON token in the input stream.
+// At the end of the input stream, Token returns nil, io.EOF.
+//
+// Token guarantees that the delimiters [ ] { } it returns are
+// properly nested and matched: if Token encounters an unexpected
+// delimiter in the input, it will return an error.
+//
+// The input stream consists of basic JSON values—bool, string,
+// number, and null—along with delimiters [ ] { } of type Delim
+// to mark the start and end of arrays and objects.
+// Commas and colons are elided.
+func (dec *Decoder) Token() (Token, error) {
+	for {
+		c, err := dec.peek()
+		if err != nil {
+			return nil, err
+		}
+		switch c {
+		case '[':
+			if !dec.tokenValueAllowed() {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
+			dec.tokenState = tokenArrayStart
+			return Delim('['), nil
+
+		case ']':
+			if dec.tokenState != tokenArrayStart && dec.tokenState != tokenArrayComma {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
+			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
+			dec.tokenValueEnd()
+			return Delim(']'), nil
+
+		case '{':
+			if !dec.tokenValueAllowed() {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
+			dec.tokenState = tokenObjectStart
+			return Delim('{'), nil
+
+		case '}':
+			if dec.tokenState != tokenObjectStart && dec.tokenState != tokenObjectComma {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
+			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
+			dec.tokenValueEnd()
+			return Delim('}'), nil
+
+		case ':':
+			if dec.tokenState != tokenObjectColon {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenState = tokenObjectValue
+			continue
+
+		case ',':
+			if dec.tokenState == tokenArrayComma {
+				dec.scanp++
+				dec.tokenState = tokenArrayValue
+				continue
+			}
+			if dec.tokenState == tokenObjectComma {
+				dec.scanp++
+				dec.tokenState = tokenObjectKey
+				continue
+			}
+			return dec.tokenError(c)
+
+		case '"':
+			if dec.tokenState == tokenObjectStart || dec.tokenState == tokenObjectKey {
+				var x string
+				old := dec.tokenState
+				dec.tokenState = tokenTopValue
+				err := dec.Decode(&x)
+				dec.tokenState = old
+				if err != nil {
+					return nil, err
+				}
+				dec.tokenState = tokenObjectColon
+				return x, nil
+			}
+			fallthrough
+
+		default:
+			if !dec.tokenValueAllowed() {
+				return dec.tokenError(c)
+			}
+			var x any
+			if err := dec.Decode(&x); err != nil {
+				return nil, err
+			}
+			return x, nil
+		}
+	}
+}
+
+func (dec *Decoder) tokenError(c byte) (Token, error) {
+	var context string
+	switch dec.tokenState {
+	case tokenTopValue:
+		context = " looking for beginning of value"
+	case tokenArrayStart, tokenArrayValue, tokenObjectValue:
+		context = " looking for beginning of value"
+	case tokenArrayComma:
+		context = " after array element"
+	case tokenObjectKey:
+		context = " looking for beginning of object key string"
+	case tokenObjectColon:
+		context = " after object key"
+	case tokenObjectComma:
+		context = " after object key:value pair"
+	}
+	return nil, &SyntaxError{"invalid character " + quoteChar(c) + context, dec.InputOffset()}
+}
+
+// More reports whether there is another element in the
+// current array or object being parsed.
+func (dec *Decoder) More() bool {
+	c, err := dec.peek()
+	return err == nil && c != ']' && c != '}'
+}
+
+func (dec *Decoder) peek() (byte, error) {
+	var err error
+	for {
+		for i := dec.scanp; i < len(dec.buf); i++ {
+			c := dec.buf[i]
+			if isSpace(c) {
+				continue
+			}
+			dec.scanp = i
+			return c, nil
+		}
+		// buffer has been scanned, now report any error
+		if err != nil {
+			return 0, err
+		}
+		err = dec.refill()
+	}
+}
+
+// InputOffset returns the input stream byte offset of the current decoder position.
+// The offset gives the location of the end of the most recently returned token
+// and the beginning of the next token.
+func (dec *Decoder) InputOffset() int64 {
+	return dec.scanned + int64(dec.scanp)
+}

gojson/stream_test.go

@@ -0,0 +1,474 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Test values for the stream test.
+// One of each JSON kind.
+var streamTest = []any{
+	0.1,
+	"hello",
+	nil,
+	true,
+	false,
+	[]any{"a", "b", "c"},
+	map[string]any{"K": "Kelvin", "ß": "long s"},
+	3.14, // another value to make sure something can follow map
+}
+
+var streamEncoded = `0.1
+"hello"
+null
+true
+false
+["a","b","c"]
+{"ß":"long s","K":"Kelvin"}
+3.14
+`
+
+func TestEncoder(t *testing.T) {
+	for i := 0; i <= len(streamTest); i++ {
+		var buf bytes.Buffer
+		enc := NewEncoder(&buf)
+		// Check that enc.SetIndent("", "") turns off indentation.
+		enc.SetIndent(">", ".")
+		enc.SetIndent("", "")
+		for j, v := range streamTest[0:i] {
+			if err := enc.Encode(v); err != nil {
+				t.Fatalf("encode #%d: %v", j, err)
+			}
+		}
+		if have, want := buf.String(), nlines(streamEncoded, i); have != want {
+			t.Errorf("encoding %d items: mismatch", i)
+			diff(t, []byte(have), []byte(want))
+			break
+		}
+	}
+}
+
+var streamEncodedIndent = `0.1
+"hello"
+null
+true
+false
+[
+>."a",
+>."b",
+>."c"
+>]
+{
+>."ß": "long s",
+>."K": "Kelvin"
+>}
+3.14
+`
+
+func TestEncoderIndent(t *testing.T) {
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	enc.SetIndent(">", ".")
+	for _, v := range streamTest {
+		enc.Encode(v)
+	}
+	if have, want := buf.String(), streamEncodedIndent; have != want {
+		t.Error("indented encoding mismatch")
+		diff(t, []byte(have), []byte(want))
+	}
+}
+
+type strMarshaler string
+
+func (s strMarshaler) MarshalJSON() ([]byte, error) {
+	return []byte(s), nil
+}
+
+type strPtrMarshaler string
+
+func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) {
+	return []byte(*s), nil
+}
+
+func TestEncoderSetEscapeHTML(t *testing.T) {
+	var c C
+	var ct CText
+	var tagStruct struct {
+		Valid   int `json:"<>&#! "`
+		Invalid int `json:"\\"`
+	}
+
+	// This case is particularly interesting, as we force the encoder to
+	// take the address of the Ptr field to use its MarshalJSON method. This
+	// is why the '&' is important.
+	marshalerStruct := &struct {
+		NonPtr strMarshaler
+		Ptr    strPtrMarshaler
+	}{`"<str>"`, `"<str>"`}
+
+	// https://golang.org/issue/34154
+	stringOption := struct {
+		Bar string `json:"bar,string"`
+	}{`<html>foobar</html>`}
+
+	for _, tt := range []struct {
+		name       string
+		v          any
+		wantEscape string
+		want       string
+	}{
+		{"c", c, `"\u003c\u0026\u003e"`, `"<&>"`},
+		{"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`},
+		{`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`},
+		{
+			"tagStruct", tagStruct,
+			`{"\u003c\u003e\u0026#! ":0,"Invalid":0}`,
+			`{"<>&#! ":0,"Invalid":0}`,
+		},
+		{
+			`"<str>"`, marshalerStruct,
+			`{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`,
+			`{"NonPtr":"<str>","Ptr":"<str>"}`,
+		},
+		{
+			"stringOption", stringOption,
+			`{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
+			`{"bar":"\"<html>foobar</html>\""}`,
+		},
+	} {
+		var buf bytes.Buffer
+		enc := NewEncoder(&buf)
+		if err := enc.Encode(tt.v); err != nil {
+			t.Errorf("Encode(%s): %s", tt.name, err)
+			continue
+		}
+		if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
+			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape)
+		}
+		buf.Reset()
+		enc.SetEscapeHTML(false)
+		if err := enc.Encode(tt.v); err != nil {
+			t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
+			continue
+		}
+		if got := strings.TrimSpace(buf.String()); got != tt.want {
+			t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q",
+				tt.name, got, tt.want)
+		}
+	}
+}
+
+func TestDecoder(t *testing.T) {
+	for i := 0; i <= len(streamTest); i++ {
+		// Use stream without newlines as input,
+		// just to stress the decoder even more.
+		// Our test input does not include back-to-back numbers.
+		// Otherwise stripping the newlines would
+		// merge two adjacent JSON values.
+		var buf bytes.Buffer
+		for _, c := range nlines(streamEncoded, i) {
+			if c != '\n' {
+				buf.WriteRune(c)
+			}
+		}
+		out := make([]any, i)
+		dec := NewDecoder(&buf)
+		for j := range out {
+			if err := dec.Decode(&out[j]); err != nil {
+				t.Fatalf("decode #%d/%d: %v", j, i, err)
+			}
+		}
+		if !reflect.DeepEqual(out, streamTest[0:i]) {
+			t.Errorf("decoding %d items: mismatch", i)
+			for j := range out {
+				if !reflect.DeepEqual(out[j], streamTest[j]) {
+					t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j])
+				}
+			}
+			break
+		}
+	}
+}
+
+func TestDecoderBuffered(t *testing.T) {
+	r := strings.NewReader(`{"Name": "Gopher"} extra `)
+	var m struct {
+		Name string
+	}
+	d := NewDecoder(r)
+	err := d.Decode(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m.Name != "Gopher" {
+		t.Errorf("Name = %q; want Gopher", m.Name)
+	}
+	rest, err := io.ReadAll(d.Buffered())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g, w := string(rest), " extra "; g != w {
+		t.Errorf("Remaining = %q; want %q", g, w)
+	}
+}
+
+func nlines(s string, n int) string {
+	if n <= 0 {
+		return ""
+	}
+	for i, c := range s {
+		if c == '\n' {
+			if n--; n == 0 {
+				return s[0 : i+1]
+			}
+		}
+	}
+	return s
+}
+
+func TestRawMessage(t *testing.T) {
+	var data struct {
+		X  float64
+		Id RawMessage
+		Y  float32
+	}
+	const raw = `["\u0056",null]`
+	const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
+	err := Unmarshal([]byte(msg), &data)
+	if err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+	if string([]byte(data.Id)) != raw {
+		t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw)
+	}
+	b, err := Marshal(&data)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if string(b) != msg {
+		t.Fatalf("Marshal: have %#q want %#q", b, msg)
+	}
+}
+
+func TestNullRawMessage(t *testing.T) {
+	var data struct {
+		X     float64
+		Id    RawMessage
+		IdPtr *RawMessage
+		Y     float32
+	}
+	const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}`
+	err := Unmarshal([]byte(msg), &data)
+	if err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+	if want, got := "null", string(data.Id); want != got {
+		t.Fatalf("Raw mismatch: have %q, want %q", got, want)
+	}
+	if data.IdPtr != nil {
+		t.Fatalf("Raw pointer mismatch: have non-nil, want nil")
+	}
+	b, err := Marshal(&data)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if string(b) != msg {
+		t.Fatalf("Marshal: have %#q want %#q", b, msg)
+	}
+}
+
+var blockingTests = []string{
+	`{"x": 1}`,
+	`[1, 2, 3]`,
+}
+
+func TestBlocking(t *testing.T) {
+	for _, enc := range blockingTests {
+		r, w := net.Pipe()
+		go w.Write([]byte(enc))
+		var val any
+
+		// If Decode reads beyond what w.Write writes above,
+		// it will block, and the test will deadlock.
+		if err := NewDecoder(r).Decode(&val); err != nil {
+			t.Errorf("decoding %s: %v", enc, err)
+		}
+		r.Close()
+		w.Close()
+	}
+}
+
+func BenchmarkEncoderEncode(b *testing.B) {
+	b.ReportAllocs()
+	type T struct {
+		X, Y string
+	}
+	v := &T{"foo", "bar"}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			if err := NewEncoder(io.Discard).Encode(v); err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
+type tokenStreamCase struct {
+	json      string
+	expTokens []any
+}
+
+type decodeThis struct {
+	v any
+}
+
+var tokenStreamCases = []tokenStreamCase{
+	// streaming token cases
+	{json: `10`, expTokens: []any{float64(10)}},
+	{json: ` [10] `, expTokens: []any{
+		Delim('['), float64(10), Delim(']')}},
+	{json: ` [false,10,"b"] `, expTokens: []any{
+		Delim('['), false, float64(10), "b", Delim(']')}},
+	{json: `{ "a": 1 }`, expTokens: []any{
+		Delim('{'), "a", float64(1), Delim('}')}},
+	{json: `{"a": 1, "b":"3"}`, expTokens: []any{
+		Delim('{'), "a", float64(1), "b", "3", Delim('}')}},
+	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{
+		Delim('['),
+		Delim('{'), "a", float64(1), Delim('}'),
+		Delim('{'), "a", float64(2), Delim('}'),
+		Delim(']')}},
+	{json: `{"obj": {"a": 1}}`, expTokens: []any{
+		Delim('{'), "obj", Delim('{'), "a", float64(1), Delim('}'),
+		Delim('}')}},
+	{json: `{"obj": [{"a": 1}]}`, expTokens: []any{
+		Delim('{'), "obj", Delim('['),
+		Delim('{'), "a", float64(1), Delim('}'),
+		Delim(']'), Delim('}')}},
+
+	// streaming tokens with intermittent Decode()
+	{json: `{ "a": 1 }`, expTokens: []any{
+		Delim('{'), "a",
+		decodeThis{float64(1)},
+		Delim('}')}},
+	{json: ` [ { "a" : 1 } ] `, expTokens: []any{
+		Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		Delim(']')}},
+	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{
+		Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		decodeThis{map[string]any{"a": float64(2)}},
+		Delim(']')}},
+	{json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []any{
+		Delim('{'), "obj", Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		Delim(']'), Delim('}')}},
+
+	{json: `{"obj": {"a": 1}}`, expTokens: []any{
+		Delim('{'), "obj",
+		decodeThis{map[string]any{"a": float64(1)}},
+		Delim('}')}},
+	{json: `{"obj": [{"a": 1}]}`, expTokens: []any{
+		Delim('{'), "obj",
+		decodeThis{[]any{
+			map[string]any{"a": float64(1)},
+		}},
+		Delim('}')}},
+	{json: ` [{"a": 1} {"a": 2}] `, expTokens: []any{
+		Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		decodeThis{&SyntaxError{"expected comma after array element", 11}},
+	}},
+	{json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []any{
+		Delim('{'), strings.Repeat("a", 513),
+		decodeThis{&SyntaxError{"expected colon after object key", 518}},
+	}},
+	{json: `{ "\a" }`, expTokens: []any{
+		Delim('{'),
+		&SyntaxError{"invalid character 'a' in string escape code", 3},
+	}},
+	{json: ` \a`, expTokens: []any{
+		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
+	}},
+}
+
+func TestDecodeInStream(t *testing.T) {
+	for ci, tcase := range tokenStreamCases {
+
+		dec := NewDecoder(strings.NewReader(tcase.json))
+		for i, etk := range tcase.expTokens {
+
+			var tk any
+			var err error
+
+			if dt, ok := etk.(decodeThis); ok {
+				etk = dt.v
+				err = dec.Decode(&tk)
+			} else {
+				tk, err = dec.Token()
+			}
+			if experr, ok := etk.(error); ok {
+				if err == nil || !reflect.DeepEqual(err, experr) {
+					t.Errorf("case %v: Expected error %#v in %q, but was %#v", ci, experr, tcase.json, err)
+				}
+				break
+			} else if err == io.EOF {
+				t.Errorf("case %v: Unexpected EOF in %q", ci, tcase.json)
+				break
+			} else if err != nil {
+				t.Errorf("case %v: Unexpected error '%#v' in %q", ci, err, tcase.json)
+				break
+			}
+			if !reflect.DeepEqual(tk, etk) {
+				t.Errorf(`case %v: %q @ %v expected %T(%v) was %T(%v)`, ci, tcase.json, i, etk, etk, tk, tk)
+				break
+			}
+		}
+	}
+}
+
+// Test from golang.org/issue/11893
+func TestHTTPDecoding(t *testing.T) {
+	const raw = `{ "foo": "bar" }`
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(raw))
+	}))
+	defer ts.Close()
+	res, err := http.Get(ts.URL)
+	if err != nil {
+		log.Fatalf("GET failed: %v", err)
+	}
+	defer res.Body.Close()
+
+	foo := struct {
+		Foo string
+	}{}
+
+	d := NewDecoder(res.Body)
+	err = d.Decode(&foo)
+	if err != nil {
+		t.Fatalf("Decode: %v", err)
+	}
+	if foo.Foo != "bar" {
+		t.Errorf("decoded %q; want \"bar\"", foo.Foo)
+	}
+
+	// make sure we get the EOF the second time
+	err = d.Decode(&foo)
+	if err != io.EOF {
+		t.Errorf("err = %v; want io.EOF", err)
+	}
+}

gojson/tables.go

@@ -0,0 +1,218 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import "unicode/utf8"
+
+// safeSet holds the value true if the ASCII character with the given array
+// position can be represented inside a JSON string without any further
+// escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), and the backslash character ("\").
+var safeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      true,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      true,
+	'=':      true,
+	'>':      true,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}
+
+// htmlSafeSet holds the value true if the ASCII character with the given
+// array position can be safely represented inside a JSON string, embedded
+// inside of HTML <script> tags, without any additional escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), the backslash character ("\"), HTML opening and closing
+// tags ("<" and ">"), and the ampersand ("&").
+var htmlSafeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      false,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      false,
+	'=':      true,
+	'>':      false,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}

gojson/tagkey_test.go

@@ -0,0 +1,120 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"testing"
+)
+
+type basicLatin2xTag struct {
+	V string `json:"$%-/"`
+}
+
+type basicLatin3xTag struct {
+	V string `json:"0123456789"`
+}
+
+type basicLatin4xTag struct {
+	V string `json:"ABCDEFGHIJKLMO"`
+}
+
+type basicLatin5xTag struct {
+	V string `json:"PQRSTUVWXYZ_"`
+}
+
+type basicLatin6xTag struct {
+	V string `json:"abcdefghijklmno"`
+}
+
+type basicLatin7xTag struct {
+	V string `json:"pqrstuvwxyz"`
+}
+
+type miscPlaneTag struct {
+	V string `json:"色は匂へど"`
+}
+
+type percentSlashTag struct {
+	V string `json:"text/html%"` // https://golang.org/issue/2718
+}
+
+type punctuationTag struct {
+	V string `json:"!#$%&()*+-./:;<=>?@[]^_{|}~ "` // https://golang.org/issue/3546
+}
+
+type dashTag struct {
+	V string `json:"-,"`
+}
+
+type emptyTag struct {
+	W string
+}
+
+type misnamedTag struct {
+	X string `jsom:"Misnamed"`
+}
+
+type badFormatTag struct {
+	Y string `:"BadFormat"`
+}
+
+type badCodeTag struct {
+	Z string `json:" !\"#&'()*+,."`
+}
+
+type spaceTag struct {
+	Q string `json:"With space"`
+}
+
+type unicodeTag struct {
+	W string `json:"Ελλάδα"`
+}
+
+var structTagObjectKeyTests = []struct {
+	raw   any
+	value string
+	key   string
+}{
+	{basicLatin2xTag{"2x"}, "2x", "$%-/"},
+	{basicLatin3xTag{"3x"}, "3x", "0123456789"},
+	{basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"},
+	{basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"},
+	{basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"},
+	{basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"},
+	{miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"},
+	{dashTag{"foo"}, "foo", "-"},
+	{emptyTag{"Pour Moi"}, "Pour Moi", "W"},
+	{misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"},
+	{badFormatTag{"Orfevre"}, "Orfevre", "Y"},
+	{badCodeTag{"Reliable Man"}, "Reliable Man", "Z"},
+	{percentSlashTag{"brut"}, "brut", "text/html%"},
+	{punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:;<=>?@[]^_{|}~ "},
+	{spaceTag{"Perreddu"}, "Perreddu", "With space"},
+	{unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"},
+}
+
+func TestStructTagObjectKey(t *testing.T) {
+	for _, tt := range structTagObjectKeyTests {
+		b, err := Marshal(tt.raw)
+		if err != nil {
+			t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err)
+		}
+		var f any
+		err = Unmarshal(b, &f)
+		if err != nil {
+			t.Fatalf("Unmarshal(%#q) failed: %v", b, err)
+		}
+		for i, v := range f.(map[string]any) {
+			switch i {
+			case tt.key:
+				if s, ok := v.(string); !ok || s != tt.value {
+					t.Fatalf("Unexpected value: %#q, want %v", s, tt.value)
+				}
+			default:
+				t.Fatalf("Unexpected key: %#q, from %#q", i, b)
+			}
+		}
+	}
+}

gojson/tags.go

@@ -0,0 +1,38 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"strings"
+)
+
+// tagOptions is the string following a comma in a struct field's "json"
+// tag, or the empty string. It does not include the leading comma.
+type tagOptions string
+
+// parseTag splits a struct field's json tag into its name and
+// comma-separated options.
+func parseTag(tag string) (string, tagOptions) {
+	tag, opt, _ := strings.Cut(tag, ",")
+	return tag, tagOptions(opt)
+}
+
+// Contains reports whether a comma-separated list of options
+// contains a particular substr flag. substr must be surrounded by a
+// string boundary or commas.
+func (o tagOptions) Contains(optionName string) bool {
+	if len(o) == 0 {
+		return false
+	}
+	s := string(o)
+	for s != "" {
+		var name string
+		name, s, _ = strings.Cut(s, ",")
+		if name == optionName {
+			return true
+		}
+	}
+	return false
+}

gojson/tags_test.go

@@ -0,0 +1,28 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"testing"
+)
+
+func TestTagParsing(t *testing.T) {
+	name, opts := parseTag("field,foobar,foo")
+	if name != "field" {
+		t.Fatalf("name = %q, want field", name)
+	}
+	for _, tt := range []struct {
+		opt  string
+		want bool
+	}{
+		{"foobar", true},
+		{"foo", true},
+		{"bar", false},
+	} {
+		if opts.Contains(tt.opt) != tt.want {
+			t.Errorf("Contains(%q) = %v", tt.opt, !tt.want)
+		}
+	}
+}

googleapi/README.md

@@ -0,0 +1,54 @@
+
+Google OAuth Setup (to send mails)
+==================================
+
+
+ - Login @ https://console.cloud.google.com
+
+ - GMail API akivieren: https://console.cloud.google.com/apis/library/gmail.googleapis.com?
+
+ - Create new Project (aka 'BackendMailAPI') @ https://console.cloud.google.com/projectcreate
+   User Type: Intern
+   Anwendungsname: 'BackendMailAPI'
+   Support-Email: ...
+   Authorisierte Domains: 'heydyno.de' (or project domain)
+   Kontakt-Email: ...
+   
+
+ - Unter "Anmeldedaten" neuer OAuth Client erstellen @ https://console.cloud.google.com/apis/credentials
+   Anwendungstyp: Web
+   Name: 'BackendMailOAuth'
+   Redirect-Uri: 'http://localhost/oauth'
+   Client-ID und Client-Key merken
+
+ - Open in Browser:
+   https://accounts.google.com/o/oauth2/v2/auth?redirect_uri=http://localhost/oauth&prompt=consent&response_type=code&client_id={...}&scope=https://www.googleapis.com/auth/gmail.send&access_type=offline
+   Code aus redirected URI merken
+
+ - Code via request einlösen (und refresh_roken merken):
+
+```
+curl --request POST \
+  --url https://oauth2.googleapis.com/token \
+  --data code={...} \
+  --data redirect_uri=http://localhost/oauth \
+  --data client_id={...} \
+  --data client_secret={...} \
+  --data grant_type=authorization_code \
+  --data scope=https://www.googleapis.com/auth/gmail.send
+```
+
+ - Fertig, mit `client_id`, `client_secret` und `refresh_token` kann das package benutzt werden
+
+
+
+
+
+
+
+
+
+
+
+
+

googleapi/attachment.go

@@ -0,0 +1,46 @@
+package googleapi
+
+import (
+	"encoding/base64"
+	"fmt"
+)
+
+type MailAttachment struct {
+	IsInline    bool
+	ContentType string
+	Filename    string
+	Data        []byte
+}
+
+func (a MailAttachment) dump() []string {
+	res := make([]string, 0, 4)
+
+	if a.ContentType != "" {
+		res = append(res, "Content-Type: "+a.ContentType+"; charset=UTF-8")
+	}
+
+	res = append(res, "Content-Transfer-Encoding: base64")
+
+	if a.IsInline {
+		if a.Filename != "" {
+			res = append(res, fmt.Sprintf("Content-Disposition: inline;filename=\"%s\"", a.Filename))
+		} else {
+			res = append(res, "Content-Disposition: inline")
+		}
+	} else {
+		if a.Filename != "" {
+			res = append(res, fmt.Sprintf("Content-Disposition: attachment;filename=\"%s\"", a.Filename))
+		} else {
+			res = append(res, "Content-Disposition: attachment")
+		}
+	}
+
+	b64 := base64.StdEncoding.EncodeToString(a.Data)
+	for i := 0; i < len(b64); i += 80 {
+		res = append(res, b64[i:min(i+80, len(b64))])
+	}
+
+	res = append(res)
+
+	return res
+}

googleapi/body.go

@@ -0,0 +1,6 @@
+package googleapi
+
+type MailBody struct {
+	Plain string
+	HTML  string
+}

googleapi/mimeMessage.go

@@ -0,0 +1,224 @@
+package googleapi
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"mime"
+	"strings"
+	"time"
+)
+
+// https://datatracker.ietf.org/doc/html/rfc2822
+func encodeMimeMail(from string, recipients []string, cc []string, bcc []string, subject string, body MailBody, attachments []MailAttachment) string {
+
+	data := make([]string, 0, 32)
+
+	data = append(data, "Date: "+time.Now().Format(time.RFC1123Z))
+	data = append(data, "MIME-Version: 1.0")
+	data = append(data, "From: "+mime.QEncoding.Encode("UTF-8", from))
+	data = append(data, "To: "+strings.Join(langext.ArrMap(recipients, func(v string) string { return mime.QEncoding.Encode("UTF-8", v) }), ", "))
+	if len(cc) > 0 {
+		data = append(data, "To: "+strings.Join(langext.ArrMap(cc, func(v string) string { return mime.QEncoding.Encode("UTF-8", v) }), ", "))
+	}
+	if len(bcc) > 0 {
+		data = append(data, "Bcc: "+strings.Join(langext.ArrMap(bcc, func(v string) string { return mime.QEncoding.Encode("UTF-8", v) }), ", "))
+	}
+	data = append(data, "Subject: "+mime.QEncoding.Encode("UTF-8", subject))
+
+	hasInlineAttachments := langext.ArrAny(attachments, func(v MailAttachment) bool { return v.IsInline })
+	hasNormalAttachments := langext.ArrAny(attachments, func(v MailAttachment) bool { return !v.IsInline })
+	hasPlain := body.Plain != ""
+	hasHTML := body.HTML != ""
+
+	mixedBoundary := langext.MustRawHexUUID()
+	relatedBoundary := langext.MustRawHexUUID()
+	altBoundary := langext.MustRawHexUUID()
+
+	inlineAttachments := langext.ArrFilter(attachments, func(v MailAttachment) bool { return v.IsInline })
+	normalAttachments := langext.ArrFilter(attachments, func(v MailAttachment) bool { return !v.IsInline })
+
+	if hasInlineAttachments && hasNormalAttachments {
+		// "mixed+related"
+
+		data = append(data, "Content-Type: multipart/mixed; boundary="+mixedBoundary)
+		data = append(data, "")
+		data = append(data, "--"+mixedBoundary)
+
+		data = append(data, "Content-Type: multipart/related; boundary="+relatedBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, relatedBoundary, altBoundary)...)
+		data = append(data, "")
+
+		for i, attachment := range inlineAttachments {
+			data = append(data, "--"+relatedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(inlineAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+relatedBoundary+"--")
+
+		for i, attachment := range normalAttachments {
+			data = append(data, "--"+mixedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(normalAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+mixedBoundary+"--")
+
+	} else if hasNormalAttachments {
+		// "mixed"
+
+		data = append(data, "Content-Type: multipart/mixed; boundary="+mixedBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, mixedBoundary, altBoundary)...)
+		if hasPlain && hasHTML {
+			data = append(data, "")
+		}
+
+		for i, attachment := range normalAttachments {
+			data = append(data, "--"+mixedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(normalAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+mixedBoundary+"--")
+
+	} else if hasInlineAttachments {
+		// "related"
+
+		data = append(data, "Content-Type: multipart/related; boundary="+relatedBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, relatedBoundary, altBoundary)...)
+		data = append(data, "")
+
+		for i, attachment := range inlineAttachments {
+			data = append(data, "--"+relatedBoundary)
+			data = append(data, attachment.dump()...)
+
+			if i < len(inlineAttachments)-1 {
+				data = append(data, "")
+			}
+		}
+
+		data = append(data, "--"+relatedBoundary+"--")
+
+	} else if hasPlain && hasHTML {
+		// "alternative"
+
+		data = append(data, "Content-Type: multipart/alternative; boundary="+altBoundary)
+		data = append(data, "")
+
+		data = append(data, dumpMailBody(body, hasInlineAttachments, hasNormalAttachments, altBoundary, altBoundary)...)
+		data = append(data, "")
+
+		data = append(data, "--"+altBoundary+"--")
+
+	} else if hasPlain {
+		// "plain"
+
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+
+	} else if hasHTML {
+		// "plain"
+
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+
+	} else {
+		// "empty??"
+
+	}
+
+	return strings.Join(data, "\r\n")
+}
+
+func dumpMailBody(body MailBody, hasInlineAttachments bool, hasNormalAttachments bool, boundary string, boundaryAlt string) []string {
+
+	if body.HTML != "" && body.Plain != "" && !hasInlineAttachments && hasNormalAttachments {
+		data := make([]string, 0, 16)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: multipart/alternative; boundary="+boundaryAlt)
+		data = append(data, "")
+		data = append(data, "--"+boundaryAlt)
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+		data = append(data, "")
+		data = append(data, "--"+boundaryAlt)
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+		data = append(data, "")
+		data = append(data, "--"+boundaryAlt+"--")
+		return data
+	}
+
+	if body.HTML != "" && body.Plain != "" && hasInlineAttachments {
+		data := make([]string, 0, 2)
+		data = append(data, "--"+boundary)
+		data = append(data, body.HTML)
+		return data
+	}
+
+	if body.HTML != "" && body.Plain != "" {
+		data := make([]string, 0, 8)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+		data = append(data, "")
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+		return data
+	}
+
+	if body.HTML != "" {
+		data := make([]string, 0, 2)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/html; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.HTML)
+		return data
+	}
+
+	if body.Plain != "" {
+		data := make([]string, 0, 2)
+		data = append(data, "--"+boundary)
+		data = append(data, "Content-Type: text/plain; charset=UTF-8")
+		data = append(data, "Content-Transfer-Encoding: 7bit")
+		data = append(data, "")
+		data = append(data, body.Plain)
+		return data
+	}
+
+	data := make([]string, 0, 16)
+	data = append(data, "--"+boundary)
+	data = append(data, "Content-Type: text/plain; charset=UTF-8")
+	data = append(data, "Content-Transfer-Encoding: 7bit")
+	data = append(data, "")
+	data = append(data, "") // no content ?!?
+	return data
+}