v0.0.247 -.-

(cherry picked from commit c0443af63b)

.gitea/workflows/Dockerfile_tests

@@ -0,0 +1,9 @@
+FROM golang:latest
+
+RUN apt install -y make curl python3 && go install gotest.tools/gotestsum@latest
+
+COPY . /source
+
+WORKDIR /source
+
+CMD ["make", "test"]

.gitea/workflows/tests.yml

@@ -0,0 +1,30 @@
+
+# https://docs.gitea.com/next/usage/actions/quickstart
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
+
+name: Build Docker and Deploy
+run-name: Build & Deploy ${{ gitea.ref }} on ${{ gitea.actor }}
+
+on: [push]
+
+jobs:
+  run_tests:
+    name: Run goext test-suite
+    runs-on: bfb-cicd-latest
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Build test docker
+        id: build_docker
+        run: echo "DOCKER_IMG_ID=$(docker build -q . -f .gitea/workflows/Dockerfile_tests || echo __err_build__)" >> $GITHUB_OUTPUT
+
+      - name: Run tests
+        run: docker run --rm "${{ steps.build_docker.outputs.DOCKER_IMG_ID }}"
+
+      - name: Cleanup
+        if: always()
+        run: docker image rm "${{ steps.build_docker.outputs.DOCKER_IMG_ID }}"
+

.idea/goext.iml

@@ -1,6 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="WEB_MODULE" version="4">
-  <component name="Go" enabled="true" />
+  <component name="Go" enabled="true">
+    <buildTags>
+      <option name="goVersion" value="1.19" />
+    </buildTags>
+  </component>
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
     <orderEntry type="inheritedJdk" />

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +1,7 @@
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="GoMixedReceiverTypes" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
     <inspection_tool class="LanguageDetectionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
     <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
       <option name="processCode" value="true" />

Makefile

@@ -3,7 +3,9 @@ run:
 	echo "This is a library - can't be run" && false
 
 test:
-	go test ./...
+	# go test ./...
+	which gotestsum || go install gotest.tools/gotestsum@latest
+	gotestsum --format "testname" -- -tags="timetzdata sqlite_fts5 sqlite_foreign_keys" "./..."
 
 version:
 	_data/version.sh

README.md

@@ -5,4 +5,37 @@ A collection of general & useful library methods
 
 This should not have any heavy dependencies (gin, mongo, etc) and add missing basic language features...
 
-Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
+Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
+
+
+### Packages:
+
+| Name         | Maintainer | Description                                                                                                   |
+|--------------|------------|---------------------------------------------------------------------------------------------------------------|
+| langext      | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
+| mathext      | Mike       | Utility/Helper functions for math                                                                             |
+| cryptext     | Mike       | Utility/Helper functions for encryption                                                                       |
+| syncext      | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
+| dataext      | Mike       | Various useful data structures                                                                                |
+| zipext       | Mike       | Utility for zip/gzip/tar etc                                                                                  |
+| reflectext   | Mike       | Utility for golagn reflection                                                                                 |
+|              |            |                                                                                                               |
+| mongoext     | Mike       | Utility/Helper functions for mongodb                                                                          |
+| cursortoken  | Mike       | MongoDB cursortoken implementation                                                                            |
+|              |            |                                                                                                               |
+| totpext      | Mike       | Implementation of TOTP (2-Factor-Auth)                                                                        |
+| termext      | Mike       | Utilities for terminals (mostly color output)                                                                 |
+| confext      | Mike       | Parses environment configuration into structs                                                                 |
+| cmdext       | Mike       | Runner for external commands/processes                                                                        |
+|              |            |                                                                                                               |
+| sq           | Mike       | Utility functions for sql based databases                                                                     |
+| tst          | Mike       | Utility functions for unit tests                                                                              |
+|              |            |                                                                                                               |
+| rfctime      | Mike       | Classes for time seriallization, with different marshallign method for mongo and json                         |
+| gojson       | Mike       | Same interface for marshalling/unmarshalling as go/json, except with proper serialization of null arrays/maps |
+|              |            |                                                                                                               |
+| bfcodegen    | Mike       | Various codegen tools (run via go generate)                                                                   |
+|              |            |                                                                                                               |
+| rext         | Mike       | Regex Wrapper, wraps regexp with a better interface                                                           |
+| wmo          | Mike       | Mongo Wrapper, wraps mongodb with a better interface                                                          |
+|              |            |                                                                                                               |

TODO.md

@@ -0,0 +1,13 @@
+
+
+ - cronext
+
+ - cursortoken
+
+ - typed/geenric mongo wrapper
+
+ - error package
+
+- rfctime.DateOnly
+- rfctime.HMSTimeOnly
+- rfctime.NanoTimeOnly

_data/version.sh

@@ -7,6 +7,29 @@ set -o pipefail  # Return value of a pipeline is the value of the last (rightmos
 IFS=$'\n\t'      # Set $IFS to only newline and tab.
 
 
+function black() { echo -e "\x1B[30m $1 \x1B[0m"; }
+function red()   { echo -e "\x1B[31m $1 \x1B[0m"; }
+function green() { echo -e "\x1B[32m $1 \x1B[0m"; }
+function yellow(){ echo -e "\x1B[33m $1 \x1B[0m"; }
+function blue()  { echo -e "\x1B[34m $1 \x1B[0m"; }
+function purple(){ echo -e "\x1B[35m $1 \x1B[0m"; }
+function cyan()  { echo -e "\x1B[36m $1 \x1B[0m"; }
+function white() { echo -e "\x1B[37m $1 \x1B[0m"; }
+
+if [ "$( git rev-parse --abbrev-ref HEAD )" != "master" ]; then
+  >&2 red "[ERROR] Can only create versions of <master>"
+  exit 1
+fi
+
+echo ""
+echo -n "Insert optional commit message: "
+read commitMessage
+echo ""
+
+git pull --ff
+
+go get -u ./...
+
 curr_vers=$(git describe --tags --abbrev=0 | sed 's/v//g')
 
 next_ver=$(echo "$curr_vers" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{if(length($NF+1)>length($NF))$(NF-1)++; $NF=sprintf("%0*d", length($NF), ($NF+1)%(10^length($NF))); print}')
@@ -16,9 +39,22 @@ echo "> Current Version: ${curr_vers}"
 echo "> Next    Version: ${next_ver}"
 echo ""
 
+printf "package goext\n\nconst GoextVersion = \"%s\"\n\nconst GoextVersionTimestamp = \"%s\"\n" "${next_ver}" "$( date +"%Y-%m-%dT%H:%M:%S%z" )" > "goextVersion.go"
+
 git add --verbose .
 
-git commit -a -m "v${next_ver}"
+msg="v${next_ver}"
+
+if [[ "$commitMessage" != "" ]]; then
+  msg="${msg} ${commitMessage}"
+fi
+
+
+if [ $# -gt 0 ]; then
+  msg="$1"
+fi
+
+git commit -a -m "${msg}"
 
 git tag "v${next_ver}"
 

bfcodegen/enum-generate.go

@@ -0,0 +1,335 @@
+package bfcodegen
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type EnumDefVal struct {
+	VarName     string
+	Value       string
+	Description *string
+}
+
+type EnumDef struct {
+	File         string
+	FileRelative string
+	EnumTypeName string
+	Type         string
+	Values       []EnumDefVal
+}
+
+var rexPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
+
+var rexValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[A-Za-z0-9_:]+"|[0-9]+))\s*(//(?P<descr>.*))?.*$`))
+
+var rexChecksumConst = rext.W(regexp.MustCompile(`const ChecksumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+func GenerateEnumSpecs(sourceDir string, destFile string) error {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[EnumGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[EnumGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allEnums := make([]EnumDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileEnums, pn, err := processFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\n")
+
+		allEnums = append(allEnums, fileEnums...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	err = os.WriteFile(destFile, []byte(fmtOutput(newChecksum, allEnums, pkgname)), 0o755)
+	if err != nil {
+		return err
+	}
+
+	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
+	if err != nil {
+		return err
+	}
+
+	if res.CommandTimedOut {
+		fmt.Println(res.StdCombined)
+		return errors.New("go fmt timed out")
+	}
+	if res.ExitCode != 0 {
+		fmt.Println(res.StdCombined)
+		return errors.New("go fmt did not succeed")
+	}
+
+	return nil
+}
+
+func processFile(basedir string, fn string) ([]EnumDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	enums := make([]EnumDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexEnumDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := EnumDef{
+				File:         fn,
+				FileRelative: rfp,
+				EnumTypeName: match.GroupByName("name").Value(),
+				Type:         match.GroupByName("type").Value(),
+				Values:       make([]EnumDefVal, 0),
+			}
+			enums = append(enums, def)
+			fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+		}
+
+		if match, ok := rexValueDef.MatchFirst(line); ok {
+			typename := match.GroupByName("type").Value()
+			def := EnumDefVal{
+				VarName:     match.GroupByName("name").Value(),
+				Value:       match.GroupByName("value").Value(),
+				Description: match.GroupByNameOrEmpty("descr").ValueOrNil(),
+			}
+
+			found := false
+			for i, v := range enums {
+				if v.EnumTypeName == typename {
+					enums[i].Values = append(enums[i].Values, def)
+					found = true
+					if def.Description != nil {
+						fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
+					} else {
+						fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+					}
+					break
+				}
+			}
+			if !found {
+				fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+			}
+		}
+	}
+
+	return enums, pkgname, nil
+}
+
+func fmtOutput(cs string, enums []EnumDef, pkgname string) string {
+	str := "// Code generated by enum-generate.go DO NOT EDIT.\n"
+	str += "\n"
+	str += "package " + pkgname + "\n"
+	str += "\n"
+
+	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/langext\"" + "\n"
+	str += "import \"gogs.mikescher.com/BlackForestBytes/goext/enums\"" + "\n"
+	str += "\n"
+
+	str += "const ChecksumGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
+	str += "\n"
+
+	for _, enumdef := range enums {
+
+		hasDescr := langext.ArrAll(enumdef.Values, func(val EnumDefVal) bool { return val.Description != nil })
+		hasStr := enumdef.Type == "string"
+
+		str += "// ================================ " + enumdef.EnumTypeName + " ================================" + "\n"
+		str += "//" + "\n"
+		str += "// File:       " + enumdef.FileRelative + "\n"
+		str += "// StringEnum: " + langext.Conditional(hasStr, "true", "false") + "\n"
+		str += "// DescrEnum:  " + langext.Conditional(hasDescr, "true", "false") + "\n"
+		str += "//" + "\n"
+		str += "" + "\n"
+
+		str += "var __" + enumdef.EnumTypeName + "Values = []" + enumdef.EnumTypeName + "{" + "\n"
+		for _, v := range enumdef.Values {
+			str += "    " + v.VarName + "," + "\n"
+		}
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		if hasDescr {
+			str += "var __" + enumdef.EnumTypeName + "Descriptions = map[" + enumdef.EnumTypeName + "]string{" + "\n"
+			for _, v := range enumdef.Values {
+				str += "    " + v.VarName + ": \"" + strings.TrimSpace(*v.Description) + "\"," + "\n"
+			}
+			str += "}" + "\n"
+			str += "" + "\n"
+		}
+
+		str += "var __" + enumdef.EnumTypeName + "Varnames = map[" + enumdef.EnumTypeName + "]string{" + "\n"
+		for _, v := range enumdef.Values {
+			str += "    " + v.VarName + ": \"" + v.VarName + "\"," + "\n"
+		}
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func (e " + enumdef.EnumTypeName + ") Valid() bool {" + "\n"
+		str += "    return langext.InArray(e, __" + enumdef.EnumTypeName + "Values)" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func (e " + enumdef.EnumTypeName + ") Values() []" + enumdef.EnumTypeName + " {" + "\n"
+		str += "    return __" + enumdef.EnumTypeName + "Values" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func (e " + enumdef.EnumTypeName + ") ValuesAny() []any {" + "\n"
+		str += "    return langext.ArrCastToAny(__" + enumdef.EnumTypeName + "Values)" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func (e " + enumdef.EnumTypeName + ") ValuesMeta() []enums.EnumMetaValue {" + "\n"
+		str += "    return " + enumdef.EnumTypeName + "ValuesMeta()"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		if hasStr {
+			str += "func (e " + enumdef.EnumTypeName + ") String() string {" + "\n"
+			str += "    return string(e)" + "\n"
+			str += "}" + "\n"
+			str += "" + "\n"
+		}
+
+		if hasDescr {
+			str += "func (e " + enumdef.EnumTypeName + ") Description() string {" + "\n"
+			str += "    if d, ok := __" + enumdef.EnumTypeName + "Descriptions[e]; ok {" + "\n"
+			str += "        return d" + "\n"
+			str += "    }" + "\n"
+			str += "    return \"\"" + "\n"
+			str += "}" + "\n"
+			str += "" + "\n"
+		}
+
+		str += "func (e " + enumdef.EnumTypeName + ") VarName() string {" + "\n"
+		str += "    if d, ok := __" + enumdef.EnumTypeName + "Varnames[e]; ok {" + "\n"
+		str += "        return d" + "\n"
+		str += "    }" + "\n"
+		str += "    return \"\"" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func (e " + enumdef.EnumTypeName + ") Meta() enums.EnumMetaValue {" + "\n"
+		if hasDescr {
+			str += "    return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())}"
+		} else {
+			str += "    return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil}"
+		}
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func Parse" + enumdef.EnumTypeName + "(vv string) (" + enumdef.EnumTypeName + ", bool) {" + "\n"
+		str += "    for _, ev := range __" + enumdef.EnumTypeName + "Values {" + "\n"
+		str += "        if string(ev) == vv {" + "\n"
+		str += "            return ev, true" + "\n"
+		str += "        }" + "\n"
+		str += "    }" + "\n"
+		str += "    return \"\", false" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func " + enumdef.EnumTypeName + "Values() []" + enumdef.EnumTypeName + " {" + "\n"
+		str += "    return __" + enumdef.EnumTypeName + "Values" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+		str += "func " + enumdef.EnumTypeName + "ValuesMeta() []enums.EnumMetaValue {" + "\n"
+		str += "    return []enums.EnumMetaValue{" + "\n"
+		for _, v := range enumdef.Values {
+			str += "        " + v.VarName + ".Meta(),\n"
+		}
+		str += "    }" + "\n"
+		str += "}" + "\n"
+		str += "" + "\n"
+
+	}
+
+	return str
+}

bfcodegen/enum-generate_test.go

@@ -0,0 +1,42 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example.tgz
+var ExampleModels []byte
+
+func TestGenerateEnumSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, ExampleModels, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateEnumSpecs(tmpDir, tmpDir+"/enums_gen.go")
+	tst.AssertNoErr(t, err)
+
+	err = GenerateEnumSpecs(tmpDir, tmpDir+"/enums_gen.go")
+	tst.AssertNoErr(t, err)
+
+}

cmdext/builder.go

@@ -0,0 +1,100 @@
+package cmdext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"time"
+)
+
+type CommandRunner struct {
+	program          string
+	args             []string
+	timeout          *time.Duration
+	env              []string
+	listener         []CommandListener
+	enforceExitCodes *[]int
+	enforceNoTimeout bool
+	enforceNoStderr  bool
+}
+
+func Runner(program string) *CommandRunner {
+	return &CommandRunner{
+		program:          program,
+		args:             make([]string, 0),
+		timeout:          nil,
+		env:              make([]string, 0),
+		listener:         make([]CommandListener, 0),
+		enforceExitCodes: nil,
+		enforceNoTimeout: false,
+		enforceNoStderr:  false,
+	}
+}
+
+func (r *CommandRunner) Arg(arg string) *CommandRunner {
+	r.args = append(r.args, arg)
+	return r
+}
+
+func (r *CommandRunner) Args(arg []string) *CommandRunner {
+	r.args = append(r.args, arg...)
+	return r
+}
+
+func (r *CommandRunner) Timeout(timeout time.Duration) *CommandRunner {
+	r.timeout = &timeout
+	return r
+}
+
+func (r *CommandRunner) Env(key, value string) *CommandRunner {
+	r.env = append(r.env, fmt.Sprintf("%s=%s", key, value))
+	return r
+}
+
+func (r *CommandRunner) RawEnv(env string) *CommandRunner {
+	r.env = append(r.env, env)
+	return r
+}
+
+func (r *CommandRunner) Envs(env []string) *CommandRunner {
+	r.env = append(r.env, env...)
+	return r
+}
+
+func (r *CommandRunner) EnsureExitcode(arg ...int) *CommandRunner {
+	r.enforceExitCodes = langext.Ptr(langext.ForceArray(arg))
+	return r
+}
+
+func (r *CommandRunner) FailOnExitCode() *CommandRunner {
+	r.enforceExitCodes = langext.Ptr([]int{0})
+	return r
+}
+
+func (r *CommandRunner) FailOnTimeout() *CommandRunner {
+	r.enforceNoTimeout = true
+	return r
+}
+
+func (r *CommandRunner) FailOnStderr() *CommandRunner {
+	r.enforceNoStderr = true
+	return r
+}
+
+func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
+	r.listener = append(r.listener, lstr)
+	return r
+}
+
+func (r *CommandRunner) ListenStdout(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStdoutLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) ListenStderr(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStderrLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) Run() (CommandResult, error) {
+	return run(*r)
+}

cmdext/cmdrunner.go

@@ -0,0 +1,198 @@
+package cmdext
+
+import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
+	"os/exec"
+	"time"
+)
+
+var ErrExitCode = errors.New("process exited with an unexpected exitcode")
+var ErrTimeout = errors.New("process did not exit after the specified timeout")
+var ErrStderrPrint = errors.New("process did print to stderr stream")
+
+type CommandResult struct {
+	StdOut          string
+	StdErr          string
+	StdCombined     string
+	ExitCode        int
+	CommandTimedOut bool
+}
+
+func run(opt CommandRunner) (CommandResult, error) {
+	cmd := exec.Command(opt.program, opt.args...)
+
+	cmd.Env = append(cmd.Env, opt.env...)
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	preader := pipeReader{
+		lineBufferSize: langext.Ptr(128 * 1024 * 1024), // 128MB max size of a single line, is hopefully enough....
+		stdout:         stdoutPipe,
+		stderr:         stderrPipe,
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	type resultObj struct {
+		stdout      string
+		stderr      string
+		stdcombined string
+		err         error
+	}
+
+	stderrFailChan := make(chan bool)
+
+	outputChan := make(chan resultObj)
+	go func() {
+		// we need to first fully read the pipes and then call Wait
+		// see https://pkg.go.dev/os/exec#Cmd.StdoutPipe
+
+		listener := make([]CommandListener, 0)
+		listener = append(listener, opt.listener...)
+
+		if opt.enforceNoStderr {
+			listener = append(listener, genericCommandListener{
+				_readRawStderr: langext.Ptr(func(v []byte) {
+					if len(v) > 0 {
+						stderrFailChan <- true
+					}
+				}),
+			})
+		}
+
+		stdout, stderr, stdcombined, err := preader.Read(listener)
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+			_ = cmd.Process.Kill()
+			return
+		}
+
+		err = cmd.Wait()
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+		} else {
+			outputChan <- resultObj{stdout, stderr, stdcombined, nil}
+		}
+
+	}()
+
+	var timeoutChan <-chan time.Time = make(chan time.Time, 1)
+	if opt.timeout != nil {
+		timeoutChan = time.After(*opt.timeout)
+	}
+
+	select {
+
+	case <-timeoutChan:
+		_ = cmd.Process.Kill()
+		for _, lstr := range opt.listener {
+			lstr.Timeout()
+		}
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
+			// most of the time the cmd.Process.Kill() should also ahve finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		}
+
+	case <-stderrFailChan:
+		_ = cmd.Process.Kill()
+		for _, lstr := range opt.listener {
+			lstr.Timeout()
+		}
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, 32*time.Millisecond); ok {
+			// most of the time the cmd.Process.Kill() should also have finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
+				ExitCode:        -1,
+				CommandTimedOut: false,
+			}
+			return res, ErrStderrPrint
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: false,
+			}
+			return res, ErrStderrPrint
+		}
+
+	case outobj := <-outputChan:
+		if exiterr, ok := outobj.err.(*exec.ExitError); ok {
+			excode := exiterr.ExitCode()
+			for _, lstr := range opt.listener {
+				lstr.Finished(excode)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        excode,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(excode, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		} else if err != nil {
+			return CommandResult{}, err
+		} else {
+			for _, lstr := range opt.listener {
+				lstr.Finished(0)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        0,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(0, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		}
+	}
+}

cmdext/cmdrunner_test.go

@@ -0,0 +1,348 @@
+package cmdext
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestStdout(t *testing.T) {
+
+	res1, err := Runner("printf").Arg("hello").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "hello" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "hello\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStdcombined(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"1\", file=sys.stderr, flush=True); time.sleep(0.1); print(\"2\", file=sys.stdout, flush=True); time.sleep(0.1); print(\"3\", file=sys.stderr, flush=True)").
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "1\n3\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "2\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "1\n2\n3\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialRead(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", flush=True); time.sleep(5); print(\"cant see me\", flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadStderr(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, flush=True); time.sleep(5); print(\"cant see me\", file=sys.stderr, flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStdout(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stdout, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "message101" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "message101" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushed(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", end=''); time.sleep(5); print(\"cant see me\", end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushedStderr(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, end=''); time.sleep(5); print(\"cant see me\", file=sys.stderr, end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestListener(t *testing.T) {
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys;" +
+			"import time;" +
+			"print(\"message 1\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 2\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 3\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 4\", file=sys.stderr, flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 5\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"final\");").
+		ListenStdout(func(s string) { fmt.Printf("@@STDOUT <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		ListenStderr(func(s string) { fmt.Printf("@@STDERR <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		Timeout(10 * time.Second).
+		Run()
+
+	if err != nil {
+		panic(err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+}
+
+func TestLongStdout(t *testing.T) {
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"X\" * 125001 + \"\\n\"); print(\"Y\" * 125001 + \"\\n\"); print(\"Z\" * 125001 + \"\\n\");").
+		Timeout(5000 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if len(res1.StdOut) != 375009 {
+		t.Errorf("len(res1.StdOut) == '%v'", len(res1.StdOut))
+	}
+
+}
+
+func TestFailOnTimeout(t *testing.T) {
+
+	_, err := Runner("sleep").Arg("2").Timeout(200 * time.Millisecond).FailOnTimeout().Run()
+	if !errors.Is(err, ErrTimeout) {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestFailOnStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").FailOnStderr().Run()
+	if err == nil {
+		t.Errorf("no err")
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != -1 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestFailOnExitcode(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).FailOnExitCode().Run()
+	if !errors.Is(err, ErrExitCode) {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode1(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).EnsureExitcode(1).Run()
+	if err != nil {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode2(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200*time.Millisecond).EnsureExitcode(0, 2, 3).Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}

cmdext/helper.go

@@ -0,0 +1,12 @@
+package cmdext
+
+import "time"
+
+func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+	b := Runner(program)
+	b = b.Args(args)
+	if timeout != nil {
+		b = b.Timeout(*timeout)
+	}
+	return b.Run()
+}

cmdext/listener.go

@@ -0,0 +1,57 @@
+package cmdext
+
+type CommandListener interface {
+	ReadRawStdout([]byte)
+	ReadRawStderr([]byte)
+
+	ReadStdoutLine(string)
+	ReadStderrLine(string)
+
+	Finished(int)
+	Timeout()
+}
+
+type genericCommandListener struct {
+	_readRawStdout  *func([]byte)
+	_readRawStderr  *func([]byte)
+	_readStdoutLine *func(string)
+	_readStderrLine *func(string)
+	_finished       *func(int)
+	_timeout        *func()
+}
+
+func (g genericCommandListener) ReadRawStdout(v []byte) {
+	if g._readRawStdout != nil {
+		(*g._readRawStdout)(v)
+	}
+}
+
+func (g genericCommandListener) ReadRawStderr(v []byte) {
+	if g._readRawStderr != nil {
+		(*g._readRawStderr)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStdoutLine(v string) {
+	if g._readStdoutLine != nil {
+		(*g._readStdoutLine)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStderrLine(v string) {
+	if g._readStderrLine != nil {
+		(*g._readStderrLine)(v)
+	}
+}
+
+func (g genericCommandListener) Finished(v int) {
+	if g._finished != nil {
+		(*g._finished)(v)
+	}
+}
+
+func (g genericCommandListener) Timeout() {
+	if g._timeout != nil {
+		(*g._timeout)()
+	}
+}

cmdext/pipereader.go

@@ -0,0 +1,158 @@
+package cmdext
+
+import (
+	"bufio"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
+	"io"
+	"sync"
+)
+
+type pipeReader struct {
+	lineBufferSize *int
+	stdout         io.ReadCloser
+	stderr         io.ReadCloser
+}
+
+// Read ready stdout and stdin until finished
+// also splits both pipes into lines and calld the listener
+func (pr *pipeReader) Read(listener []CommandListener) (string, string, string, error) {
+	type combevt struct {
+		line string
+		stop bool
+	}
+
+	errch := make(chan error, 8)
+
+	wg := sync.WaitGroup{}
+
+	// [1] read raw stdout
+
+	wg.Add(1)
+	stdoutBufferReader, stdoutBufferWriter := io.Pipe()
+	stdout := ""
+	go func() {
+		buf := make([]byte, 128)
+		for true {
+			n, out := pr.stdout.Read(buf)
+			if n > 0 {
+				txt := string(buf[:n])
+				stdout += txt
+				_, _ = stdoutBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStdout(buf[:n])
+				}
+			}
+			if out == io.EOF {
+				break
+			}
+			if out != nil {
+				errch <- out
+				break
+			}
+		}
+		_ = stdoutBufferWriter.Close()
+		wg.Done()
+	}()
+
+	// [2] read raw stderr
+
+	wg.Add(1)
+	stderrBufferReader, stderrBufferWriter := io.Pipe()
+	stderr := ""
+	go func() {
+		buf := make([]byte, 128)
+		for true {
+			n, err := pr.stderr.Read(buf)
+
+			if n > 0 {
+				txt := string(buf[:n])
+				stderr += txt
+				_, _ = stderrBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStderr(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stderrBufferWriter.Close()
+		wg.Done()
+	}()
+
+	combch := make(chan combevt, 32)
+
+	// [3] collect stdout line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stdoutBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStdoutLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [4] collect stderr line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stderrBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStderrLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [5] combine stdcombined
+
+	wg.Add(1)
+	stdcombined := ""
+	go func() {
+		stopctr := 0
+		for stopctr < 2 {
+			vvv := <-combch
+			if vvv.stop {
+				stopctr++
+			} else {
+				stdcombined += vvv.line + "\n" // this comes from bufio.Scanner and has no newlines...
+			}
+		}
+		wg.Done()
+	}()
+
+	// wait for all (5) goroutines to finish
+	wg.Wait()
+
+	if err, ok := syncext.ReadNonBlocking(errch); ok {
+		return "", "", "", err
+	}
+
+	return stdout, stderr, stdcombined, nil
+}

confext/confParser.go

@@ -0,0 +1,198 @@
+package confext
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"math/bits"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ApplyEnvOverrides overrides field values from environment variables
+//
+// fields must be tagged with `env:"env_key"`
+//
+// only works on exported fields
+//
+// fields without an env tag are ignored
+// fields with an `env:"-"` tag are ignore
+//
+// sub-structs are recursively parsed (if they have an env tag) and the env-variable keys are delimited by the delim parameter
+// sub-structs with `env:""` are also parsed, but the delimited is skipped (they are handled as if they were one level higher)
+func ApplyEnvOverrides[T any](prefix string, c *T, delim string) error {
+	rval := reflect.ValueOf(c).Elem()
+
+	return processEnvOverrides(rval, delim, prefix)
+}
+
+func processEnvOverrides(rval reflect.Value, delim string, prefix string) error {
+	rtyp := rval.Type()
+
+	for i := 0; i < rtyp.NumField(); i++ {
+
+		rsfield := rtyp.Field(i)
+		rvfield := rval.Field(i)
+
+		if !rsfield.IsExported() {
+			continue
+		}
+
+		if rvfield.Kind() == reflect.Struct {
+
+			envkey, found := rsfield.Tag.Lookup("env")
+			if !found || envkey == "-" {
+				continue
+			}
+
+			subPrefix := prefix
+			if envkey != "" {
+				subPrefix = subPrefix + envkey + delim
+			}
+
+			err := processEnvOverrides(rvfield, delim, subPrefix)
+			if err != nil {
+				return err
+			}
+		}
+
+		envkey := rsfield.Tag.Get("env")
+		if envkey == "" || envkey == "-" {
+			continue
+		}
+
+		fullEnvKey := prefix + envkey
+
+		envval, efound := os.LookupEnv(fullEnvKey)
+		if !efound {
+			continue
+		}
+
+		if rvfield.Type().Kind() == reflect.Pointer {
+
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type().Elem())
+			if err != nil {
+				return err
+			}
+
+			// converts reflect.Value to pointer
+			ptrval := reflect.New(rvfield.Type().Elem())
+			ptrval.Elem().Set(newval)
+
+			rvfield.Set(ptrval)
+
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
+
+		} else {
+
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type())
+			if err != nil {
+				return err
+			}
+
+			rvfield.Set(newval)
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
+
+		}
+
+	}
+
+	return nil
+}
+
+func parseEnvToValue(envval string, fullEnvKey string, rvtype reflect.Type) (reflect.Value, error) {
+	if rvtype == reflect.TypeOf("") {
+
+		return reflect.ValueOf(envval), nil
+
+	} else if rvtype == reflect.TypeOf(int(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int64(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 64)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int64(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int32(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 32)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int32(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int8(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int8(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(time.Duration(0)) {
+
+		dur, err := timeext.ParseDurationShortString(envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(dur), nil
+
+	} else if rvtype == reflect.TypeOf(time.UnixMilli(0)) {
+
+		tim, err := time.Parse(time.RFC3339Nano, envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(tim), nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf(int(0))) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+		envcvl := reflect.ValueOf(envint).Convert(rvtype)
+
+		return envcvl, nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf(false)) {
+
+		if strings.TrimSpace(strings.ToLower(envval)) == "true" {
+			return reflect.ValueOf(true).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "false" {
+			return reflect.ValueOf(false).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "1" {
+			return reflect.ValueOf(true).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "0" {
+			return reflect.ValueOf(false).Convert(rvtype), nil
+		} else {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,bool> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf("")) {
+
+		envcvl := reflect.ValueOf(envval).Convert(rvtype)
+		return envcvl, nil
+
+	} else {
+		return reflect.Value{}, errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvtype.Kind().String(), rvtype.String()))
+	}
+}

confext/confParser_test.go

@@ -0,0 +1,282 @@
+package confext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+	"time"
+)
+
+func TestApplyEnvOverridesNoop(t *testing.T) {
+
+	type aliasint int
+	type aliasstring string
+
+	type testdata struct {
+		V1 int           `env:"TEST_V1"`
+		VX string        ``
+		V2 string        `env:"TEST_V2"`
+		V3 int8          `env:"TEST_V3"`
+		V4 int32         `env:"TEST_V4"`
+		V5 int64         `env:"TEST_V5"`
+		V6 aliasint      `env:"TEST_V6"`
+		VY aliasint      ``
+		V7 aliasstring   `env:"TEST_V7"`
+		V8 time.Duration `env:"TEST_V8"`
+		V9 time.Time     `env:"TEST_V9"`
+	}
+
+	input := testdata{
+		V1: 1,
+		VX: "X",
+		V2: "2",
+		V3: 3,
+		V4: 4,
+		V5: 5,
+		V6: 6,
+		VY: 99,
+		V7: "7",
+		V8: 9,
+		V9: time.Unix(1671102873, 0),
+	}
+
+	output := input
+
+	err := ApplyEnvOverrides("", &output, ".")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertEqual(t, input, output)
+}
+
+func TestApplyEnvOverridesSimple(t *testing.T) {
+
+	type aliasint int
+	type aliasstring string
+
+	type testdata struct {
+		V1 int           `env:"TEST_V1"`
+		VX string        ``
+		V2 string        `env:"TEST_V2"`
+		V3 int8          `env:"TEST_V3"`
+		V4 int32         `env:"TEST_V4"`
+		V5 int64         `env:"TEST_V5"`
+		V6 aliasint      `env:"TEST_V6"`
+		VY aliasint      ``
+		V7 aliasstring   `env:"TEST_V7"`
+		V8 time.Duration `env:"TEST_V8"`
+		V9 time.Time     `env:"TEST_V9"`
+		VA bool          `env:"TEST_VA"`
+	}
+
+	data := testdata{
+		V1: 1,
+		VX: "X",
+		V2: "2",
+		V3: 3,
+		V4: 4,
+		V5: 5,
+		V6: 6,
+		VY: 99,
+		V7: "7",
+		V8: 9,
+		V9: time.Unix(1671102873, 0),
+		VA: false,
+	}
+
+	t.Setenv("TEST_V1", "846")
+	t.Setenv("TEST_V2", "hello_world")
+	t.Setenv("TEST_V3", "6")
+	t.Setenv("TEST_V4", "333")
+	t.Setenv("TEST_V5", "-937")
+	t.Setenv("TEST_V6", "070")
+	t.Setenv("TEST_V7", "AAAAAA")
+	t.Setenv("TEST_V8", "1min4s")
+	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+	t.Setenv("TEST_VA", "true")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertEqual(t, data.V1, 846)
+	tst.AssertEqual(t, data.V2, "hello_world")
+	tst.AssertEqual(t, data.V3, 6)
+	tst.AssertEqual(t, data.V4, 333)
+	tst.AssertEqual(t, data.V5, -937)
+	tst.AssertEqual(t, data.V6, 70)
+	tst.AssertEqual(t, data.V7, "AAAAAA")
+	tst.AssertEqual(t, data.V8, time.Second*64)
+	tst.AssertEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.VA, true)
+}
+
+func TestApplyEnvOverridesRecursive(t *testing.T) {
+
+	type subdata struct {
+		V1 int           `env:"SUB_V1"`
+		VX string        ``
+		V2 string        `env:"SUB_V2"`
+		V8 time.Duration `env:"SUB_V3"`
+		V9 time.Time     `env:"SUB_V4"`
+	}
+
+	type testdata struct {
+		V1   int     `env:"TEST_V1"`
+		VX   string  ``
+		Sub1 subdata ``
+		Sub2 subdata `env:"TEST_V2"`
+		Sub3 subdata `env:"TEST_V3"`
+		Sub4 subdata `env:""`
+		V5   string  `env:"-"`
+	}
+
+	data := testdata{
+		V1: 1,
+		VX: "2",
+		V5: "no",
+		Sub1: subdata{
+			V1: 3,
+			VX: "4",
+			V2: "5",
+			V8: 6 * time.Second,
+			V9: time.Date(2000, 1, 7, 1, 1, 1, 0, time.UTC),
+		},
+		Sub2: subdata{
+			V1: 8,
+			VX: "9",
+			V2: "10",
+			V8: 11 * time.Second,
+			V9: time.Date(2000, 1, 12, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+		Sub3: subdata{
+			V1: 13,
+			VX: "14",
+			V2: "15",
+			V8: 16 * time.Second,
+			V9: time.Date(2000, 1, 17, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+		Sub4: subdata{
+			V1: 18,
+			VX: "19",
+			V2: "20",
+			V8: 21 * time.Second,
+			V9: time.Date(2000, 1, 22, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+	}
+
+	t.Setenv("TEST_V1", "999")
+	t.Setenv("-", "yes")
+
+	t.Setenv("TEST_V2_SUB_V1", "846")
+	t.Setenv("TEST_V2_SUB_V2", "222_hello_world")
+	t.Setenv("TEST_V2_SUB_V3", "1min4s")
+	t.Setenv("TEST_V2_SUB_V4", "2009-11-10T23:00:00Z")
+
+	t.Setenv("TEST_V3_SUB_V1", "33846")
+	t.Setenv("TEST_V3_SUB_V2", "33_hello_world")
+	t.Setenv("TEST_V3_SUB_V3", "33min4s")
+	t.Setenv("TEST_V3_SUB_V4", "2033-11-10T23:00:00Z")
+
+	t.Setenv("SUB_V1", "11")
+	t.Setenv("SUB_V2", "22")
+	t.Setenv("SUB_V3", "33min")
+	t.Setenv("SUB_V4", "2044-01-01T00:00:00Z")
+
+	err := ApplyEnvOverrides("", &data, "_")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertEqual(t, data.V1, 999)
+	tst.AssertEqual(t, data.VX, "2")
+	tst.AssertEqual(t, data.V5, "no")
+	tst.AssertEqual(t, data.Sub1.V1, 3)
+	tst.AssertEqual(t, data.Sub1.VX, "4")
+	tst.AssertEqual(t, data.Sub1.V2, "5")
+	tst.AssertEqual(t, data.Sub1.V8, time.Second*6)
+	tst.AssertEqual(t, data.Sub1.V9, time.Unix(947206861, 0).UTC())
+	tst.AssertEqual(t, data.Sub2.V1, 846)
+	tst.AssertEqual(t, data.Sub2.VX, "9")
+	tst.AssertEqual(t, data.Sub2.V2, "222_hello_world")
+	tst.AssertEqual(t, data.Sub2.V8, time.Second*64)
+	tst.AssertEqual(t, data.Sub2.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.Sub3.V1, 33846)
+	tst.AssertEqual(t, data.Sub3.VX, "14")
+	tst.AssertEqual(t, data.Sub3.V2, "33_hello_world")
+	tst.AssertEqual(t, data.Sub3.V8, time.Second*1984)
+	tst.AssertEqual(t, data.Sub3.V9, time.Unix(2015276400, 0).UTC())
+	tst.AssertEqual(t, data.Sub4.V1, 11)
+	tst.AssertEqual(t, data.Sub4.VX, "19")
+	tst.AssertEqual(t, data.Sub4.V2, "22")
+	tst.AssertEqual(t, data.Sub4.V8, time.Second*1980)
+	tst.AssertEqual(t, data.Sub4.V9, time.Unix(2335219200, 0).UTC())
+}
+
+func TestApplyEnvOverridesPointer(t *testing.T) {
+
+	type aliasint int
+	type aliasstring string
+
+	type testdata struct {
+		V1 *int           `env:"TEST_V1"`
+		VX *string        ``
+		V2 *string        `env:"TEST_V2"`
+		V3 *int8          `env:"TEST_V3"`
+		V4 *int32         `env:"TEST_V4"`
+		V5 *int64         `env:"TEST_V5"`
+		V6 *aliasint      `env:"TEST_V6"`
+		VY *aliasint      ``
+		V7 *aliasstring   `env:"TEST_V7"`
+		V8 *time.Duration `env:"TEST_V8"`
+		V9 *time.Time     `env:"TEST_V9"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("TEST_V1", "846")
+	t.Setenv("TEST_V2", "hello_world")
+	t.Setenv("TEST_V3", "6")
+	t.Setenv("TEST_V4", "333")
+	t.Setenv("TEST_V5", "-937")
+	t.Setenv("TEST_V6", "070")
+	t.Setenv("TEST_V7", "AAAAAA")
+	t.Setenv("TEST_V8", "1min4s")
+	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertDeRefEqual(t, data.V1, 846)
+	tst.AssertDeRefEqual(t, data.V2, "hello_world")
+	tst.AssertDeRefEqual(t, data.V3, 6)
+	tst.AssertDeRefEqual(t, data.V4, 333)
+	tst.AssertDeRefEqual(t, data.V5, -937)
+	tst.AssertDeRefEqual(t, data.V6, 70)
+	tst.AssertDeRefEqual(t, data.V7, "AAAAAA")
+	tst.AssertDeRefEqual(t, data.V8, time.Second*64)
+	tst.AssertDeRefEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+}
+
+func assertEqual[T comparable](t *testing.T, actual T, expected T) {
+	if actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}
+
+func assertPtrEqual[T comparable](t *testing.T, actual *T, expected T) {
+	if actual == nil {
+		t.Errorf("values differ: Actual: NIL, Expected: '%v'", expected)
+	}
+	if *actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

cryptext/aes.go

@@ -0,0 +1,132 @@
+package cryptext
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"golang.org/x/crypto/scrypt"
+	"io"
+)
+
+// https://stackoverflow.com/a/18819040/1761622
+
+type aesPayload struct {
+	Salt    []byte `json:"s"`
+	IV      []byte `json:"i"`
+	Data    []byte `json:"d"`
+	Rounds  int    `json:"r"`
+	Version uint   `json:"v"`
+}
+
+func EncryptAESSimple(password []byte, data []byte, rounds int) (string, error) {
+
+	salt := make([]byte, 8)
+	_, err := io.ReadFull(rand.Reader, salt)
+	if err != nil {
+		return "", err
+	}
+
+	key, err := scrypt.Key(password, salt, rounds, 8, 1, 32)
+	if err != nil {
+		return "", err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	h := sha256.New()
+	h.Write(data)
+	checksum := h.Sum(nil)
+	if len(checksum) != 32 {
+		return "", errors.New("wrong cs size")
+	}
+
+	ciphertext := make([]byte, 32+len(data))
+
+	iv := make([]byte, aes.BlockSize)
+	_, err = io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return "", err
+	}
+
+	combinedData := make([]byte, 0, 32+len(data))
+	combinedData = append(combinedData, checksum...)
+	combinedData = append(combinedData, data...)
+
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext, combinedData)
+
+	pl := aesPayload{
+		Salt:    salt,
+		IV:      iv,
+		Data:    ciphertext,
+		Version: 1,
+		Rounds:  rounds,
+	}
+
+	jbin, err := json.Marshal(pl)
+	if err != nil {
+		return "", err
+	}
+
+	res := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(jbin)
+
+	return res, nil
+}
+
+func DecryptAESSimple(password []byte, encText string) ([]byte, error) {
+
+	jbin, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(encText)
+	if err != nil {
+		return nil, err
+	}
+
+	var pl aesPayload
+	err = json.Unmarshal(jbin, &pl)
+	if err != nil {
+		return nil, err
+	}
+
+	if pl.Version != 1 {
+		return nil, errors.New("unsupported version")
+	}
+
+	key, err := scrypt.Key(password, pl.Salt, pl.Rounds, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	dest := make([]byte, len(pl.Data))
+
+	cfb := cipher.NewCFBDecrypter(block, pl.IV)
+	cfb.XORKeyStream(dest, pl.Data)
+
+	if len(dest) < 32 {
+		return nil, errors.New("payload too small")
+	}
+
+	chck := dest[:32]
+	data := dest[32:]
+
+	h := sha256.New()
+	h.Write(data)
+	chck2 := h.Sum(nil)
+
+	if !bytes.Equal(chck, chck2) {
+		return nil, errors.New("checksum mismatch")
+	}
+
+	return data, nil
+}

cryptext/aes_test.go

@@ -0,0 +1,36 @@
+package cryptext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestEncryptAESSimple(t *testing.T) {
+
+	pw := []byte("hunter12")
+
+	str1 := []byte("Hello World")
+
+	str2, err := EncryptAESSimple(pw, str1, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%s\n", str2)
+
+	str3, err := DecryptAESSimple(pw, str2)
+	if err != nil {
+		panic(err)
+	}
+
+	tst.AssertEqual(t, string(str1), string(str3))
+
+	str4, err := EncryptAESSimple(pw, str3, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	tst.AssertNotEqual(t, string(str2), string(str4))
+
+}

cryptext/hash_test.go

@@ -1,25 +1,20 @@
 package cryptext
 
 import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
 func TestStrSha256(t *testing.T) {
-	assertEqual(t, StrSha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	assertEqual(t, StrSha256("0"), "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9")
-	assertEqual(t, StrSha256("80085"), "b3786e141d65638ad8a98173e26b5f6a53c927737b23ff31fb1843937250f44b")
-	assertEqual(t, StrSha256("Hello World"), "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e")
+	tst.AssertEqual(t, StrSha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	tst.AssertEqual(t, StrSha256("0"), "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9")
+	tst.AssertEqual(t, StrSha256("80085"), "b3786e141d65638ad8a98173e26b5f6a53c927737b23ff31fb1843937250f44b")
+	tst.AssertEqual(t, StrSha256("Hello World"), "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e")
 }
 
 func TestBytesSha256(t *testing.T) {
-	assertEqual(t, BytesSha256([]byte{}), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	assertEqual(t, BytesSha256([]byte{0}), "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d")
-	assertEqual(t, BytesSha256([]byte{128}), "76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71")
-	assertEqual(t, BytesSha256([]byte{0, 1, 2, 4, 8, 16, 32, 64, 128, 255}), "55016a318ba538e00123c736b2a8b6db368d00e7e25727547655b653e5853603")
-}
-
-func assertEqual(t *testing.T, actual string, expected string) {
-	if actual != expected {
-		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
-	}
+	tst.AssertEqual(t, BytesSha256([]byte{}), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	tst.AssertEqual(t, BytesSha256([]byte{0}), "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d")
+	tst.AssertEqual(t, BytesSha256([]byte{128}), "76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71")
+	tst.AssertEqual(t, BytesSha256([]byte{0, 1, 2, 4, 8, 16, 32, 64, 128, 255}), "55016a318ba538e00123c736b2a8b6db368d00e7e25727547655b653e5853603")
 }

cryptext/passHash.go

@@ -0,0 +1,429 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/totpext"
+	"golang.org/x/crypto/bcrypt"
+	"strconv"
+	"strings"
+)
+
+const LatestPassHashVersion = 5
+
+// PassHash
+// - [v0]: plaintext password ( `0|...` )                  // simple, used to write PW's directly in DB
+// - [v1]: sha256(plaintext)                               // simple hashing
+// - [v2]: seed | sha256<seed>(plaintext)                  // add seed
+// - [v3]: seed | sha256<seed>(plaintext) | [hex(totp)]    // add TOTP support
+// - [v4]: bcrypt(plaintext) | [hex(totp)]                 // use proper bcrypt
+// - [v5]: bcrypt(sha512(plaintext)) | [hex(totp)]         // hash pw before bcrypt (otherwise max pw-len = 72)
+type PassHash string
+
+func (ph PassHash) Valid() bool {
+	_, _, _, _, _, valid := ph.Data()
+	return valid
+}
+
+func (ph PassHash) HasTOTP() bool {
+	_, _, _, otp, _, _ := ph.Data()
+	return otp
+}
+
+func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bool, _totpsecret []byte, _valid bool) {
+
+	split := strings.Split(string(ph), "|")
+	if len(split) == 0 {
+		return -1, nil, nil, false, nil, false
+	}
+
+	version, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return -1, nil, nil, false, nil, false
+	}
+
+	if version == 0 {
+		if len(split) != 2 {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), nil, []byte(split[1]), false, nil, true
+	}
+
+	if version == 1 {
+		if len(split) != 2 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), nil, payload, false, nil, true
+	}
+
+	//
+	if version == 2 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		seed, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[2])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), seed, payload, false, nil, true
+	}
+
+	if version == 3 {
+		if len(split) != 4 {
+			return -1, nil, nil, false, nil, false
+		}
+		seed, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[2])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[3] != "0" {
+			totpsecret, err = hex.DecodeString(split[3])
+			totp = true
+		}
+		return int(version), seed, payload, totp, totpsecret, true
+	}
+
+	if version == 4 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload := []byte(split[1])
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[2] != "0" {
+			totpsecret, err = hex.DecodeString(split[2])
+			totp = true
+		}
+		return int(version), nil, payload, totp, totpsecret, true
+	}
+
+	if version == 5 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload := []byte(split[1])
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[2] != "0" {
+			totpsecret, err = hex.DecodeString(split[2])
+			totp = true
+		}
+		return int(version), nil, payload, totp, totpsecret, true
+	}
+
+	return -1, nil, nil, false, nil, false
+}
+
+func (ph PassHash) Verify(plainpass string, totp *string) bool {
+	version, seed, payload, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return false
+	}
+
+	if hastotp && totp == nil {
+		return false
+	}
+
+	if version == 0 {
+		return langext.ArrEqualsExact([]byte(plainpass), payload)
+	}
+
+	if version == 1 {
+		return langext.ArrEqualsExact(hash256(plainpass), payload)
+	}
+
+	if version == 2 {
+		return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload)
+	}
+
+	if version == 3 {
+		if !hastotp {
+			return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload)
+		} else {
+			return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload) && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	if version == 4 {
+		if !hastotp {
+			return bcrypt.CompareHashAndPassword(payload, []byte(plainpass)) == nil
+		} else {
+			return bcrypt.CompareHashAndPassword(payload, []byte(plainpass)) == nil && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	if version == 5 {
+		if !hastotp {
+			return bcrypt.CompareHashAndPassword(payload, hash512(plainpass)) == nil
+		} else {
+			return bcrypt.CompareHashAndPassword(payload, hash512(plainpass)) == nil && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	return false
+}
+
+func (ph PassHash) NeedsPasswordUpgrade() bool {
+	version, _, _, _, _, valid := ph.Data()
+	return valid && version < LatestPassHashVersion
+}
+
+func (ph PassHash) Upgrade(plainpass string) (PassHash, error) {
+	version, _, _, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid password")
+	}
+	if version == LatestPassHashVersion {
+		return ph, nil
+	}
+	if hastotp {
+		return HashPassword(plainpass, totpsecret)
+	} else {
+		return HashPassword(plainpass, nil)
+	}
+}
+
+func (ph PassHash) ClearTOTP() (PassHash, error) {
+	version, _, _, _, _, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return ph, nil
+	}
+
+	if version == 1 {
+		return ph, nil
+	}
+
+	if version == 2 {
+		return ph, nil
+	}
+
+	if version == 3 {
+		split := strings.Split(string(ph), "|")
+		split[3] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 4 {
+		split := strings.Split(string(ph), "|")
+		split[2] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 5 {
+		split := strings.Split(string(ph), "|")
+		split[2] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) WithTOTP(totpSecret []byte) (PassHash, error) {
+	version, _, _, _, _, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 1 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 2 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 3 {
+		split := strings.Split(string(ph), "|")
+		split[3] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 4 {
+		split := strings.Split(string(ph), "|")
+		split[2] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 5 {
+		split := strings.Split(string(ph), "|")
+		split[2] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) Change(newPlainPass string) (PassHash, error) {
+	version, _, _, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return HashPasswordV0(newPlainPass)
+	}
+
+	if version == 1 {
+		return HashPasswordV1(newPlainPass)
+	}
+
+	if version == 2 {
+		return HashPasswordV2(newPlainPass)
+	}
+
+	if version == 3 {
+		return HashPasswordV3(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	if version == 4 {
+		return HashPasswordV4(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	if version == 5 {
+		return HashPasswordV5(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) String() string {
+	return string(ph)
+}
+
+func HashPassword(plainpass string, totpSecret []byte) (PassHash, error) {
+	return HashPasswordV5(plainpass, totpSecret)
+}
+
+func HashPasswordV5(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	payload, err := bcrypt.GenerateFromPassword(hash512(plainpass), bcrypt.MinCost)
+	if err != nil {
+		return "", err
+	}
+
+	return PassHash(fmt.Sprintf("5|%s|%s", string(payload), strtotp)), nil
+}
+
+func HashPasswordV4(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	payload, err := bcrypt.GenerateFromPassword([]byte(plainpass), bcrypt.MinCost)
+	if err != nil {
+		return "", err
+	}
+
+	return PassHash(fmt.Sprintf("4|%s|%s", string(payload), strtotp)), nil
+}
+
+func HashPasswordV3(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	seed, err := newSeed()
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash256Seeded(plainpass, seed)
+
+	return PassHash(fmt.Sprintf("3|%s|%s|%s",
+		base64.RawStdEncoding.EncodeToString(seed),
+		base64.RawStdEncoding.EncodeToString(checksum),
+		strtotp)), nil
+}
+
+func HashPasswordV2(plainpass string) (PassHash, error) {
+	seed, err := newSeed()
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash256Seeded(plainpass, seed)
+
+	return PassHash(fmt.Sprintf("2|%s|%s", base64.RawStdEncoding.EncodeToString(seed), base64.RawStdEncoding.EncodeToString(checksum))), nil
+}
+
+func HashPasswordV1(plainpass string) (PassHash, error) {
+	return PassHash(fmt.Sprintf("1|%s", base64.RawStdEncoding.EncodeToString(hash256(plainpass)))), nil
+}
+
+func HashPasswordV0(plainpass string) (PassHash, error) {
+	return PassHash(fmt.Sprintf("0|%s", plainpass)), nil
+}
+
+func hash512(s string) []byte {
+	h := sha512.New()
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func hash256(s string) []byte {
+	h := sha256.New()
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func hash256Seeded(s string, seed []byte) []byte {
+	h := sha256.New()
+	h.Write(seed)
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func newSeed() ([]byte, error) {
+	secret := make([]byte, 32)
+	_, err := rand.Read(secret)
+	if err != nil {
+		return nil, err
+	}
+	return secret, nil
+}

cryptext/passHash_test.go

@@ -0,0 +1,210 @@
+package cryptext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/totpext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestPassHash1(t *testing.T) {
+	ph, err := HashPassword("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashTOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPassword("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashUpgrade_V0(t *testing.T) {
+	ph, err := HashPasswordV0("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V1(t *testing.T) {
+	ph, err := HashPasswordV1("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V2(t *testing.T) {
+	ph, err := HashPasswordV2("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V3(t *testing.T) {
+	ph, err := HashPasswordV3("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V3_TOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPasswordV3("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashUpgrade_V4(t *testing.T) {
+	ph, err := HashPasswordV4("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V4_TOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPasswordV4("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", langext.Ptr(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}

cursortoken/direction.go

@@ -0,0 +1,8 @@
+package cursortoken
+
+type SortDirection string //@enum:type
+
+const (
+	SortASC  SortDirection = "ASC"
+	SortDESC SortDirection = "DESC"
+)

cursortoken/filter.go

@@ -0,0 +1,10 @@
+package cursortoken
+
+import (
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type Filter interface {
+	FilterQuery() mongo.Pipeline
+	Pagination() (string, SortDirection, string, SortDirection)
+}

cursortoken/token.go

@@ -0,0 +1,184 @@
+package cursortoken
+
+import (
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"strings"
+	"time"
+)
+
+type Mode string
+
+const (
+	CTMStart  Mode = "START"
+	CTMNormal Mode = "NORMAL"
+	CTMEnd    Mode = "END"
+)
+
+type Extra struct {
+	Timestamp *time.Time
+	Id        *string
+	Page      *int
+	PageSize  *int
+}
+
+type CursorToken struct {
+	Mode               Mode
+	ValuePrimary       string
+	ValueSecondary     string
+	Direction          SortDirection
+	DirectionSecondary SortDirection
+	PageSize           int
+	Extra              Extra
+}
+
+type cursorTokenSerialize struct {
+	ValuePrimary       *string        `json:"v1,omitempty"`
+	ValueSecondary     *string        `json:"v2,omitempty"`
+	Direction          *SortDirection `json:"dir,omitempty"`
+	DirectionSecondary *SortDirection `json:"dir2,omitempty"`
+	PageSize           *int           `json:"size,omitempty"`
+
+	ExtraTimestamp *time.Time `json:"ts,omitempty"`
+	ExtraId        *string    `json:"id,omitempty"`
+	ExtraPage      *int       `json:"pg,omitempty"`
+	ExtraPageSize  *int       `json:"sz,omitempty"`
+}
+
+func Start() CursorToken {
+	return CursorToken{
+		Mode:               CTMStart,
+		ValuePrimary:       "",
+		ValueSecondary:     "",
+		Direction:          "",
+		DirectionSecondary: "",
+		PageSize:           0,
+		Extra:              Extra{},
+	}
+}
+
+func End() CursorToken {
+	return CursorToken{
+		Mode:               CTMEnd,
+		ValuePrimary:       "",
+		ValueSecondary:     "",
+		Direction:          "",
+		DirectionSecondary: "",
+		PageSize:           0,
+		Extra:              Extra{},
+	}
+}
+
+func (c *CursorToken) Token() string {
+	if c.Mode == CTMStart {
+		return "@start"
+	}
+	if c.Mode == CTMEnd {
+		return "@end"
+	}
+
+	// We kinda manually implement omitempty for the CursorToken here
+	// because omitempty does not work for time.Time and otherwise we would always
+	// get weird time values when decoding a token that initially didn't have an Timestamp set
+	// For this usecase we treat Unix=0 as an empty timestamp
+
+	sertok := cursorTokenSerialize{}
+
+	if c.ValuePrimary != "" {
+		sertok.ValuePrimary = &c.ValuePrimary
+	}
+	if c.ValueSecondary != "" {
+		sertok.ValueSecondary = &c.ValueSecondary
+	}
+	if c.Direction != "" {
+		sertok.Direction = &c.Direction
+	}
+	if c.DirectionSecondary != "" {
+		sertok.DirectionSecondary = &c.DirectionSecondary
+	}
+	if c.PageSize != 0 {
+		sertok.PageSize = &c.PageSize
+	}
+
+	sertok.ExtraTimestamp = c.Extra.Timestamp
+	sertok.ExtraId = c.Extra.Id
+	sertok.ExtraPage = c.Extra.Page
+	sertok.ExtraPageSize = c.Extra.PageSize
+
+	body, err := json.Marshal(sertok)
+	if err != nil {
+		panic(err)
+	}
+
+	return "tok_" + base32.StdEncoding.EncodeToString(body)
+}
+
+func Decode(tok string) (CursorToken, error) {
+	if tok == "" {
+		return Start(), nil
+	}
+	if strings.ToLower(tok) == "@start" {
+		return Start(), nil
+	}
+	if strings.ToLower(tok) == "@end" {
+		return End(), nil
+	}
+
+	if !strings.HasPrefix(tok, "tok_") {
+		return CursorToken{}, errors.New("could not decode token, missing prefix")
+	}
+
+	body, err := base32.StdEncoding.DecodeString(tok[len("tok_"):])
+	if err != nil {
+		return CursorToken{}, err
+	}
+
+	var tokenDeserialize cursorTokenSerialize
+	err = json.Unmarshal(body, &tokenDeserialize)
+	if err != nil {
+		return CursorToken{}, err
+	}
+
+	token := CursorToken{Mode: CTMNormal}
+
+	if tokenDeserialize.ValuePrimary != nil {
+		token.ValuePrimary = *tokenDeserialize.ValuePrimary
+	}
+	if tokenDeserialize.ValueSecondary != nil {
+		token.ValueSecondary = *tokenDeserialize.ValueSecondary
+	}
+	if tokenDeserialize.Direction != nil {
+		token.Direction = *tokenDeserialize.Direction
+	}
+	if tokenDeserialize.DirectionSecondary != nil {
+		token.DirectionSecondary = *tokenDeserialize.DirectionSecondary
+	}
+	if tokenDeserialize.PageSize != nil {
+		token.PageSize = *tokenDeserialize.PageSize
+	}
+
+	token.Extra.Timestamp = tokenDeserialize.ExtraTimestamp
+	token.Extra.Id = tokenDeserialize.ExtraId
+	token.Extra.Page = tokenDeserialize.ExtraPage
+	token.Extra.PageSize = tokenDeserialize.ExtraPageSize
+
+	return token, nil
+}
+
+func (c *CursorToken) ValuePrimaryObjectId() (primitive.ObjectID, bool) {
+	if oid, err := primitive.ObjectIDFromHex(c.ValuePrimary); err == nil {
+		return oid, true
+	} else {
+		return primitive.ObjectID{}, false
+	}
+}
+
+func (c *CursorToken) ValueSecondaryObjectId() (primitive.ObjectID, bool) {
+	if oid, err := primitive.ObjectIDFromHex(c.ValueSecondary); err == nil {
+		return oid, true
+	} else {
+		return primitive.ObjectID{}, false
+	}
+}

dataext/bufferedReadCloser.go

@@ -1,56 +1,157 @@
 package dataext
 
-import "io"
+import (
+	"errors"
+	"io"
+)
+
+type brcMode int
+
+const (
+	modeSourceReading  brcMode = 0
+	modeSourceFinished brcMode = 1
+	modeBufferReading  brcMode = 2
+	modeBufferFinished brcMode = 3
+)
 
 type BufferedReadCloser interface {
 	io.ReadCloser
 	BufferedAll() ([]byte, error)
+	Reset() error
 }
 
 type bufferedReadCloser struct {
-	buffer   []byte
-	inner    io.ReadCloser
-	finished bool
-}
-
-func (b *bufferedReadCloser) Read(p []byte) (int, error) {
-
-	n, err := b.inner.Read(p)
-	if n > 0 {
-		b.buffer = append(b.buffer, p[0:n]...)
-	}
-
-	if err == io.EOF {
-		b.finished = true
-	}
-
-	return n, err
+	buffer []byte
+	inner  io.ReadCloser
+	mode   brcMode
+	off    int
 }
 
 func NewBufferedReadCloser(sub io.ReadCloser) BufferedReadCloser {
 	return &bufferedReadCloser{
-		buffer:   make([]byte, 0, 1024),
-		inner:    sub,
-		finished: false,
+		buffer: make([]byte, 0, 1024),
+		inner:  sub,
+		mode:   modeSourceReading,
+		off:    0,
+	}
+}
+
+func (b *bufferedReadCloser) Read(p []byte) (int, error) {
+	switch b.mode {
+	case modeSourceReading:
+		n, err := b.inner.Read(p)
+		if n > 0 {
+			b.buffer = append(b.buffer, p[0:n]...)
+		}
+
+		if err == io.EOF {
+			b.mode = modeSourceFinished
+		}
+
+		return n, err
+
+	case modeSourceFinished:
+		return 0, io.EOF
+
+	case modeBufferReading:
+
+		if len(b.buffer) <= b.off {
+			b.mode = modeBufferFinished
+			if len(p) == 0 {
+				return 0, nil
+			}
+			return 0, io.EOF
+		}
+
+		n := copy(p, b.buffer[b.off:])
+		b.off += n
+		return n, nil
+
+	case modeBufferFinished:
+		return 0, io.EOF
+
+	default:
+		return 0, errors.New("object in undefined status")
 	}
 }
 
 func (b *bufferedReadCloser) Close() error {
-	err := b.inner.Close()
-	if err != nil {
-		b.finished = true
+	switch b.mode {
+	case modeSourceReading:
+		_, err := b.BufferedAll()
+		if err != nil {
+			return err
+		}
+		err = b.inner.Close()
+		if err != nil {
+			return err
+		}
+		b.mode = modeSourceFinished
+		return nil
+
+	case modeSourceFinished:
+		return nil
+
+	case modeBufferReading:
+		b.mode = modeBufferFinished
+		return nil
+
+	case modeBufferFinished:
+		return nil
+
+	default:
+		return errors.New("object in undefined status")
 	}
-	return err
+
 }
 
 func (b *bufferedReadCloser) BufferedAll() ([]byte, error) {
-	arr := make([]byte, 1024)
-	for !b.finished {
-		_, err := b.Read(arr)
-		if err != nil && err != io.EOF {
-			return nil, err
+	switch b.mode {
+	case modeSourceReading:
+		arr := make([]byte, 1024)
+		for b.mode == modeSourceReading {
+			_, err := b.Read(arr)
+			if err != nil && err != io.EOF {
+				return nil, err
+			}
 		}
-	}
+		return b.buffer, nil
 
-	return b.buffer, nil
+	case modeSourceFinished:
+		return b.buffer, nil
+
+	case modeBufferReading:
+		return b.buffer, nil
+
+	case modeBufferFinished:
+		return b.buffer, nil
+
+	default:
+		return nil, errors.New("object in undefined status")
+	}
+}
+
+func (b *bufferedReadCloser) Reset() error {
+	switch b.mode {
+	case modeSourceReading:
+		fallthrough
+	case modeSourceFinished:
+		err := b.Close()
+		if err != nil {
+			return err
+		}
+		b.mode = modeBufferReading
+		b.off = 0
+		return nil
+
+	case modeBufferReading:
+		fallthrough
+	case modeBufferFinished:
+		b.mode = modeBufferReading
+		b.off = 0
+		return nil
+
+	default:
+		return errors.New("object in undefined status")
+	}
 }

dataext/lruMap.go

@@ -19,40 +19,38 @@ import (
 // There are also a bunch of unit tests to ensure that the cache is always in a consistent state
 //
 
-type LRUData interface{}
-
-type LRUMap struct {
+type LRUMap[TKey comparable, TData any] struct {
 	maxsize int
 	lock    sync.Mutex
 
-	cache map[string]*cacheNode
+	cache map[TKey]*cacheNode[TKey, TData]
 
-	lfuHead *cacheNode
-	lfuTail *cacheNode
+	lfuHead *cacheNode[TKey, TData]
+	lfuTail *cacheNode[TKey, TData]
 }
 
-type cacheNode struct {
-	key    string
-	data   LRUData
-	parent *cacheNode
-	child  *cacheNode
+type cacheNode[TKey comparable, TData any] struct {
+	key    TKey
+	data   TData
+	parent *cacheNode[TKey, TData]
+	child  *cacheNode[TKey, TData]
 }
 
-func NewLRUMap(size int) *LRUMap {
+func NewLRUMap[TKey comparable, TData any](size int) *LRUMap[TKey, TData] {
 	if size <= 2 && size != 0 {
 		panic("Size must be > 2  (or 0)")
 	}
 
-	return &LRUMap{
+	return &LRUMap[TKey, TData]{
 		maxsize: size,
 		lock:    sync.Mutex{},
-		cache:   make(map[string]*cacheNode, size+1),
+		cache:   make(map[TKey]*cacheNode[TKey, TData], size+1),
 		lfuHead: nil,
 		lfuTail: nil,
 	}
 }
 
-func (c *LRUMap) Put(key string, value LRUData) {
+func (c *LRUMap[TKey, TData]) Put(key TKey, value TData) {
 	if c.maxsize == 0 {
 		return // cache disabled
 	}
@@ -70,7 +68,7 @@ func (c *LRUMap) Put(key string, value LRUData) {
 	}
 
 	// key does not exist: insert into map and add to top of LFU
-	node = &cacheNode{
+	node = &cacheNode[TKey, TData]{
 		key:    key,
 		data:   value,
 		parent: nil,
@@ -95,9 +93,9 @@ func (c *LRUMap) Put(key string, value LRUData) {
 	}
 }
 
-func (c *LRUMap) TryGet(key string) (LRUData, bool) {
+func (c *LRUMap[TKey, TData]) TryGet(key TKey) (TData, bool) {
 	if c.maxsize == 0 {
-		return nil, false // cache disabled
+		return *new(TData), false // cache disabled
 	}
 
 	c.lock.Lock()
@@ -105,13 +103,13 @@ func (c *LRUMap) TryGet(key string) (LRUData, bool) {
 
 	val, ok := c.cache[key]
 	if !ok {
-		return nil, false
+		return *new(TData), false
 	}
 	c.moveNodeToTop(val)
 	return val.data, ok
 }
 
-func (c *LRUMap) moveNodeToTop(node *cacheNode) {
+func (c *LRUMap[TKey, TData]) moveNodeToTop(node *cacheNode[TKey, TData]) {
 	// (only called in critical section !)
 
 	if c.lfuHead == node { // fast case
@@ -144,7 +142,7 @@ func (c *LRUMap) moveNodeToTop(node *cacheNode) {
 	}
 }
 
-func (c *LRUMap) Size() int {
+func (c *LRUMap[TKey, TData]) Size() int {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	return len(c.cache)

dataext/lruMap_test.go

@@ -12,7 +12,7 @@ func init() {
 }
 
 func TestResultCache1(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key := randomKey()
@@ -39,7 +39,7 @@ func TestResultCache1(t *testing.T) {
 	if !ok {
 		t.Errorf("cache TryGet returned no value")
 	}
-	if !eq(cacheval, val) {
+	if cacheval != val {
 		t.Errorf("cache TryGet returned different value (%+v <> %+v)", cacheval, val)
 	}
 
@@ -50,7 +50,7 @@ func TestResultCache1(t *testing.T) {
 }
 
 func TestResultCache2(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -150,7 +150,7 @@ func TestResultCache2(t *testing.T) {
 }
 
 func TestResultCache3(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -160,20 +160,20 @@ func TestResultCache3(t *testing.T) {
 	cache.Put(key1, val1)
 	verifyLRUList(cache, t)
 
-	if val, ok := cache.TryGet(key1); !ok || !eq(val, val1) {
+	if val, ok := cache.TryGet(key1); !ok || val != val1 {
 		t.Errorf("Value in cache should be [val1]")
 	}
 
 	cache.Put(key1, val2)
 	verifyLRUList(cache, t)
 
-	if val, ok := cache.TryGet(key1); !ok || !eq(val, val2) {
+	if val, ok := cache.TryGet(key1); !ok || val != val2 {
 		t.Errorf("Value in cache should be [val2]")
 	}
 }
 
 // does a basic consistency check over the internal cache representation
-func verifyLRUList(cache *LRUMap, t *testing.T) {
+func verifyLRUList[TKey comparable, TData any](cache *LRUMap[TKey, TData], t *testing.T) {
 	size := 0
 
 	tailFound := false
@@ -250,23 +250,10 @@ func randomKey() string {
 	return strconv.FormatInt(rand.Int63(), 16)
 }
 
-func randomVal() LRUData {
+func randomVal() string {
 	v, err := langext.NewHexUUID()
 	if err != nil {
 		panic(err)
 	}
-	return &v
-}
-
-func eq(a LRUData, b LRUData) bool {
-	v1, ok1 := a.(*string)
-	v2, ok2 := b.(*string)
-	if ok1 && ok2 {
-		if v1 == nil || v2 == nil {
-			return false
-		}
-		return v1 == v2
-	}
-
-	return false
+	return v
 }

dataext/merge_test.go

@@ -2,6 +2,7 @@ package dataext
 
 import (
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
@@ -43,10 +44,10 @@ func TestObjectMerge(t *testing.T) {
 
 	valueMerge := ObjectMerge(valueA, valueB)
 
-	assertPtrEqual(t, "Field1", valueMerge.Field1, valueB.Field1)
-	assertPtrEqual(t, "Field2", valueMerge.Field2, valueA.Field2)
-	assertPtrEqual(t, "Field3", valueMerge.Field3, valueB.Field3)
-	assertPtrEqual(t, "Field4", valueMerge.Field4, nil)
+	tst.AssertIdentPtrEqual(t, "Field1", valueMerge.Field1, valueB.Field1)
+	tst.AssertIdentPtrEqual(t, "Field2", valueMerge.Field2, valueA.Field2)
+	tst.AssertIdentPtrEqual(t, "Field3", valueMerge.Field3, valueB.Field3)
+	tst.AssertIdentPtrEqual(t, "Field4", valueMerge.Field4, nil)
 
 }
 

dataext/stack.go

@@ -0,0 +1,116 @@
+package dataext
+
+import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"sync"
+)
+
+var ErrEmptyStack = errors.New("stack is empty")
+
+type Stack[T any] struct {
+	lock *sync.Mutex
+	data []T
+}
+
+func NewStack[T any](threadsafe bool, initialCapacity int) *Stack[T] {
+	var lck *sync.Mutex = nil
+	if threadsafe {
+		lck = &sync.Mutex{}
+	}
+	return &Stack[T]{
+		lock: lck,
+		data: make([]T, 0, initialCapacity),
+	}
+}
+
+func (s *Stack[T]) Push(v T) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	s.data = append(s.data, v)
+}
+
+func (s *Stack[T]) Pop() (T, error) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+	if l == 0 {
+		return *new(T), ErrEmptyStack
+	}
+
+	result := s.data[l-1]
+	s.data = s.data[:l-1]
+
+	return result, nil
+}
+
+func (s *Stack[T]) OptPop() *T {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+	if l == 0 {
+		return nil
+	}
+
+	result := s.data[l-1]
+	s.data = s.data[:l-1]
+
+	return langext.Ptr(result)
+}
+
+func (s *Stack[T]) Peek() (T, error) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+
+	if l == 0 {
+		return *new(T), ErrEmptyStack
+	}
+
+	return s.data[l-1], nil
+}
+
+func (s *Stack[T]) OptPeek() *T {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+
+	if l == 0 {
+		return nil
+	}
+
+	return langext.Ptr(s.data[l-1])
+}
+
+func (s *Stack[T]) Length() int {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	return len(s.data)
+}
+
+func (s *Stack[T]) Empty() bool {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	return len(s.data) == 0
+}

dataext/structHash.go

@@ -0,0 +1,254 @@
+package dataext
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"hash"
+	"io"
+	"reflect"
+	"sort"
+)
+
+type StructHashOptions struct {
+	HashAlgo    hash.Hash
+	Tag         *string
+	SkipChannel bool
+	SkipFunc    bool
+}
+
+func StructHash(dat any, opt ...StructHashOptions) (r []byte, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			r = nil
+			err = errors.New(fmt.Sprintf("recovered panic: %v", rec))
+		}
+	}()
+
+	shopt := StructHashOptions{}
+	if len(opt) > 1 {
+		return nil, errors.New("multiple options supplied")
+	} else if len(opt) == 1 {
+		shopt = opt[0]
+	}
+
+	if shopt.HashAlgo == nil {
+		shopt.HashAlgo = sha256.New()
+	}
+
+	writer := new(bytes.Buffer)
+
+	if langext.IsNil(dat) {
+		shopt.HashAlgo.Reset()
+		shopt.HashAlgo.Write(writer.Bytes())
+		res := shopt.HashAlgo.Sum(nil)
+		return res, nil
+	}
+
+	err = binarize(writer, reflect.ValueOf(dat), shopt)
+	if err != nil {
+		return nil, err
+	}
+
+	shopt.HashAlgo.Reset()
+	shopt.HashAlgo.Write(writer.Bytes())
+	res := shopt.HashAlgo.Sum(nil)
+
+	return res, nil
+}
+
+func writeBinarized(writer io.Writer, dat any) error {
+	tmp := bytes.Buffer{}
+	err := binary.Write(&tmp, binary.LittleEndian, dat)
+	if err != nil {
+		return err
+	}
+	err = binary.Write(writer, binary.LittleEndian, uint64(tmp.Len()))
+	if err != nil {
+		return err
+	}
+	_, err = writer.Write(tmp.Bytes())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func binarize(writer io.Writer, dat reflect.Value, opt StructHashOptions) error {
+	var err error
+
+	err = binary.Write(writer, binary.LittleEndian, uint8(dat.Kind()))
+	switch dat.Kind() {
+	case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice, reflect.Interface:
+		if dat.IsNil() {
+			err = binary.Write(writer, binary.LittleEndian, uint64(0))
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+	}
+
+	err = binary.Write(writer, binary.LittleEndian, uint64(len(dat.Type().String())))
+	if err != nil {
+		return err
+	}
+	_, err = writer.Write([]byte(dat.Type().String()))
+	if err != nil {
+		return err
+	}
+
+	switch dat.Type().Kind() {
+	case reflect.Invalid:
+		return errors.New("cannot binarize value of kind <Invalid>")
+	case reflect.Bool:
+		return writeBinarized(writer, dat.Bool())
+	case reflect.Int:
+		return writeBinarized(writer, int64(dat.Int()))
+	case reflect.Int8:
+		fallthrough
+	case reflect.Int16:
+		fallthrough
+	case reflect.Int32:
+		fallthrough
+	case reflect.Int64:
+		return writeBinarized(writer, dat.Interface())
+	case reflect.Uint:
+		return writeBinarized(writer, uint64(dat.Int()))
+	case reflect.Uint8:
+		fallthrough
+	case reflect.Uint16:
+		fallthrough
+	case reflect.Uint32:
+		fallthrough
+	case reflect.Uint64:
+		return writeBinarized(writer, dat.Interface())
+	case reflect.Uintptr:
+		return errors.New("cannot binarize value of kind <Uintptr>")
+	case reflect.Float32:
+		fallthrough
+	case reflect.Float64:
+		return writeBinarized(writer, dat.Interface())
+	case reflect.Complex64:
+		return errors.New("cannot binarize value of kind <Complex64>")
+	case reflect.Complex128:
+		return errors.New("cannot binarize value of kind <Complex128>")
+	case reflect.Slice:
+		fallthrough
+	case reflect.Array:
+		return binarizeArrayOrSlice(writer, dat, opt)
+	case reflect.Chan:
+		if opt.SkipChannel {
+			return nil
+		}
+		return errors.New("cannot binarize value of kind <Chan>")
+	case reflect.Func:
+		if opt.SkipFunc {
+			return nil
+		}
+		return errors.New("cannot binarize value of kind <Func>")
+	case reflect.Interface:
+		return binarize(writer, dat.Elem(), opt)
+	case reflect.Map:
+		return binarizeMap(writer, dat, opt)
+	case reflect.Pointer:
+		return binarize(writer, dat.Elem(), opt)
+	case reflect.String:
+		v := dat.String()
+		err = binary.Write(writer, binary.LittleEndian, uint64(len(v)))
+		if err != nil {
+			return err
+		}
+		_, err = writer.Write([]byte(v))
+		if err != nil {
+			return err
+		}
+		return nil
+	case reflect.Struct:
+		return binarizeStruct(writer, dat, opt)
+	case reflect.UnsafePointer:
+		return errors.New("cannot binarize value of kind <UnsafePointer>")
+	default:
+		return errors.New("cannot binarize value of unknown kind <" + dat.Type().Kind().String() + ">")
+	}
+}
+
+func binarizeStruct(writer io.Writer, dat reflect.Value, opt StructHashOptions) error {
+	err := binary.Write(writer, binary.LittleEndian, uint64(dat.NumField()))
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < dat.NumField(); i++ {
+
+		if opt.Tag != nil {
+			if _, ok := dat.Type().Field(i).Tag.Lookup(*opt.Tag); !ok {
+				continue
+			}
+		}
+
+		err = binary.Write(writer, binary.LittleEndian, uint64(len(dat.Type().Field(i).Name)))
+		if err != nil {
+			return err
+		}
+		_, err = writer.Write([]byte(dat.Type().Field(i).Name))
+		if err != nil {
+			return err
+		}
+
+		err = binarize(writer, dat.Field(i), opt)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func binarizeArrayOrSlice(writer io.Writer, dat reflect.Value, opt StructHashOptions) error {
+	err := binary.Write(writer, binary.LittleEndian, uint64(dat.Len()))
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < dat.Len(); i++ {
+		err := binarize(writer, dat.Index(i), opt)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func binarizeMap(writer io.Writer, dat reflect.Value, opt StructHashOptions) error {
+	err := binary.Write(writer, binary.LittleEndian, uint64(dat.Len()))
+	if err != nil {
+		return err
+	}
+
+	sub := make([][]byte, 0, dat.Len())
+
+	for _, k := range dat.MapKeys() {
+		tmp := bytes.Buffer{}
+		err = binarize(&tmp, dat.MapIndex(k), opt)
+		if err != nil {
+			return err
+		}
+		sub = append(sub, tmp.Bytes())
+	}
+
+	sort.Slice(sub, func(i1, i2 int) bool { return bytes.Compare(sub[i1], sub[i2]) < 0 })
+
+	for _, v := range sub {
+		_, err = writer.Write(v)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

dataext/structHash_test.go

@@ -0,0 +1,136 @@
+package dataext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func noErrStructHash(t *testing.T, dat any, opt ...StructHashOptions) []byte {
+	res, err := StructHash(dat, opt...)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+		return nil
+	}
+	return res
+}
+
+func TestStructHashSimple(t *testing.T) {
+
+	tst.AssertHexEqual(t, "209bf774af36cc3a045c152d9f1269ef3684ad819c1359ee73ff0283a308fefa", noErrStructHash(t, "Hello"))
+	tst.AssertHexEqual(t, "c32f3626b981ae2997db656f3acad3f1dc9d30ef6b6d14296c023e391b25f71a", noErrStructHash(t, 0))
+	tst.AssertHexEqual(t, "01b781b03e9586b257d387057dfc70d9f06051e7d3c1e709a57e13cc8daf3e35", noErrStructHash(t, []byte{}))
+	tst.AssertHexEqual(t, "93e1dcd45c732fe0079b0fb3204c7c803f0921835f6bfee2e6ff263e73eed53c", noErrStructHash(t, []int{}))
+	tst.AssertHexEqual(t, "54f637a376aad55b3160d98ebbcae8099b70d91b9400df23fb3709855d59800a", noErrStructHash(t, []int{1, 2, 3}))
+	tst.AssertHexEqual(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", noErrStructHash(t, nil))
+	tst.AssertHexEqual(t, "349a7db91aa78fd30bbaa7c7f9c7bfb2fcfe72869b4861162a96713a852f60d3", noErrStructHash(t, []any{1, "", nil}))
+	tst.AssertHexEqual(t, "ca51aab87808bf0062a4a024de6aac0c2bad54275cc857a4944569f89fd245ad", noErrStructHash(t, struct{}{}))
+
+}
+
+func TestStructHashSimpleStruct(t *testing.T) {
+
+	type t0 struct {
+		F1 int
+		F2 []string
+		F3 *int
+	}
+
+	tst.AssertHexEqual(t, "a90bff751c70c738bb5cfc9b108e783fa9c19c0bc9273458e0aaee6e74aa1b92", noErrStructHash(t, t0{
+		F1: 10,
+		F2: []string{"1", "2", "3"},
+		F3: nil,
+	}))
+
+	tst.AssertHexEqual(t, "5d09090dc34ac59dd645f197a255f653387723de3afa1b614721ea5a081c675f", noErrStructHash(t, t0{
+		F1: 10,
+		F2: []string{"1", "2", "3"},
+		F3: langext.Ptr(99),
+	}))
+
+}
+
+func TestStructHashLayeredStruct(t *testing.T) {
+
+	type t1_1 struct {
+		F10 float32
+		F12 float64
+		F15 bool
+	}
+	type t1_2 struct {
+		SV1 *t1_1
+		SV2 *t1_1
+		SV3 t1_1
+	}
+
+	tst.AssertHexEqual(t, "fd4ca071fb40a288fee4b7a3dfdaab577b30cb8f80f81ec511e7afd72dc3b469", noErrStructHash(t, t1_2{
+		SV1: nil,
+		SV2: nil,
+		SV3: t1_1{
+			F10: 1,
+			F12: 2,
+			F15: false,
+		},
+	}))
+	tst.AssertHexEqual(t, "3fbf7c67d8121deda075cc86319a4e32d71744feb2cebf89b43bc682f072a029", noErrStructHash(t, t1_2{
+		SV1: nil,
+		SV2: &t1_1{},
+		SV3: t1_1{
+			F10: 3,
+			F12: 4,
+			F15: true,
+		},
+	}))
+	tst.AssertHexEqual(t, "b1791ccd1b346c3ede5bbffda85555adcd8216b93ffca23f14fe175ec47c5104", noErrStructHash(t, t1_2{
+		SV1: &t1_1{},
+		SV2: &t1_1{},
+		SV3: t1_1{
+			F10: 5,
+			F12: 6,
+			F15: false,
+		},
+	}))
+
+}
+
+func TestStructHashMap(t *testing.T) {
+
+	type t0 struct {
+		F1 int
+		F2 map[string]int
+	}
+
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+		F1: 10,
+		F2: map[string]int{
+			"x": 1,
+			"0": 2,
+			"a": 99,
+		},
+	}))
+
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+		F1: 10,
+		F2: map[string]int{
+			"a": 99,
+			"x": 1,
+			"0": 2,
+		},
+	}))
+
+	m3 := make(map[string]int, 99)
+	m3["a"] = 0
+	m3["x"] = 0
+	m3["0"] = 0
+
+	m3["0"] = 99
+	m3["x"] = 1
+	m3["a"] = 2
+
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+		F1: 10,
+		F2: m3,
+	}))
+
+}

dataext/syncSet.go

@@ -2,31 +2,34 @@ package dataext
 
 import "sync"
 
-type SyncStringSet struct {
-	data map[string]bool
+type SyncSet[TData comparable] struct {
+	data map[TData]bool
 	lock sync.Mutex
 }
 
-func (s *SyncStringSet) Add(value string) bool {
+// Add adds `value` to the set
+// returns true  if the value was actually inserted
+// returns false if the value already existed
+func (s *SyncSet[TData]) Add(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
-	_, ok := s.data[value]
+	_, existsInPreState := s.data[value]
 	s.data[value] = true
 
-	return !ok
+	return !existsInPreState
 }
 
-func (s *SyncStringSet) AddAll(values []string) {
+func (s *SyncSet[TData]) AddAll(values []TData) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
 	for _, value := range values {
@@ -34,12 +37,12 @@ func (s *SyncStringSet) AddAll(values []string) {
 	}
 }
 
-func (s *SyncStringSet) Contains(value string) bool {
+func (s *SyncSet[TData]) Contains(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
 	_, ok := s.data[value]
@@ -47,15 +50,15 @@ func (s *SyncStringSet) Contains(value string) bool {
 	return ok
 }
 
-func (s *SyncStringSet) Get() []string {
+func (s *SyncSet[TData]) Get() []TData {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
-	r := make([]string, 0, len(s.data))
+	r := make([]TData, 0, len(s.data))
 
 	for k := range s.data {
 		r = append(r, k)

enums/enum.go

@@ -0,0 +1,24 @@
+package enums
+
+type Enum interface {
+	Valid() bool
+	ValuesAny() []any
+	ValuesMeta() []EnumMetaValue
+	VarName() string
+}
+
+type StringEnum interface {
+	Enum
+	String() string
+}
+
+type DescriptionEnum interface {
+	Enum
+	Description() string
+}
+
+type EnumMetaValue struct {
+	VarName     string  `json:"varName"`
+	Value       any     `json:"value"`
+	Description *string `json:"description"`
+}

exerr/builder.go

@@ -0,0 +1,470 @@
+package exerr
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+	"gogs.mikescher.com/BlackForestBytes/goext/enums"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"net/http"
+	"os"
+	"runtime/debug"
+	"strings"
+	"time"
+)
+
+//
+// ==== USAGE =====
+//
+// If some method returns an error _always wrap it into an exerror:
+//     value, err := do_something(..)
+//     if err != nil {
+//         return nil, exerror.Wrap(err, "do something failed").Build()
+//     }
+//
+// If possible add metadata to the error (eg the id that was not found, ...), the methods are the same as in zerolog
+//     return nil, exerror.Wrap(err, "do something failed").Str("someid", id).Int("count", in.Count).Build()
+//
+// You can change the errortype with `.User()` and `.System()` (User-errors are 400 and System-errors 500)
+// You can also manually set the statuscode with `.WithStatuscode(http.NotFound)`
+// You can set the type with `WithType(..)`
+//
+// New Errors (that don't wrap an existing err object) are created with New
+//     return nil, exerror.New(exerror.TypeInternal, "womethign wen horrible wrong").Build()
+// You can eitehr use an existing ErrorType, the "catch-all" ErrInternal, or add you own ErrType in consts.go
+//
+// All errors should be handled one of the following four ways:
+//   - return the error to the caller and let him handle it:
+//     (also auto-prints the error to the log)
+//     => Wrap/New + Build
+//   - Print the error
+//     (also auto-sends it to the error-service)
+//     This is useful for errors that happen asynchron or are non-fatal for the current request
+//     => Wrap/New + Print
+//   - Return the error to the Rest-API caller
+//     (also auto-prints the error to the log)
+//     (also auto-sends it to the error-service)
+//     => Wrap/New + Output
+//   - Print and stop the service
+//     (also auto-sends it to the error-service)
+//     => Wrap/New + Fatal
+//
+
+var stackSkipLogger zerolog.Logger
+
+func init() {
+	cw := zerolog.ConsoleWriter{
+		Out:        os.Stdout,
+		TimeFormat: "2006-01-02 15:04:05 Z07:00",
+	}
+
+	multi := zerolog.MultiLevelWriter(cw)
+	stackSkipLogger = zerolog.New(multi).With().Timestamp().CallerWithSkipFrameCount(4).Logger()
+}
+
+type Builder struct {
+	errorData       *ExErr
+	containsGinData bool
+}
+
+func Get(err error) *Builder {
+	return &Builder{errorData: FromError(err)}
+}
+
+func New(t ErrorType, msg string) *Builder {
+	return &Builder{errorData: newExErr(CatSystem, t, msg)}
+}
+
+func Wrap(err error, msg string) *Builder {
+	if err == nil {
+		return &Builder{errorData: newExErr(CatSystem, TypeInternal, msg)} // prevent NPE if we call Wrap with err==nil
+	}
+
+	if !pkgconfig.RecursiveErrors {
+		v := FromError(err)
+		v.Message = msg
+		return &Builder{errorData: v}
+	}
+	return &Builder{errorData: wrapExErr(FromError(err), msg, CatWrap, 1)}
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) WithType(t ErrorType) *Builder {
+	b.errorData.Type = t
+	return b
+}
+
+func (b *Builder) WithStatuscode(status int) *Builder {
+	b.errorData.StatusCode = &status
+	return b
+}
+
+func (b *Builder) WithMessage(msg string) *Builder {
+	b.errorData.Message = msg
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+// Err changes the Severity to ERROR (default)
+// The error will be:
+//
+//   - On Build():
+//
+//   - Short-Logged as Err
+//
+//   - On Print():
+//
+//   - Logged as Err
+//
+//   - Send to the error-service
+//
+//   - On Output():
+//
+//   - Logged as Err
+//
+//   - Send to the error-service
+func (b *Builder) Err() *Builder {
+	b.errorData.Severity = SevErr
+	return b
+}
+
+// Warn changes the Severity to WARN
+// The error will be:
+//
+//   - On Build():
+//
+//   - -(nothing)-
+//
+//   - On Print():
+//
+//   - Short-Logged as Warn
+//
+//   - On Output():
+//
+//   - Logged as Warn
+func (b *Builder) Warn() *Builder {
+	b.errorData.Severity = SevWarn
+	return b
+}
+
+// Info changes the Severity to INFO
+// The error will be:
+//
+//   - On Build():
+//
+//   - -(nothing)-
+//
+//   - On Print():
+//
+//   - -(nothing)-
+//
+//   - On Output():
+//
+//   - -(nothing)-
+func (b *Builder) Info() *Builder {
+	b.errorData.Severity = SevInfo
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+// User sets the Category to CatUser
+//
+// Errors with category
+func (b *Builder) User() *Builder {
+	b.errorData.Category = CatUser
+	return b
+}
+
+func (b *Builder) System() *Builder {
+	b.errorData.Category = CatSystem
+	return b
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) Id(key string, val fmt.Stringer) *Builder {
+	return b.addMeta(key, MDTID, newIDWrap(val))
+}
+
+func (b *Builder) StrPtr(key string, val *string) *Builder {
+	return b.addMeta(key, MDTStringPtr, val)
+}
+
+func (b *Builder) Str(key string, val string) *Builder {
+	return b.addMeta(key, MDTString, val)
+}
+
+func (b *Builder) Int(key string, val int) *Builder {
+	return b.addMeta(key, MDTInt, val)
+}
+
+func (b *Builder) Int8(key string, val int8) *Builder {
+	return b.addMeta(key, MDTInt8, val)
+}
+
+func (b *Builder) Int16(key string, val int16) *Builder {
+	return b.addMeta(key, MDTInt16, val)
+}
+
+func (b *Builder) Int32(key string, val int32) *Builder {
+	return b.addMeta(key, MDTInt32, val)
+}
+
+func (b *Builder) Int64(key string, val int64) *Builder {
+	return b.addMeta(key, MDTInt64, val)
+}
+
+func (b *Builder) Float32(key string, val float32) *Builder {
+	return b.addMeta(key, MDTFloat32, val)
+}
+
+func (b *Builder) Float64(key string, val float64) *Builder {
+	return b.addMeta(key, MDTFloat64, val)
+}
+
+func (b *Builder) Bool(key string, val bool) *Builder {
+	return b.addMeta(key, MDTBool, val)
+}
+
+func (b *Builder) Bytes(key string, val []byte) *Builder {
+	return b.addMeta(key, MDTBytes, val)
+}
+
+func (b *Builder) ObjectID(key string, val primitive.ObjectID) *Builder {
+	return b.addMeta(key, MDTObjectID, val)
+}
+
+func (b *Builder) Time(key string, val time.Time) *Builder {
+	return b.addMeta(key, MDTTime, val)
+}
+
+func (b *Builder) Dur(key string, val time.Duration) *Builder {
+	return b.addMeta(key, MDTDuration, val)
+}
+
+func (b *Builder) Strs(key string, val []string) *Builder {
+	return b.addMeta(key, MDTStringArray, val)
+}
+
+func (b *Builder) Ints(key string, val []int) *Builder {
+	return b.addMeta(key, MDTIntArray, val)
+}
+
+func (b *Builder) Ints32(key string, val []int32) *Builder {
+	return b.addMeta(key, MDTInt32Array, val)
+}
+
+func (b *Builder) Type(key string, cls interface{}) *Builder {
+	return b.addMeta(key, MDTString, fmt.Sprintf("%T", cls))
+}
+
+func (b *Builder) Interface(key string, val interface{}) *Builder {
+	return b.addMeta(key, MDTAny, newAnyWrap(val))
+}
+
+func (b *Builder) Any(key string, val any) *Builder {
+	return b.addMeta(key, MDTAny, newAnyWrap(val))
+}
+
+func (b *Builder) Stringer(key string, val fmt.Stringer) *Builder {
+	if val == nil {
+		return b.addMeta(key, MDTString, "(!nil)")
+	} else {
+		return b.addMeta(key, MDTString, val.String())
+	}
+}
+
+func (b *Builder) Enum(key string, val enums.Enum) *Builder {
+	return b.addMeta(key, MDTEnum, newEnumWrap(val))
+}
+
+func (b *Builder) Stack() *Builder {
+	return b.addMeta("@Stack", MDTString, string(debug.Stack()))
+}
+
+func (b *Builder) Errs(key string, val []error) *Builder {
+	for i, valerr := range val {
+		b.addMeta(fmt.Sprintf("%v[%v]", key, i), MDTString, Get(valerr).errorData.FormatLog(LogPrintFull))
+	}
+	return b
+}
+
+func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request) *Builder {
+	if v := ctx.Value("start_timestamp"); v != nil {
+		if t, ok := v.(time.Time); ok {
+			b.Time("ctx.startTimestamp", t)
+			b.Time("ctx.endTimestamp", time.Now())
+		}
+	}
+	b.Str("gin.method", req.Method)
+	b.Str("gin.path", g.FullPath())
+	b.Strs("gin.header", extractHeader(g.Request.Header))
+	if req.URL != nil {
+		b.Str("gin.url", req.URL.String())
+	}
+	if ctxVal := g.GetString("apiversion"); ctxVal != "" {
+		b.Str("gin.context.apiversion", ctxVal)
+	}
+	if ctxVal := g.GetString("uid"); ctxVal != "" {
+		b.Str("gin.context.uid", ctxVal)
+	}
+	if ctxVal := g.GetString("fcmId"); ctxVal != "" {
+		b.Str("gin.context.fcmid", ctxVal)
+	}
+	if ctxVal := g.GetString("reqid"); ctxVal != "" {
+		b.Str("gin.context.reqid", ctxVal)
+	}
+	if req.Method != "GET" && req.Body != nil {
+
+		if req.Header.Get("Content-Type") == "application/json" {
+			if brc, ok := req.Body.(dataext.BufferedReadCloser); ok {
+				if bin, err := brc.BufferedAll(); err == nil {
+					if len(bin) < 16*1024 {
+						var prettyJSON bytes.Buffer
+						err = json.Indent(&prettyJSON, bin, "", "  ")
+						if err == nil {
+							b.Str("gin.body", string(prettyJSON.Bytes()))
+						} else {
+							b.Bytes("gin.body", bin)
+						}
+					} else {
+						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+					}
+				}
+			}
+		}
+
+		if req.Header.Get("Content-Type") == "multipart/form-data" || req.Header.Get("Content-Type") == "x-www-form-urlencoded" {
+			if brc, ok := req.Body.(dataext.BufferedReadCloser); ok {
+				if bin, err := brc.BufferedAll(); err == nil {
+					if len(bin) < 16*1024 {
+						b.Bytes("gin.body", bin)
+					} else {
+						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+					}
+				}
+			}
+		}
+
+	}
+
+	b.containsGinData = true
+	return b
+}
+
+func formatHeader(header map[string][]string) string {
+	ml := 1
+	for k, _ := range header {
+		if len(k) > ml {
+			ml = len(k)
+		}
+	}
+	r := ""
+	for k, v := range header {
+		if r != "" {
+			r += "\n"
+		}
+		for _, hval := range v {
+			value := hval
+			value = strings.ReplaceAll(value, "\n", "\\n")
+			value = strings.ReplaceAll(value, "\r", "\\r")
+			value = strings.ReplaceAll(value, "\t", "\\t")
+			r += langext.StrPadRight(k, " ", ml) + " := " + value
+		}
+	}
+	return r
+}
+
+func extractHeader(header map[string][]string) []string {
+	r := make([]string, 0, len(header))
+	for k, v := range header {
+		for _, hval := range v {
+			value := hval
+			value = strings.ReplaceAll(value, "\n", "\\n")
+			value = strings.ReplaceAll(value, "\r", "\\r")
+			value = strings.ReplaceAll(value, "\t", "\\t")
+			r = append(r, k+": "+value)
+		}
+	}
+	return r
+}
+
+// ----------------------------------------------------------------------------
+
+// Build creates a new error, ready to pass up the stack
+// If the errors is not SevWarn or SevInfo it gets also logged (in short form, without stacktrace) onto stdout
+func (b *Builder) Build() error {
+	warnOnPkgConfigNotInitialized()
+
+	if pkgconfig.ZeroLogErrTraces && (b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal) {
+		b.errorData.ShortLog(stackSkipLogger.Error())
+	} else if pkgconfig.ZeroLogAllTraces {
+		b.errorData.ShortLog(stackSkipLogger.Error())
+	}
+
+	b.CallListener(MethodBuild)
+
+	return b.errorData
+}
+
+// Output prints the error onto the gin stdout.
+// The error also gets printed to stdout/stderr
+// If the error is SevErr|SevFatal we also send it to the error-service
+func (b *Builder) Output(ctx context.Context, g *gin.Context) {
+	if !b.containsGinData && g.Request != nil {
+		// Auto-Add gin metadata if the caller hasn't already done it
+		b.GinReq(ctx, g, g.Request)
+	}
+
+	b.errorData.Output(g)
+
+	if b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal {
+		b.errorData.Log(stackSkipLogger.Error())
+	} else if b.errorData.Severity == SevWarn {
+		b.errorData.Log(stackSkipLogger.Warn())
+	}
+
+	b.CallListener(MethodOutput)
+}
+
+// Print prints the error
+// If the error is SevErr we also send it to the error-service
+func (b *Builder) Print() {
+	if b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal {
+		b.errorData.Log(stackSkipLogger.Error())
+	} else if b.errorData.Severity == SevWarn {
+		b.errorData.ShortLog(stackSkipLogger.Warn())
+	}
+
+	b.CallListener(MethodPrint)
+}
+
+func (b *Builder) Format(level LogPrintLevel) string {
+	return b.errorData.FormatLog(level)
+}
+
+// Fatal prints the error and terminates the program
+// If the error is SevErr we also send it to the error-service
+func (b *Builder) Fatal() {
+	b.errorData.Severity = SevFatal
+	b.errorData.Log(stackSkipLogger.WithLevel(zerolog.FatalLevel))
+
+	b.CallListener(MethodFatal)
+
+	os.Exit(1)
+}
+
+// ----------------------------------------------------------------------------
+
+func (b *Builder) addMeta(key string, mdtype metaDataType, val interface{}) *Builder {
+	b.errorData.Meta.add(key, mdtype, val)
+	return b
+}

exerr/constructor.go

@@ -0,0 +1,204 @@
+package exerr
+
+import (
+	"encoding/json"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"reflect"
+	"time"
+)
+
+var reflectTypeStr = reflect.TypeOf("")
+
+func FromError(err error) *ExErr {
+	if verr, ok := err.(*ExErr); ok {
+		// A simple ExErr
+		return verr
+	}
+
+	// A foreign error (eg a MongoDB exception)
+	return &ExErr{
+		UniqueID:       newID(),
+		Category:       CatForeign,
+		Type:           TypeInternal,
+		Severity:       SevErr,
+		Timestamp:      time.Time{},
+		StatusCode:     nil,
+		Message:        err.Error(),
+		WrappedErrType: fmt.Sprintf("%T", err),
+		WrappedErr:     err,
+		Caller:         "",
+		OriginalError:  nil,
+		Meta:           getForeignMeta(err),
+	}
+}
+
+func newExErr(cat ErrorCategory, errtype ErrorType, msg string) *ExErr {
+	return &ExErr{
+		UniqueID:       newID(),
+		Category:       cat,
+		Type:           errtype,
+		Severity:       SevErr,
+		Timestamp:      time.Now(),
+		StatusCode:     nil,
+		Message:        msg,
+		WrappedErrType: "",
+		WrappedErr:     nil,
+		Caller:         callername(2),
+		OriginalError:  nil,
+		Meta:           make(map[string]MetaValue),
+	}
+}
+
+func wrapExErr(e *ExErr, msg string, cat ErrorCategory, stacktraceskip int) *ExErr {
+	return &ExErr{
+		UniqueID:       newID(),
+		Category:       cat,
+		Type:           TypeWrap,
+		Severity:       SevErr,
+		Timestamp:      time.Now(),
+		StatusCode:     e.StatusCode,
+		Message:        msg,
+		WrappedErrType: "",
+		WrappedErr:     nil,
+		Caller:         callername(1 + stacktraceskip),
+		OriginalError:  e,
+		Meta:           make(map[string]MetaValue),
+	}
+}
+
+func getForeignMeta(err error) (mm MetaMap) {
+	mm = make(map[string]MetaValue)
+
+	defer func() {
+		if panicerr := recover(); panicerr != nil {
+			New(TypePanic, "Panic while trying to get foreign meta").
+				Str("source", err.Error()).
+				Interface("panic-object", panicerr).
+				Stack().
+				Print()
+		}
+	}()
+
+	rval := reflect.ValueOf(err)
+	if rval.Kind() == reflect.Interface || rval.Kind() == reflect.Ptr {
+		rval = reflect.ValueOf(err).Elem()
+	}
+
+	mm.add("foreign.errortype", MDTString, rval.Type().String())
+
+	for k, v := range addMetaPrefix("foreign", getReflectedMetaValues(err, 8)) {
+		mm[k] = v
+	}
+
+	return mm
+}
+
+func getReflectedMetaValues(value interface{}, remainingDepth int) map[string]MetaValue {
+
+	if remainingDepth <= 0 {
+		return map[string]MetaValue{}
+	}
+
+	if langext.IsNil(value) {
+		return map[string]MetaValue{"": {DataType: MDTNil, Value: nil}}
+	}
+
+	rval := reflect.ValueOf(value)
+
+	if rval.Type().Kind() == reflect.Ptr {
+
+		if rval.IsNil() {
+			return map[string]MetaValue{"*": {DataType: MDTNil, Value: nil}}
+		}
+
+		elem := rval.Elem()
+
+		return addMetaPrefix("*", getReflectedMetaValues(elem.Interface(), remainingDepth-1))
+	}
+
+	if !rval.CanInterface() {
+		return map[string]MetaValue{"": {DataType: MDTString, Value: "<<no-interface>>"}}
+	}
+
+	raw := rval.Interface()
+
+	switch ifraw := raw.(type) {
+	case time.Time:
+		return map[string]MetaValue{"": {DataType: MDTTime, Value: ifraw}}
+	case time.Duration:
+		return map[string]MetaValue{"": {DataType: MDTDuration, Value: ifraw}}
+	case int:
+		return map[string]MetaValue{"": {DataType: MDTInt, Value: ifraw}}
+	case int8:
+		return map[string]MetaValue{"": {DataType: MDTInt8, Value: ifraw}}
+	case int16:
+		return map[string]MetaValue{"": {DataType: MDTInt16, Value: ifraw}}
+	case int32:
+		return map[string]MetaValue{"": {DataType: MDTInt32, Value: ifraw}}
+	case int64:
+		return map[string]MetaValue{"": {DataType: MDTInt64, Value: ifraw}}
+	case string:
+		return map[string]MetaValue{"": {DataType: MDTString, Value: ifraw}}
+	case bool:
+		return map[string]MetaValue{"": {DataType: MDTBool, Value: ifraw}}
+	case []byte:
+		return map[string]MetaValue{"": {DataType: MDTBytes, Value: ifraw}}
+	case float32:
+		return map[string]MetaValue{"": {DataType: MDTFloat32, Value: ifraw}}
+	case float64:
+		return map[string]MetaValue{"": {DataType: MDTFloat64, Value: ifraw}}
+	case []int:
+		return map[string]MetaValue{"": {DataType: MDTIntArray, Value: ifraw}}
+	case []int32:
+		return map[string]MetaValue{"": {DataType: MDTInt32Array, Value: ifraw}}
+	case primitive.ObjectID:
+		return map[string]MetaValue{"": {DataType: MDTObjectID, Value: ifraw}}
+	case []string:
+		return map[string]MetaValue{"": {DataType: MDTStringArray, Value: ifraw}}
+	}
+
+	if rval.Type().Kind() == reflect.Struct {
+		m := make(map[string]MetaValue)
+		for i := 0; i < rval.NumField(); i++ {
+			fieldtype := rval.Type().Field(i)
+
+			fieldname := fieldtype.Name
+
+			if fieldtype.IsExported() {
+				for k, v := range addMetaPrefix(fieldname, getReflectedMetaValues(rval.Field(i).Interface(), remainingDepth-1)) {
+					m[k] = v
+				}
+			}
+		}
+		return m
+	}
+
+	if rval.Type().ConvertibleTo(reflectTypeStr) {
+		return map[string]MetaValue{"": {DataType: MDTString, Value: rval.Convert(reflectTypeStr).String()}}
+	}
+
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		panic(err) // gets recovered later up
+	}
+
+	return map[string]MetaValue{"": {DataType: MDTString, Value: string(jsonval)}}
+}
+
+func addMetaPrefix(prefix string, m map[string]MetaValue) map[string]MetaValue {
+	if len(m) == 1 {
+		for k, v := range m {
+			if k == "" {
+				return map[string]MetaValue{prefix: v}
+			}
+		}
+	}
+
+	r := make(map[string]MetaValue, len(m))
+	for k, v := range m {
+		r[prefix+"."+k] = v
+	}
+	return r
+}

exerr/data.go

@@ -0,0 +1,76 @@
+package exerr
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+)
+
+type ErrorCategory struct{ Category string }
+
+var (
+	CatWrap    = ErrorCategory{"Wrap"}    // The error is simply wrapping another error (e.g. when a grpc call returns an error)
+	CatSystem  = ErrorCategory{"System"}  // An internal system error (e.g. connection to db failed)
+	CatUser    = ErrorCategory{"User"}    // The user (the API caller) did something wrong (e.g. he has no permissions to do this)
+	CatForeign = ErrorCategory{"Foreign"} // A foreign error that some component threw (e.g. an unknown mongodb error), happens if we call Wrap(..) on an non-bmerror value
+)
+
+//goland:noinspection GoUnusedGlobalVariable
+var AllCategories = []ErrorCategory{CatWrap, CatSystem, CatUser, CatForeign}
+
+type ErrorSeverity struct{ Severity string }
+
+var (
+	SevTrace = ErrorSeverity{"Trace"}
+	SevDebug = ErrorSeverity{"Debug"}
+	SevInfo  = ErrorSeverity{"Info"}
+	SevWarn  = ErrorSeverity{"Warn"}
+	SevErr   = ErrorSeverity{"Err"}
+	SevFatal = ErrorSeverity{"Fatal"}
+)
+
+//goland:noinspection GoUnusedGlobalVariable
+var AllSeverities = []ErrorSeverity{SevTrace, SevDebug, SevInfo, SevWarn, SevErr, SevFatal}
+
+type ErrorType struct {
+	Key               string
+	DefaultStatusCode *int
+}
+
+//goland:noinspection GoUnusedGlobalVariable
+var (
+	TypeInternal       = NewType("INTERNAL_ERROR", langext.Ptr(500))
+	TypePanic          = NewType("PANIC", langext.Ptr(500))
+	TypeNotImplemented = NewType("NOT_IMPLEMENTED", langext.Ptr(500))
+
+	TypeWrap = NewType("Wrap", nil)
+
+	TypeBindFailURI      = NewType("BINDFAIL_URI", langext.Ptr(400))
+	TypeBindFailQuery    = NewType("BINDFAIL_QUERY", langext.Ptr(400))
+	TypeBindFailJSON     = NewType("BINDFAIL_JSON", langext.Ptr(400))
+	TypeBindFailFormData = NewType("BINDFAIL_FORMDATA", langext.Ptr(400))
+	TypeBindFailHeader   = NewType("BINDFAIL_HEADER", langext.Ptr(400))
+
+	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
+	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))
+
+	// other values come from pkgconfig
+)
+
+var registeredTypes = dataext.SyncSet[string]{}
+
+func NewType(key string, defStatusCode *int) ErrorType {
+	insertOkay := registeredTypes.Add(key)
+	if !insertOkay {
+		panic("Cannot register same ErrType ('" + key + "') more than once")
+	}
+
+	return ErrorType{key, defStatusCode}
+}
+
+type LogPrintLevel string
+
+const (
+	LogPrintFull     LogPrintLevel = "Full"
+	LogPrintOverview LogPrintLevel = "Overview"
+	LogPrintShort    LogPrintLevel = "Short"
+)

exerr/errinit.go

@@ -0,0 +1,80 @@
+package exerr
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+)
+
+type ErrorPackageConfig struct {
+	ZeroLogErrTraces       bool                                             // autom print zerolog logs on .Build()  (for SevErr and SevFatal)
+	ZeroLogAllTraces       bool                                             // autom print zerolog logs on .Build()  (for all Severities)
+	RecursiveErrors        bool                                             // errors contains their Origin-Error
+	ExtendedGinOutput      bool                                             // Log extended data (trace, meta, ...) to gin in err.Output()
+	IncludeMetaInGinOutput bool                                             // Log meta fields ( from e.g. `.Str(key, val).Build()` ) to gin in err.Output()
+	ExtendGinOutput        func(err *ExErr, json map[string]any)            // (Optionally) extend the gin output with more fields
+	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any) // (Optionally) extend the gin `__data` output with more fields
+}
+
+type ErrorPackageConfigInit struct {
+	ZeroLogErrTraces       *bool
+	ZeroLogAllTraces       *bool
+	RecursiveErrors        *bool
+	ExtendedGinOutput      *bool
+	IncludeMetaInGinOutput *bool
+	ExtendGinOutput        func(err *ExErr, json map[string]any)
+	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any)
+}
+
+var initialized = false
+
+var pkgconfig = ErrorPackageConfig{
+	ZeroLogErrTraces:       true,
+	ZeroLogAllTraces:       false,
+	RecursiveErrors:        true,
+	ExtendedGinOutput:      false,
+	IncludeMetaInGinOutput: true,
+	ExtendGinOutput:        func(err *ExErr, json map[string]any) {},
+	ExtendGinDataOutput:    func(err *ExErr, depth int, json map[string]any) {},
+}
+
+// Init initializes the exerr packages
+// Must be called at the program start, before (!) any errors
+// Is not thread-safe
+func Init(cfg ErrorPackageConfigInit) {
+	if initialized {
+		panic("Cannot re-init error package")
+	}
+
+	ego := func(err *ExErr, json map[string]any) {}
+	egdo := func(err *ExErr, depth int, json map[string]any) {}
+
+	if cfg.ExtendGinOutput != nil {
+		ego = cfg.ExtendGinOutput
+	}
+	if cfg.ExtendGinDataOutput != nil {
+		egdo = cfg.ExtendGinDataOutput
+	}
+
+	pkgconfig = ErrorPackageConfig{
+		ZeroLogErrTraces:       langext.Coalesce(cfg.ZeroLogErrTraces, pkgconfig.ZeroLogErrTraces),
+		ZeroLogAllTraces:       langext.Coalesce(cfg.ZeroLogAllTraces, pkgconfig.ZeroLogAllTraces),
+		RecursiveErrors:        langext.Coalesce(cfg.RecursiveErrors, pkgconfig.RecursiveErrors),
+		ExtendedGinOutput:      langext.Coalesce(cfg.ExtendedGinOutput, pkgconfig.ExtendedGinOutput),
+		IncludeMetaInGinOutput: langext.Coalesce(cfg.IncludeMetaInGinOutput, pkgconfig.IncludeMetaInGinOutput),
+		ExtendGinOutput:        ego,
+		ExtendGinDataOutput:    egdo,
+	}
+
+	initialized = true
+}
+
+func warnOnPkgConfigNotInitialized() {
+	if !initialized {
+		fmt.Printf("\n")
+		fmt.Printf("%s\n", langext.StrRepeat("=", 80))
+		fmt.Printf("%s\n", "[WARNING] exerr package used without initializiation")
+		fmt.Printf("%s\n", "          call exerr.Init() in your main() function")
+		fmt.Printf("%s\n", langext.StrRepeat("=", 80))
+		fmt.Printf("\n")
+	}
+}

exerr/exerr.go

@@ -0,0 +1,298 @@
+package exerr
+
+import (
+	"github.com/rs/xid"
+	"github.com/rs/zerolog"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"reflect"
+	"strings"
+	"time"
+)
+
+type ExErr struct {
+	UniqueID string `json:"uniqueID"`
+
+	Timestamp time.Time     `json:"timestamp"`
+	Category  ErrorCategory `json:"category"`
+	Severity  ErrorSeverity `json:"severity"`
+	Type      ErrorType     `json:"type"`
+
+	StatusCode *int `json:"statusCode"`
+
+	Message        string `json:"message"`
+	WrappedErrType string `json:"wrappedErrType"`
+	WrappedErr     any    `json:"-"`
+	Caller         string `json:"caller"`
+
+	OriginalError *ExErr `json:"originalError"`
+
+	Meta MetaMap `json:"meta"`
+}
+
+func (ee *ExErr) Error() string {
+	return ee.Message
+}
+
+// Unwrap must be implemented so that some error.XXX methods work
+func (ee *ExErr) Unwrap() error {
+	if ee.OriginalError == nil {
+		return nil // this is neccessary - otherwise we return a wrapped nil and the `x == nil` comparison fails (= panic in errors.Is and other failures)
+	}
+	return ee.OriginalError
+}
+
+// Is must be implemented so that error.Is(x) works
+func (ee *ExErr) Is(e error) bool {
+	return IsFrom(ee, e)
+}
+
+// As must be implemented so that error.As(x) works
+//
+//goland:noinspection GoTypeAssertionOnErrors
+func (ee *ExErr) As(target any) bool {
+	if dstErr, ok := target.(*ExErr); ok {
+
+		if dst0, ok := ee.contains(dstErr); ok {
+			dstErr = dst0
+			return true
+		} else {
+			return false
+		}
+
+	} else {
+
+		val := reflect.ValueOf(target)
+
+		typStr := val.Type().Elem().String()
+
+		for curr := ee; curr != nil; curr = curr.OriginalError {
+			if curr.Category == CatForeign && curr.WrappedErrType == typStr && curr.WrappedErr != nil {
+				val.Elem().Set(reflect.ValueOf(curr.WrappedErr))
+				return true
+			}
+		}
+
+		return false
+	}
+}
+
+func (ee *ExErr) Log(evt *zerolog.Event) {
+	evt.Msg(ee.FormatLog(LogPrintFull))
+}
+
+func (ee *ExErr) FormatLog(lvl LogPrintLevel) string {
+	if lvl == LogPrintShort {
+
+		msg := ee.Message
+		if ee.OriginalError != nil && ee.OriginalError.Category == CatForeign {
+			msg = msg + " (" + strings.ReplaceAll(ee.OriginalError.Message, "\n", " ") + ")"
+		}
+
+		if ee.Type != TypeWrap {
+			return "[" + ee.Type.Key + "] " + msg
+		} else {
+			return msg
+		}
+
+	} else if lvl == LogPrintOverview {
+
+		str := "[" + ee.RecursiveType().Key + "] <" + ee.UniqueID + "> " + strings.ReplaceAll(ee.RecursiveMessage(), "\n", " ") + "\n"
+
+		indent := ""
+		for curr := ee; curr != nil; curr = curr.OriginalError {
+			indent += "  "
+
+			str += indent
+			str += "-> "
+			strmsg := strings.Trim(curr.Message, " \r\n\t")
+			if lbidx := strings.Index(curr.Message, "\n"); lbidx >= 0 {
+				strmsg = strmsg[0:lbidx]
+			}
+			strmsg = langext.StrLimit(strmsg, 61, "...")
+			str += strmsg
+			str += "\n"
+
+		}
+		return str
+
+	} else if lvl == LogPrintFull {
+
+		str := "[" + ee.RecursiveType().Key + "] <" + ee.UniqueID + "> " + strings.ReplaceAll(ee.RecursiveMessage(), "\n", " ") + "\n"
+
+		indent := ""
+		for curr := ee; curr != nil; curr = curr.OriginalError {
+			indent += "  "
+
+			etype := ee.Type.Key
+			if ee.Type == TypeWrap {
+				etype = "~"
+			}
+
+			str += indent
+			str += "-> ["
+			str += etype
+			if curr.Category == CatForeign {
+				str += "|Foreign"
+			}
+			str += "] "
+			str += strings.ReplaceAll(curr.Message, "\n", " ")
+			if curr.Caller != "" {
+				str += " (@ "
+				str += curr.Caller
+				str += ")"
+			}
+			str += "\n"
+
+			if curr.Meta.Any() {
+				meta := indent + "   {" + curr.Meta.FormatOneLine(240) + "}"
+				if len(meta) < 200 {
+					str += meta
+					str += "\n"
+				} else {
+					str += curr.Meta.FormatMultiLine(indent+"   ", "  ", 1024)
+					str += "\n"
+				}
+			}
+		}
+		return str
+
+	} else {
+
+		return "[?[" + ee.UniqueID + "]?]"
+
+	}
+}
+
+func (ee *ExErr) ShortLog(evt *zerolog.Event) {
+	ee.Meta.Apply(evt).Msg(ee.FormatLog(LogPrintShort))
+}
+
+// RecursiveMessage returns the message to show
+// = first error (top-down) that is not wrapping/foreign/empty
+func (ee *ExErr) RecursiveMessage() string {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Message != "" && curr.Category != CatWrap && curr.Category != CatForeign {
+			return curr.Message
+		}
+	}
+
+	// fallback to self
+	return ee.Message
+}
+
+// RecursiveType returns the statuscode to use
+// = first error (top-down) that is not wrapping/empty
+func (ee *ExErr) RecursiveType() ErrorType {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Type != TypeWrap {
+			return curr.Type
+		}
+
+	}
+
+	// fallback to self
+	return ee.Type
+}
+
+// RecursiveStatuscode returns the HTTP Statuscode to use
+// = first error (top-down) that has a statuscode set
+func (ee *ExErr) RecursiveStatuscode() *int {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.StatusCode != nil {
+			return langext.Ptr(*curr.StatusCode)
+		}
+	}
+
+	return nil
+}
+
+// RecursiveCategory returns the ErrorCategory to use
+// = first error (top-down) that has a statuscode set
+func (ee *ExErr) RecursiveCategory() ErrorCategory {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.Category != CatWrap {
+			return curr.Category
+		}
+	}
+
+	// fallback to <empty>
+	return ee.Category
+}
+
+// RecursiveMeta searches (top-down) for teh first error that has a meta value with teh specified key
+// and returns its value (or nil)
+func (ee *ExErr) RecursiveMeta(key string) *MetaValue {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if metaval, ok := curr.Meta[key]; ok {
+			return langext.Ptr(metaval)
+		}
+	}
+
+	return nil
+}
+
+// Depth returns the depth of recursively contained errors
+func (ee *ExErr) Depth() int {
+	if ee.OriginalError == nil {
+		return 1
+	} else {
+		return ee.OriginalError.Depth() + 1
+	}
+}
+
+// contains test if the supplied error is contained in this error (anywhere in the chain)
+func (ee *ExErr) contains(original *ExErr) (*ExErr, bool) {
+	if original == nil {
+		return nil, false
+	}
+
+	if ee == original {
+		return ee, true
+	}
+
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if curr.equalsDirectProperties(curr) {
+			return curr, true
+		}
+	}
+
+	return nil, false
+}
+
+// equalsDirectProperties tests if ee and other are equals, but only looks at primary properties (not `OriginalError` or `Meta`)
+func (ee *ExErr) equalsDirectProperties(other *ExErr) bool {
+
+	if ee.UniqueID != other.UniqueID {
+		return false
+	}
+	if ee.Timestamp != other.Timestamp {
+		return false
+	}
+	if ee.Category != other.Category {
+		return false
+	}
+	if ee.Severity != other.Severity {
+		return false
+	}
+	if ee.Type != other.Type {
+		return false
+	}
+	if ee.StatusCode != other.StatusCode {
+		return false
+	}
+	if ee.Message != other.Message {
+		return false
+	}
+	if ee.WrappedErrType != other.WrappedErrType {
+		return false
+	}
+	if ee.Caller != other.Caller {
+		return false
+	}
+
+	return true
+}
+
+func newID() string {
+	return xid.New().String()
+}

exerr/exerr_test.go

@@ -0,0 +1,93 @@
+package exerr
+
+import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+type golangErr struct {
+	Message string
+}
+
+func (g golangErr) Error() string {
+	return g.Message
+}
+
+type golangErr2 struct {
+	Message string
+}
+
+func (g golangErr2) Error() string {
+	return g.Message
+}
+
+type simpleError struct {
+}
+
+func (g simpleError) Error() string {
+	return "Something simple went wroong"
+}
+
+type simpleError2 struct {
+}
+
+func (g simpleError2) Error() string {
+	return "Something simple went wroong"
+}
+
+func TestExErrIs1(t *testing.T) {
+	e0 := simpleError{}
+
+	wrap := Wrap(e0, "something went wrong").Str("test", "123").Build()
+
+	tst.AssertTrue(t, errors.Is(wrap, simpleError{}))
+	tst.AssertFalse(t, errors.Is(wrap, golangErr{}))
+	tst.AssertFalse(t, errors.Is(wrap, golangErr{"error1"}))
+}
+
+func TestExErrIs2(t *testing.T) {
+	e0 := golangErr{"error1"}
+
+	wrap := Wrap(e0, "something went wrong").Str("test", "123").Build()
+
+	tst.AssertTrue(t, errors.Is(wrap, e0))
+	tst.AssertTrue(t, errors.Is(wrap, golangErr{"error1"}))
+	tst.AssertFalse(t, errors.Is(wrap, golangErr{"error2"}))
+	tst.AssertFalse(t, errors.Is(wrap, simpleError{}))
+}
+
+func TestExErrAs(t *testing.T) {
+
+	e0 := golangErr{"error1"}
+
+	w0 := Wrap(e0, "something went wrong").Str("test", "123").Build()
+
+	{
+		out := golangErr{}
+		ok := errors.As(w0, &out)
+		tst.AssertTrue(t, ok)
+		tst.AssertEqual(t, out.Message, "error1")
+	}
+
+	w1 := Wrap(w0, "outher error").Build()
+
+	{
+		out := golangErr{}
+		ok := errors.As(w1, &out)
+		tst.AssertTrue(t, ok)
+		tst.AssertEqual(t, out.Message, "error1")
+	}
+
+	{
+		out := golangErr2{}
+		ok := errors.As(w1, &out)
+		tst.AssertFalse(t, ok)
+	}
+
+	{
+		out := simpleError2{}
+		ok := errors.As(w1, &out)
+		tst.AssertFalse(t, ok)
+	}
+}

exerr/gin.go

@@ -0,0 +1,112 @@
+package exerr
+
+import (
+	"github.com/gin-gonic/gin"
+	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"net/http"
+	"time"
+)
+
+func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) langext.H {
+	ginJson := langext.H{}
+
+	if ee.UniqueID != "" {
+		ginJson["id"] = ee.UniqueID
+	}
+	if ee.Category != CatWrap {
+		ginJson["category"] = ee.Category
+	}
+	if ee.Type != TypeWrap {
+		ginJson["type"] = ee.Type
+	}
+	if ee.StatusCode != nil {
+		ginJson["statuscode"] = ee.StatusCode
+	}
+	if ee.Message != "" {
+		ginJson["message"] = ee.Message
+	}
+	if ee.Caller != "" {
+		ginJson["caller"] = ee.Caller
+	}
+	if ee.Severity != SevErr {
+		ginJson["severity"] = ee.Severity
+	}
+	if ee.Timestamp != (time.Time{}) {
+		ginJson["time"] = ee.Timestamp.Format(time.RFC3339)
+	}
+	if ee.WrappedErrType != "" {
+		ginJson["wrappedErrType"] = ee.WrappedErrType
+	}
+	if ee.OriginalError != nil {
+		ginJson["original"] = ee.OriginalError.toJson(depth+1, applyExtendListener, outputMeta)
+	}
+
+	if outputMeta {
+		metaJson := langext.H{}
+		for metaKey, metaVal := range ee.Meta {
+			metaJson[metaKey] = metaVal.rawValueForJson()
+		}
+		ginJson["meta"] = metaJson
+	}
+
+	if applyExtendListener {
+		pkgconfig.ExtendGinDataOutput(ee, depth, ginJson)
+	}
+
+	return ginJson
+}
+
+// ToAPIJson converts the ExError to a json object
+// (the same object as used in the Output(gin) method)
+//
+// Parameters:
+//   - [applyExtendListener]:  if false the pkgconfig.ExtendGinOutput / pkgconfig.ExtendGinDataOutput will not be applied
+//   - [includeWrappedErrors]: if false we do not include the recursive/wrapped errors in `__data`
+//   - [includeMetaFields]:    if true  we also include meta-values (aka from `.Str(key, value).Build()`), needs includeWrappedErrors=true
+func (ee *ExErr) ToAPIJson(applyExtendListener bool, includeWrappedErrors bool, includeMetaFields bool) langext.H {
+
+	apiOutput := langext.H{
+		"errorid":   ee.UniqueID,
+		"message":   ee.RecursiveMessage(),
+		"errorcode": ee.RecursiveType().Key,
+		"category":  ee.RecursiveCategory().Category,
+	}
+
+	if includeWrappedErrors {
+		apiOutput["__data"] = ee.toJson(0, applyExtendListener, includeMetaFields)
+	}
+
+	if applyExtendListener {
+		pkgconfig.ExtendGinOutput(ee, apiOutput)
+	}
+
+	return apiOutput
+}
+
+func (ee *ExErr) Output(g *gin.Context) {
+
+	warnOnPkgConfigNotInitialized()
+
+	var statuscode = http.StatusInternalServerError
+
+	var baseCat = ee.RecursiveCategory()
+	var baseType = ee.RecursiveType()
+	var baseStatuscode = ee.RecursiveStatuscode()
+
+	if baseCat == CatUser {
+		statuscode = http.StatusBadRequest
+	} else if baseCat == CatSystem {
+		statuscode = http.StatusInternalServerError
+	}
+
+	if baseStatuscode != nil {
+		statuscode = *ee.StatusCode
+	} else if baseType.DefaultStatusCode != nil {
+		statuscode = *baseType.DefaultStatusCode
+	}
+
+	ginOutput := ee.ToAPIJson(true, pkgconfig.ExtendedGinOutput, pkgconfig.IncludeMetaInGinOutput)
+
+	g.Render(statuscode, json.GoJsonRender{Data: ginOutput, NilSafeSlices: true, NilSafeMaps: true})
+}

exerr/helper.go

@@ -0,0 +1,88 @@
+package exerr
+
+import "fmt"
+
+// IsType test if the supplied error is of the specified ErrorType.
+func IsType(err error, errType ErrorType) bool {
+	if err == nil {
+		return false
+	}
+
+	bmerr := FromError(err)
+	for bmerr != nil {
+		if bmerr.Type == errType {
+			return true
+		}
+		bmerr = bmerr.OriginalError
+	}
+
+	return false
+}
+
+// IsFrom test if the supplied error stems originally from original
+func IsFrom(e error, original error) bool {
+	if e == nil {
+		return false
+	}
+
+	//goland:noinspection GoDirectComparisonOfErrors
+	if e == original {
+		return true
+	}
+
+	bmerr := FromError(e)
+	for bmerr == nil {
+		return false
+	}
+
+	for curr := bmerr; curr != nil; curr = curr.OriginalError {
+		if curr.Category == CatForeign && curr.Message == original.Error() && curr.WrappedErrType == fmt.Sprintf("%T", original) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// HasSourceMessage tests if the supplied error stems originally from an error with the message msg
+func HasSourceMessage(e error, msg string) bool {
+	if e == nil {
+		return false
+	}
+
+	bmerr := FromError(e)
+	for bmerr == nil {
+		return false
+	}
+
+	for curr := bmerr; curr != nil; curr = curr.OriginalError {
+		if curr.OriginalError == nil && curr.Message == msg {
+			return true
+		}
+	}
+
+	return false
+}
+
+func MessageMatch(e error, matcher func(string) bool) bool {
+	if e == nil {
+		return false
+	}
+
+	if matcher(e.Error()) {
+		return true
+	}
+
+	bmerr := FromError(e)
+	for bmerr == nil {
+		return false
+	}
+
+	for curr := bmerr; curr != nil; curr = curr.OriginalError {
+		if matcher(curr.Message) {
+			return true
+		}
+	}
+
+	return false
+}

exerr/listener.go

@@ -0,0 +1,37 @@
+package exerr
+
+import (
+	"sync"
+)
+
+type Method string
+
+const (
+	MethodOutput Method = "OUTPUT"
+	MethodPrint  Method = "PRINT"
+	MethodBuild  Method = "BUILD"
+	MethodFatal  Method = "FATAL"
+)
+
+type Listener = func(method Method, v *ExErr)
+
+var listenerLock = sync.Mutex{}
+var listener = make([]Listener, 0)
+
+func RegisterListener(l Listener) {
+	listenerLock.Lock()
+	defer listenerLock.Unlock()
+
+	listener = append(listener, l)
+}
+
+func (b *Builder) CallListener(m Method) {
+	valErr := b.errorData
+
+	listenerLock.Lock()
+	defer listenerLock.Unlock()
+
+	for _, v := range listener {
+		v(m, valErr)
+	}
+}

exerr/meta.go

@@ -0,0 +1,713 @@
+package exerr
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/rs/zerolog"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// This is a buffed up map[string]any
+// we also save type information of the map-values
+// which allows us to deserialize them back into te correct types later
+
+type MetaMap map[string]MetaValue
+
+type metaDataType string
+
+const (
+	MDTString      metaDataType = "String"
+	MDTStringPtr   metaDataType = "StringPtr"
+	MDTInt         metaDataType = "Int"
+	MDTInt8        metaDataType = "Int8"
+	MDTInt16       metaDataType = "Int16"
+	MDTInt32       metaDataType = "Int32"
+	MDTInt64       metaDataType = "Int64"
+	MDTFloat32     metaDataType = "Float32"
+	MDTFloat64     metaDataType = "Float64"
+	MDTBool        metaDataType = "Bool"
+	MDTBytes       metaDataType = "Bytes"
+	MDTObjectID    metaDataType = "ObjectID"
+	MDTTime        metaDataType = "Time"
+	MDTDuration    metaDataType = "Duration"
+	MDTStringArray metaDataType = "StringArr"
+	MDTIntArray    metaDataType = "IntArr"
+	MDTInt32Array  metaDataType = "Int32Arr"
+	MDTID          metaDataType = "ID"
+	MDTAny         metaDataType = "Interface"
+	MDTNil         metaDataType = "Nil"
+	MDTEnum        metaDataType = "Enum"
+)
+
+type MetaValue struct {
+	DataType metaDataType `json:"dataType"`
+	Value    interface{}  `json:"value"`
+}
+
+type metaValueSerialization struct {
+	DataType metaDataType `bson:"dataType"`
+	Value    string       `bson:"value"`
+	Raw      interface{}  `bson:"raw"`
+}
+
+func (v MetaValue) SerializeValue() (string, error) {
+	switch v.DataType {
+	case MDTString:
+		return v.Value.(string), nil
+	case MDTID:
+		return v.Value.(IDWrap).Serialize(), nil
+	case MDTAny:
+		return v.Value.(AnyWrap).Serialize(), nil
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r := v.Value.(*string)
+		if r != nil {
+			return "*" + *r, nil
+		} else {
+			return "#", nil
+		}
+	case MDTInt:
+		return strconv.Itoa(v.Value.(int)), nil
+	case MDTInt8:
+		return strconv.FormatInt(int64(v.Value.(int8)), 10), nil
+	case MDTInt16:
+		return strconv.FormatInt(int64(v.Value.(int16)), 10), nil
+	case MDTInt32:
+		return strconv.FormatInt(int64(v.Value.(int32)), 10), nil
+	case MDTInt64:
+		return strconv.FormatInt(v.Value.(int64), 10), nil
+	case MDTFloat32:
+		return strconv.FormatFloat(float64(v.Value.(float32)), 'X', -1, 32), nil
+	case MDTFloat64:
+		return strconv.FormatFloat(v.Value.(float64), 'X', -1, 64), nil
+	case MDTBool:
+		if v.Value.(bool) {
+			return "true", nil
+		} else {
+			return "false", nil
+		}
+	case MDTBytes:
+		return hex.EncodeToString(v.Value.([]byte)), nil
+	case MDTObjectID:
+		return v.Value.(primitive.ObjectID).Hex(), nil
+	case MDTTime:
+		return strconv.FormatInt(v.Value.(time.Time).Unix(), 10) + "|" + strconv.FormatInt(int64(v.Value.(time.Time).Nanosecond()), 10), nil
+	case MDTDuration:
+		return v.Value.(time.Duration).String(), nil
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r, err := json.Marshal(v.Value.([]string))
+		if err != nil {
+			return "", err
+		}
+		return string(r), nil
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r, err := json.Marshal(v.Value.([]int))
+		if err != nil {
+			return "", err
+		}
+		return string(r), nil
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return "#", nil
+		}
+		r, err := json.Marshal(v.Value.([]int32))
+		if err != nil {
+			return "", err
+		}
+		return string(r), nil
+	case MDTNil:
+		return "", nil
+	case MDTEnum:
+		return v.Value.(EnumWrap).Serialize(), nil
+	}
+	return "", errors.New("Unknown type: " + string(v.DataType))
+}
+
+func (v MetaValue) ShortString(lim int) string {
+	switch v.DataType {
+	case MDTString:
+		r := strings.ReplaceAll(v.Value.(string), "\r", "")
+		r = strings.ReplaceAll(r, "\n", "\\n")
+		r = strings.ReplaceAll(r, "\t", "\\t")
+		return langext.StrLimit(r, lim, "...")
+	case MDTID:
+		return v.Value.(IDWrap).String()
+	case MDTAny:
+		return v.Value.(AnyWrap).String()
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r := langext.CoalesceString(v.Value.(*string), "<<null>>")
+		r = strings.ReplaceAll(r, "\r", "")
+		r = strings.ReplaceAll(r, "\n", "\\n")
+		r = strings.ReplaceAll(r, "\t", "\\t")
+		return langext.StrLimit(r, lim, "...")
+	case MDTInt:
+		return strconv.Itoa(v.Value.(int))
+	case MDTInt8:
+		return strconv.FormatInt(int64(v.Value.(int8)), 10)
+	case MDTInt16:
+		return strconv.FormatInt(int64(v.Value.(int16)), 10)
+	case MDTInt32:
+		return strconv.FormatInt(int64(v.Value.(int32)), 10)
+	case MDTInt64:
+		return strconv.FormatInt(v.Value.(int64), 10)
+	case MDTFloat32:
+		return strconv.FormatFloat(float64(v.Value.(float32)), 'g', 4, 32)
+	case MDTFloat64:
+		return strconv.FormatFloat(v.Value.(float64), 'g', 4, 64)
+	case MDTBool:
+		return fmt.Sprintf("%v", v.Value.(bool))
+	case MDTBytes:
+		return langext.StrLimit(hex.EncodeToString(v.Value.([]byte)), lim, "...")
+	case MDTObjectID:
+		return v.Value.(primitive.ObjectID).Hex()
+	case MDTTime:
+		return v.Value.(time.Time).Format(time.RFC3339)
+	case MDTDuration:
+		return v.Value.(time.Duration).String()
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.Marshal(v.Value.([]string))
+		if err != nil {
+			return "(err)"
+		}
+		return langext.StrLimit(string(r), lim, "...")
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.Marshal(v.Value.([]int))
+		if err != nil {
+			return "(err)"
+		}
+		return langext.StrLimit(string(r), lim, "...")
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.Marshal(v.Value.([]int32))
+		if err != nil {
+			return "(err)"
+		}
+		return langext.StrLimit(string(r), lim, "...")
+	case MDTNil:
+		return "<<null>>"
+	case MDTEnum:
+		return v.Value.(EnumWrap).String()
+	}
+	return "(err)"
+}
+
+func (v MetaValue) Apply(key string, evt *zerolog.Event) *zerolog.Event {
+	switch v.DataType {
+	case MDTString:
+		return evt.Str(key, v.Value.(string))
+	case MDTID:
+		return evt.Str(key, v.Value.(IDWrap).Value)
+	case MDTAny:
+		if v.Value.(AnyWrap).IsError {
+			return evt.Str(key, "(err)")
+		} else {
+			return evt.Str(key, v.Value.(AnyWrap).Json)
+		}
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return evt.Str(key, "<<null>>")
+		}
+		return evt.Str(key, langext.CoalesceString(v.Value.(*string), "<<null>>"))
+	case MDTInt:
+		return evt.Int(key, v.Value.(int))
+	case MDTInt8:
+		return evt.Int8(key, v.Value.(int8))
+	case MDTInt16:
+		return evt.Int16(key, v.Value.(int16))
+	case MDTInt32:
+		return evt.Int32(key, v.Value.(int32))
+	case MDTInt64:
+		return evt.Int64(key, v.Value.(int64))
+	case MDTFloat32:
+		return evt.Float32(key, v.Value.(float32))
+	case MDTFloat64:
+		return evt.Float64(key, v.Value.(float64))
+	case MDTBool:
+		return evt.Bool(key, v.Value.(bool))
+	case MDTBytes:
+		return evt.Bytes(key, v.Value.([]byte))
+	case MDTObjectID:
+		return evt.Str(key, v.Value.(primitive.ObjectID).Hex())
+	case MDTTime:
+		return evt.Time(key, v.Value.(time.Time))
+	case MDTDuration:
+		return evt.Dur(key, v.Value.(time.Duration))
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return evt.Strs(key, nil)
+		}
+		return evt.Strs(key, v.Value.([]string))
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return evt.Ints(key, nil)
+		}
+		return evt.Ints(key, v.Value.([]int))
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return evt.Ints32(key, nil)
+		}
+		return evt.Ints32(key, v.Value.([]int32))
+	case MDTNil:
+		return evt.Str(key, "<<null>>")
+	case MDTEnum:
+		if v.Value.(EnumWrap).IsNil {
+			return evt.Any(key, nil)
+		} else if v.Value.(EnumWrap).ValueRaw != nil {
+			return evt.Any(key, v.Value.(EnumWrap).ValueRaw)
+		} else {
+			return evt.Str(key, v.Value.(EnumWrap).ValueString)
+		}
+	}
+	return evt.Str(key, "(err)")
+}
+
+func (v MetaValue) MarshalJSON() ([]byte, error) {
+	str, err := v.SerializeValue()
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(string(v.DataType) + ":" + str)
+}
+
+func (v *MetaValue) UnmarshalJSON(data []byte) error {
+	var str = ""
+	err := json.Unmarshal(data, &str)
+	if err != nil {
+		return err
+	}
+
+	split := strings.SplitN(str, ":", 2)
+	if len(split) != 2 {
+		return errors.New("failed to decode MetaValue: '" + str + "'")
+	}
+
+	return v.Deserialize(split[1], metaDataType(split[0]))
+}
+
+func (v MetaValue) MarshalBSON() ([]byte, error) {
+	serval, err := v.SerializeValue()
+	if err != nil {
+		return nil, Wrap(err, "failed to bson-marshal MetaValue (serialize)").Build()
+	}
+
+	// this is an kinda ugly hack - but serialization to mongodb and back can loose the correct type information....
+	bin, err := bson.Marshal(metaValueSerialization{
+		DataType: v.DataType,
+		Value:    serval,
+		Raw:      v.Value,
+	})
+	if err != nil {
+		return nil, Wrap(err, "failed to bson-marshal MetaValue (marshal)").Build()
+	}
+
+	return bin, nil
+}
+
+func (v *MetaValue) UnmarshalBSON(bytes []byte) error {
+	var serval metaValueSerialization
+	err := bson.Unmarshal(bytes, &serval)
+	if err != nil {
+		return Wrap(err, "failed to bson-unmarshal MetaValue (unmarshal)").Build()
+	}
+
+	err = v.Deserialize(serval.Value, serval.DataType)
+	if err != nil {
+		return Wrap(err, "failed to deserialize MetaValue from bson").Str("raw", serval.Value).Build()
+	}
+
+	return nil
+}
+
+func (v *MetaValue) Deserialize(value string, datatype metaDataType) error {
+	switch datatype {
+	case MDTString:
+		v.Value = value
+		v.DataType = datatype
+		return nil
+	case MDTID:
+		v.Value = deserializeIDWrap(value)
+		v.DataType = datatype
+		return nil
+	case MDTAny:
+		v.Value = deserializeAnyWrap(value)
+		v.DataType = datatype
+		return nil
+	case MDTStringPtr:
+		if len(value) <= 0 || (value[0] != '*' && value[0] != '#') {
+			return errors.New("Invalid StringPtr: " + value)
+		} else if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		} else {
+			v.Value = langext.Ptr(value[1:])
+			v.DataType = datatype
+			return nil
+		}
+	case MDTInt:
+		pv, err := strconv.ParseInt(value, 10, 0)
+		if err != nil {
+			return err
+		}
+		v.Value = int(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt8:
+		pv, err := strconv.ParseInt(value, 10, 8)
+		if err != nil {
+			return err
+		}
+		v.Value = int8(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt16:
+		pv, err := strconv.ParseInt(value, 10, 16)
+		if err != nil {
+			return err
+		}
+		v.Value = int16(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt32:
+		pv, err := strconv.ParseInt(value, 10, 32)
+		if err != nil {
+			return err
+		}
+		v.Value = int32(pv)
+		v.DataType = datatype
+		return nil
+	case MDTInt64:
+		pv, err := strconv.ParseInt(value, 10, 64)
+		if err != nil {
+			return err
+		}
+		v.Value = pv
+		v.DataType = datatype
+		return nil
+	case MDTFloat32:
+		pv, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			return err
+		}
+		v.Value = float32(pv)
+		v.DataType = datatype
+		return nil
+	case MDTFloat64:
+		pv, err := strconv.ParseFloat(value, 64)
+		if err != nil {
+			return err
+		}
+		v.Value = pv
+		v.DataType = datatype
+		return nil
+	case MDTBool:
+		if value == "true" {
+			v.Value = true
+			v.DataType = datatype
+			return nil
+		}
+		if value == "false" {
+			v.Value = false
+			v.DataType = datatype
+			return nil
+		}
+		return errors.New("invalid bool value: " + value)
+	case MDTBytes:
+		r, err := hex.DecodeString(value)
+		if err != nil {
+			return err
+		}
+		v.Value = r
+		v.DataType = datatype
+		return nil
+	case MDTObjectID:
+		r, err := primitive.ObjectIDFromHex(value)
+		if err != nil {
+			return err
+		}
+		v.Value = r
+		v.DataType = datatype
+		return nil
+	case MDTTime:
+		ps := strings.Split(value, "|")
+		if len(ps) != 2 {
+			return errors.New("invalid time.time: " + value)
+		}
+		p1, err := strconv.ParseInt(ps[0], 10, 64)
+		if err != nil {
+			return err
+		}
+		p2, err := strconv.ParseInt(ps[1], 10, 32)
+		if err != nil {
+			return err
+		}
+		v.Value = time.Unix(p1, p2)
+		v.DataType = datatype
+		return nil
+	case MDTDuration:
+		r, err := time.ParseDuration(value)
+		if err != nil {
+			return err
+		}
+		v.Value = r
+		v.DataType = datatype
+		return nil
+	case MDTStringArray:
+		if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		}
+		pj := make([]string, 0)
+		err := json.Unmarshal([]byte(value), &pj)
+		if err != nil {
+			return err
+		}
+		v.Value = pj
+		v.DataType = datatype
+		return nil
+	case MDTIntArray:
+		if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		}
+		pj := make([]int, 0)
+		err := json.Unmarshal([]byte(value), &pj)
+		if err != nil {
+			return err
+		}
+		v.Value = pj
+		v.DataType = datatype
+		return nil
+	case MDTInt32Array:
+		if value == "#" {
+			v.Value = nil
+			v.DataType = datatype
+			return nil
+		}
+		pj := make([]int32, 0)
+		err := json.Unmarshal([]byte(value), &pj)
+		if err != nil {
+			return err
+		}
+		v.Value = pj
+		v.DataType = datatype
+		return nil
+	case MDTNil:
+		v.Value = nil
+		v.DataType = datatype
+		return nil
+	case MDTEnum:
+		v.Value = deserializeEnumWrap(value)
+		v.DataType = datatype
+		return nil
+	}
+	return errors.New("Unknown type: " + string(datatype))
+}
+
+func (v MetaValue) ValueString() string {
+	switch v.DataType {
+	case MDTString:
+		return v.Value.(string)
+	case MDTID:
+		return v.Value.(IDWrap).String()
+	case MDTAny:
+		return v.Value.(AnyWrap).String()
+	case MDTStringPtr:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		return langext.CoalesceString(v.Value.(*string), "<<null>>")
+	case MDTInt:
+		return strconv.Itoa(v.Value.(int))
+	case MDTInt8:
+		return strconv.FormatInt(int64(v.Value.(int8)), 10)
+	case MDTInt16:
+		return strconv.FormatInt(int64(v.Value.(int16)), 10)
+	case MDTInt32:
+		return strconv.FormatInt(int64(v.Value.(int32)), 10)
+	case MDTInt64:
+		return strconv.FormatInt(v.Value.(int64), 10)
+	case MDTFloat32:
+		return strconv.FormatFloat(float64(v.Value.(float32)), 'g', 4, 32)
+	case MDTFloat64:
+		return strconv.FormatFloat(v.Value.(float64), 'g', 4, 64)
+	case MDTBool:
+		return fmt.Sprintf("%v", v.Value.(bool))
+	case MDTBytes:
+		return hex.EncodeToString(v.Value.([]byte))
+	case MDTObjectID:
+		return v.Value.(primitive.ObjectID).Hex()
+	case MDTTime:
+		return v.Value.(time.Time).Format(time.RFC3339Nano)
+	case MDTDuration:
+		return v.Value.(time.Duration).String()
+	case MDTStringArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.MarshalIndent(v.Value.([]string), "", "  ")
+		if err != nil {
+			return "(err)"
+		}
+		return string(r)
+	case MDTIntArray:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.MarshalIndent(v.Value.([]int), "", "  ")
+		if err != nil {
+			return "(err)"
+		}
+		return string(r)
+	case MDTInt32Array:
+		if langext.IsNil(v.Value) {
+			return "<<null>>"
+		}
+		r, err := json.MarshalIndent(v.Value.([]int32), "", "  ")
+		if err != nil {
+			return "(err)"
+		}
+		return string(r)
+	case MDTNil:
+		return "<<null>>"
+	case MDTEnum:
+		return v.Value.(EnumWrap).String()
+	}
+	return "(err)"
+}
+
+// rawValueForJson returns most-of-the-time the `Value` field
+// but for some datatyes we do special processing
+// all, so we can pluck the output value in json.Marshal without any suprises
+func (v MetaValue) rawValueForJson() any {
+	if v.DataType == MDTAny {
+		if v.Value.(AnyWrap).IsNil {
+			return nil
+		}
+		return v.Value.(AnyWrap).Serialize()
+	}
+	if v.DataType == MDTID {
+		if v.Value.(IDWrap).IsNil {
+			return nil
+		}
+		return v.Value.(IDWrap).Value
+	}
+	if v.DataType == MDTBytes {
+		return hex.EncodeToString(v.Value.([]byte))
+	}
+	if v.DataType == MDTDuration {
+		return v.Value.(time.Duration).String()
+	}
+	if v.DataType == MDTTime {
+		return v.Value.(time.Time).Format(time.RFC3339Nano)
+	}
+	if v.DataType == MDTObjectID {
+		return v.Value.(primitive.ObjectID).Hex()
+	}
+	if v.DataType == MDTNil {
+		return nil
+	}
+	if v.DataType == MDTEnum {
+		if v.Value.(EnumWrap).IsNil {
+			return nil
+		}
+		if v.Value.(EnumWrap).ValueRaw != nil {
+			return v.Value.(EnumWrap).ValueRaw
+		}
+		return v.Value.(EnumWrap).ValueString
+	}
+	return v.Value
+}
+
+func (mm MetaMap) FormatOneLine(singleMaxLen int) string {
+	r := ""
+
+	i := 0
+	for key, val := range mm {
+		if i > 0 {
+			r += ", "
+		}
+
+		r += "\"" + key + "\""
+		r += ": "
+		r += "\"" + val.ShortString(singleMaxLen) + "\""
+
+		i++
+	}
+
+	return r
+}
+
+func (mm MetaMap) FormatMultiLine(indentFront string, indentKeys string, maxLenValue int) string {
+	r := ""
+
+	r += indentFront + "{" + "\n"
+	for key, val := range mm {
+		if key == "gin.body" {
+			continue
+		}
+
+		r += indentFront
+		r += indentKeys
+		r += "\"" + key + "\""
+		r += ": "
+		r += "\"" + val.ShortString(maxLenValue) + "\""
+		r += ",\n"
+	}
+	r += indentFront + "}"
+
+	return r
+}
+
+func (mm MetaMap) Any() bool {
+	return len(mm) > 0
+}
+
+func (mm MetaMap) Apply(evt *zerolog.Event) *zerolog.Event {
+	for key, val := range mm {
+		evt = val.Apply(key, evt)
+	}
+	return evt
+}
+
+func (mm MetaMap) add(key string, mdtype metaDataType, val interface{}) {
+	if _, ok := mm[key]; !ok {
+		mm[key] = MetaValue{DataType: mdtype, Value: val}
+		return
+	}
+	for i := 2; ; i++ {
+		realkey := key + "-" + strconv.Itoa(i)
+		if _, ok := mm[realkey]; !ok {
+			mm[realkey] = MetaValue{DataType: mdtype, Value: val}
+			return
+		}
+	}
+}

exerr/stacktrace.go

@@ -0,0 +1,14 @@
+package exerr
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func callername(skip int) string {
+	pc := make([]uintptr, 15)
+	n := runtime.Callers(skip+2, pc)
+	frames := runtime.CallersFrames(pc[:n])
+	frame, _ := frames.Next()
+	return fmt.Sprintf("%s:%d %s", frame.File, frame.Line, frame.Function)
+}

exerr/typeWrapper.go

@@ -0,0 +1,189 @@
+package exerr
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/rs/zerolog/log"
+	"gogs.mikescher.com/BlackForestBytes/goext/enums"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"strings"
+)
+
+//
+// These are wrapper objects, because for some metadata-types we need to serialize a bit more complex data
+// (eg thy actual type for ID objects, or the json representation for any types)
+//
+
+type IDWrap struct {
+	Type  string
+	Value string
+	IsNil bool
+}
+
+func newIDWrap(val fmt.Stringer) IDWrap {
+	t := fmt.Sprintf("%T", val)
+	arr := strings.Split(t, ".")
+	if len(arr) > 0 {
+		t = arr[len(arr)-1]
+	}
+
+	if langext.IsNil(val) {
+		return IDWrap{Type: t, Value: "", IsNil: true}
+	}
+
+	v := val.String()
+	return IDWrap{Type: t, Value: v, IsNil: false}
+}
+
+func (w IDWrap) Serialize() string {
+	if w.IsNil {
+		return "!nil" + ":" + w.Type
+	}
+	return w.Type + ":" + w.Value
+}
+
+func (w IDWrap) String() string {
+	if w.IsNil {
+		return w.Type + "<<nil>>"
+	}
+	return w.Type + "(" + w.Value + ")"
+}
+
+func deserializeIDWrap(v string) IDWrap {
+	r := strings.SplitN(v, ":", 2)
+
+	if len(r) == 2 && r[0] == "!nil" {
+		return IDWrap{Type: r[1], Value: v, IsNil: true}
+	}
+
+	if len(r) == 0 {
+		return IDWrap{}
+	} else if len(r) == 1 {
+		return IDWrap{Type: "", Value: v, IsNil: false}
+	} else {
+		return IDWrap{Type: r[0], Value: r[1], IsNil: false}
+	}
+}
+
+type AnyWrap struct {
+	Type    string
+	Json    string
+	IsError bool
+	IsNil   bool
+}
+
+func newAnyWrap(val any) (result AnyWrap) {
+	result = AnyWrap{Type: "", Json: "", IsError: true, IsNil: false} // ensure a return in case of recover()
+
+	defer func() {
+		if err := recover(); err != nil {
+			// send error should never crash our program
+			log.Error().Interface("err", err).Msg("Panic while trying to marshal anywrap ( bmerror.Interface )")
+		}
+	}()
+
+	t := fmt.Sprintf("%T", val)
+
+	if langext.IsNil(val) {
+		return AnyWrap{Type: t, Json: "", IsError: false, IsNil: true}
+	}
+
+	j, err := json.Marshal(val)
+	if err == nil {
+		return AnyWrap{Type: t, Json: string(j), IsError: false, IsNil: false}
+	} else {
+		return AnyWrap{Type: t, Json: "", IsError: true, IsNil: false}
+	}
+}
+
+func (w AnyWrap) Serialize() string {
+	if w.IsError {
+		return "ERR" + ":" + w.Type + ":" + w.Json
+	} else if w.IsNil {
+		return "NIL" + ":" + w.Type + ":" + w.Json
+	} else {
+		return "OK" + ":" + w.Type + ":" + w.Json
+	}
+}
+
+func (w AnyWrap) String() string {
+	if w.IsError {
+		return "(error)"
+	} else if w.IsNil {
+		return "(nil)"
+	} else {
+		return w.Json
+	}
+}
+
+func deserializeAnyWrap(v string) AnyWrap {
+	r := strings.SplitN(v, ":", 3)
+	if len(r) != 3 {
+		return AnyWrap{IsError: true, Type: "", Json: "", IsNil: false}
+	} else {
+		if r[0] == "OK" {
+			return AnyWrap{IsError: false, Type: r[1], Json: r[2], IsNil: false}
+		} else if r[0] == "ERR" {
+			return AnyWrap{IsError: true, Type: r[1], Json: r[2], IsNil: false}
+		} else if r[0] == "NIL" {
+			return AnyWrap{IsError: false, Type: r[1], Json: "", IsNil: true}
+		} else {
+			return AnyWrap{IsError: true, Type: "", Json: "", IsNil: false}
+		}
+	}
+}
+
+type EnumWrap struct {
+	Type        string
+	ValueString string
+	ValueRaw    enums.Enum // `ValueRaw` is lost during serialization roundtrip
+	IsNil       bool
+}
+
+func newEnumWrap(val enums.Enum) EnumWrap {
+	t := fmt.Sprintf("%T", val)
+	arr := strings.Split(t, ".")
+	if len(arr) > 0 {
+		t = arr[len(arr)-1]
+	}
+
+	if langext.IsNil(val) {
+		return EnumWrap{Type: t, ValueString: "", ValueRaw: val, IsNil: true}
+	}
+
+	if enumstr, ok := val.(enums.StringEnum); ok {
+		return EnumWrap{Type: t, ValueString: enumstr.String(), ValueRaw: val, IsNil: false}
+	}
+
+	return EnumWrap{Type: t, ValueString: fmt.Sprintf("%v", val), ValueRaw: val, IsNil: false}
+}
+
+func (w EnumWrap) Serialize() string {
+	if w.IsNil {
+		return "!nil" + ":" + w.Type
+	}
+	return w.Type + ":" + w.ValueString
+}
+
+func (w EnumWrap) String() string {
+	if w.IsNil {
+		return w.Type + "<<nil>>"
+	}
+	return "[" + w.Type + "] " + w.ValueString
+}
+
+func deserializeEnumWrap(v string) EnumWrap {
+	r := strings.SplitN(v, ":", 2)
+
+	if len(r) == 2 && r[0] == "!nil" {
+		return EnumWrap{Type: r[1], ValueString: v, ValueRaw: nil, IsNil: true}
+	}
+
+	if len(r) == 0 {
+		return EnumWrap{}
+	} else if len(r) == 1 {
+		return EnumWrap{Type: "", ValueString: v, ValueRaw: nil, IsNil: false}
+	} else {
+		return EnumWrap{Type: r[0], ValueString: r[1], ValueRaw: nil, IsNil: false}
+	}
+}

ginext/appContext.go

@@ -0,0 +1,59 @@
+package ginext
+
+import (
+	"context"
+	"github.com/gin-gonic/gin"
+	"time"
+)
+
+type AppContext struct {
+	inner      context.Context
+	cancelFunc context.CancelFunc
+	cancelled  bool
+	GinContext *gin.Context
+}
+
+func CreateAppContext(g *gin.Context, innerCtx context.Context, cancelFn context.CancelFunc) *AppContext {
+	for key, value := range g.Keys {
+		innerCtx = context.WithValue(innerCtx, key, value)
+	}
+	return &AppContext{
+		inner:      innerCtx,
+		cancelFunc: cancelFn,
+		cancelled:  false,
+		GinContext: g,
+	}
+}
+
+func (ac *AppContext) Deadline() (deadline time.Time, ok bool) {
+	return ac.inner.Deadline()
+}
+
+func (ac *AppContext) Done() <-chan struct{} {
+	return ac.inner.Done()
+}
+
+func (ac *AppContext) Err() error {
+	return ac.inner.Err()
+}
+
+func (ac *AppContext) Value(key any) any {
+	return ac.inner.Value(key)
+}
+
+func (ac *AppContext) Set(key, value any) {
+	ac.inner = context.WithValue(ac.inner, key, value)
+}
+
+func (ac *AppContext) Cancel() {
+	ac.cancelled = true
+	ac.cancelFunc()
+}
+
+func (ac *AppContext) RequestURI() string {
+	if ac.GinContext != nil && ac.GinContext.Request != nil {
+		return ac.GinContext.Request.Method + " :: " + ac.GinContext.Request.RequestURI
+	} else {
+		return ""
+	}
+}

ginext/commonHandler.go

@@ -0,0 +1,23 @@
+package ginext
+
+import (
+	"net/http"
+)
+
+func RedirectFound(newuri string) WHandlerFunc {
+	return func(pctx PreContext) HTTPResponse {
+		return Redirect(http.StatusFound, newuri)
+	}
+}
+
+func RedirectTemporary(newuri string) WHandlerFunc {
+	return func(pctx PreContext) HTTPResponse {
+		return Redirect(http.StatusTemporaryRedirect, newuri)
+	}
+}
+
+func RedirectPermanent(newuri string) WHandlerFunc {
+	return func(pctx PreContext) HTTPResponse {
+		return Redirect(http.StatusPermanentRedirect, newuri)
+	}
+}

ginext/commonMiddlewares.go

@@ -0,0 +1,12 @@
+package ginext
+
+import (
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+)
+
+func BodyBuffer(g *gin.Context) {
+	if g.Request.Body != nil {
+		g.Request.Body = dataext.NewBufferedReadCloser(g.Request.Body)
+	}
+}

ginext/cors.go

@@ -0,0 +1,21 @@
+package ginext
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func CorsMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
+		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
+		c.Writer.Header().Set("Access-Control-Allow-Methods", "OPTIONS, GET, POST, PUT, PATCH, DELETE, COUNT")
+
+		if c.Request.Method == "OPTIONS" {
+			c.AbortWithStatus(http.StatusOK)
+		} else {
+			c.Next()
+		}
+	}
+}

ginext/engine.go

@@ -0,0 +1,149 @@
+package ginext
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+type GinWrapper struct {
+	engine          *gin.Engine
+	SuppressGinLogs bool
+
+	allowCors      bool
+	ginDebug       bool
+	bufferBody     bool
+	requestTimeout time.Duration
+
+	routeSpecs []ginRouteSpec
+}
+
+type ginRouteSpec struct {
+	Method      string
+	URL         string
+	Middlewares []string
+	Handler     string
+}
+
+// NewEngine creates a new (wrapped) ginEngine
+// Parameters are:
+// - [allowCors]    Add cors handler to allow all CORS requests on the default http methods
+// - [ginDebug]     Set gin.debug to true (adds more logs)
+// - [bufferBody]   Buffers the input body stream, this way the ginext error handler can later include the whole request body
+// - [timeout]      The default handler timeout
+func NewEngine(allowCors bool, ginDebug bool, bufferBody bool, timeout time.Duration) *GinWrapper {
+	engine := gin.New()
+
+	wrapper := &GinWrapper{
+		engine:          engine,
+		SuppressGinLogs: false,
+		allowCors:       allowCors,
+		ginDebug:        ginDebug,
+		bufferBody:      bufferBody,
+		requestTimeout:  timeout,
+	}
+
+	engine.RedirectFixedPath = false
+	engine.RedirectTrailingSlash = false
+
+	if allowCors {
+		engine.Use(CorsMiddleware())
+	}
+
+	// do not debug-print routes
+	gin.DebugPrintRouteFunc = func(_, _, _ string, _ int) {}
+
+	if ginDebug {
+		gin.SetMode(gin.ReleaseMode)
+
+		ginlogger := gin.Logger()
+		engine.Use(func(context *gin.Context) {
+			if !wrapper.SuppressGinLogs {
+				ginlogger(context)
+			}
+		})
+	} else {
+		gin.SetMode(gin.DebugMode)
+	}
+
+	return wrapper
+}
+
+func (w *GinWrapper) ListenAndServeHTTP(addr string, postInit func(port string)) (chan error, *http.Server) {
+
+	w.DebugPrintRoutes()
+
+	httpserver := &http.Server{
+		Addr:    addr,
+		Handler: w.engine,
+	}
+
+	errChan := make(chan error)
+
+	go func() {
+
+		ln, err := net.Listen("tcp", httpserver.Addr)
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		_, port, err := net.SplitHostPort(ln.Addr().String())
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		log.Info().Str("address", httpserver.Addr).Msg("HTTP-Server started on http://localhost:" + port)
+
+		if postInit != nil {
+			postInit(port) // the net.Listener a few lines above is at this point actually already buffering requests
+		}
+
+		errChan <- httpserver.Serve(ln)
+	}()
+
+	return errChan, httpserver
+}
+
+func (w *GinWrapper) DebugPrintRoutes() {
+	if !w.ginDebug {
+		return
+	}
+
+	lines := make([][4]string, 0)
+
+	pad := [4]int{0, 0, 0, 0}
+
+	for _, spec := range w.routeSpecs {
+
+		line := [4]string{
+			spec.Method,
+			spec.URL,
+			strings.Join(spec.Middlewares, " -> "),
+			spec.Handler,
+		}
+
+		lines = append(lines, line)
+
+		pad[0] = mathext.Max(pad[0], len(line[0]))
+		pad[1] = mathext.Max(pad[1], len(line[1]))
+		pad[2] = mathext.Max(pad[2], len(line[2]))
+		pad[3] = mathext.Max(pad[3], len(line[3]))
+	}
+
+	for _, line := range lines {
+
+		fmt.Printf("Gin-Route: %s  %s  -->  %s  -->  %s\n",
+			langext.StrPadRight("["+line[0]+"]", " ", pad[0]+2),
+			langext.StrPadRight(line[1], " ", pad[1]),
+			langext.StrPadRight(line[2], " ", pad[2]),
+			langext.StrPadRight(line[3], " ", pad[3]))
+	}
+}

ginext/funcWrapper.go

@@ -0,0 +1,39 @@
+package ginext
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+)
+
+type WHandlerFunc func(PreContext) HTTPResponse
+
+func Wrap(w *GinWrapper, fn WHandlerFunc) gin.HandlerFunc {
+
+	return func(g *gin.Context) {
+
+		reqctx := g.Request.Context()
+
+		wrap, stackTrace, panicObj := callPanicSafe(fn, PreContext{wrapper: w, ginCtx: g})
+		if panicObj != nil {
+
+			fmt.Printf("\n======== ======== STACKTRACE ======== ========\n%s\n======== ======== ======== ========\n\n", stackTrace)
+
+			err := exerr.
+				New(exerr.TypePanic, "Panic occured (in gin handler)").
+				Any("panicObj", panicObj).
+				Str("trace", stackTrace).
+				Build()
+
+			wrap = Error(err)
+		}
+
+		if g.Writer.Written() {
+			panic("Writing in WrapperFunc is not supported")
+		}
+
+		if reqctx.Err() == nil {
+			wrap.Write(g)
+		}
+	}
+}

ginext/preContext.go

@@ -0,0 +1,145 @@
+package ginext
+
+import (
+	"context"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/gin-gonic/gin/binding"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"runtime/debug"
+	"time"
+)
+
+type PreContext struct {
+	ginCtx  *gin.Context
+	wrapper *GinWrapper
+	uri     any
+	query   any
+	body    any
+	form    any
+	header  any
+	timeout *time.Duration
+}
+
+func (pctx *PreContext) URI(uri any) *PreContext {
+	pctx.uri = uri
+	return pctx
+}
+
+func (pctx *PreContext) Query(query any) *PreContext {
+	pctx.query = query
+	return pctx
+}
+
+func (pctx *PreContext) Body(body any) *PreContext {
+	pctx.body = body
+	return pctx
+}
+
+func (pctx *PreContext) Form(form any) *PreContext {
+	pctx.form = form
+	return pctx
+}
+
+func (pctx *PreContext) Header(header any) *PreContext {
+	pctx.header = header
+	return pctx
+}
+
+func (pctx *PreContext) WithTimeout(to time.Duration) *PreContext {
+	pctx.timeout = &to
+	return pctx
+}
+
+func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
+	if pctx.uri != nil {
+		if err := pctx.ginCtx.ShouldBindUri(pctx.uri); err != nil {
+			err = exerr.Wrap(err, "Failed to read uri").
+				WithType(exerr.TypeBindFailURI).
+				Str("struct_type", fmt.Sprintf("%T", pctx.uri)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.query != nil {
+		if err := pctx.ginCtx.ShouldBindQuery(pctx.query); err != nil {
+			err = exerr.Wrap(err, "Failed to read query").
+				WithType(exerr.TypeBindFailQuery).
+				Str("struct_type", fmt.Sprintf("%T", pctx.query)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.body != nil {
+		if pctx.ginCtx.ContentType() == "application/json" {
+			if err := pctx.ginCtx.ShouldBindJSON(pctx.body); err != nil {
+				err = exerr.Wrap(err, "Failed to read json-body").
+					WithType(exerr.TypeBindFailJSON).
+					Str("struct_type", fmt.Sprintf("%T", pctx.body)).
+					Build()
+				return nil, nil, langext.Ptr(Error(err))
+			}
+		} else {
+			err := exerr.New(exerr.TypeBindFailJSON, "missing JSON body").
+				Str("struct_type", fmt.Sprintf("%T", pctx.body)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.form != nil {
+		if pctx.ginCtx.ContentType() == "multipart/form-data" {
+			if err := pctx.ginCtx.ShouldBindWith(pctx.form, binding.Form); err != nil {
+				err = exerr.Wrap(err, "Failed to read multipart-form").
+					WithType(exerr.TypeBindFailFormData).
+					Str("struct_type", fmt.Sprintf("%T", pctx.form)).
+					Build()
+				return nil, nil, langext.Ptr(Error(err))
+			}
+		} else if pctx.ginCtx.ContentType() == "application/x-www-form-urlencoded" {
+			if err := pctx.ginCtx.ShouldBindWith(pctx.form, binding.Form); err != nil {
+				err = exerr.Wrap(err, "Failed to read urlencoded-form").
+					WithType(exerr.TypeBindFailFormData).
+					Str("struct_type", fmt.Sprintf("%T", pctx.form)).
+					Build()
+				return nil, nil, langext.Ptr(Error(err))
+			}
+		} else {
+			err := exerr.New(exerr.TypeBindFailFormData, "missing form body").
+				Str("struct_type", fmt.Sprintf("%T", pctx.form)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	if pctx.header != nil {
+		if err := pctx.ginCtx.ShouldBindHeader(pctx.query); err != nil {
+			err = exerr.Wrap(err, "Failed to read header").
+				WithType(exerr.TypeBindFailHeader).
+				Str("struct_type", fmt.Sprintf("%T", pctx.query)).
+				Build()
+			return nil, nil, langext.Ptr(Error(err))
+		}
+	}
+
+	ictx, cancel := context.WithTimeout(context.Background(), langext.Coalesce(pctx.timeout, pctx.wrapper.requestTimeout))
+	actx := CreateAppContext(pctx.ginCtx, ictx, cancel)
+
+	return actx, pctx.ginCtx, nil
+}
+
+func callPanicSafe(fn WHandlerFunc, pctx PreContext) (res HTTPResponse, stackTrace string, panicObj any) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			res = nil
+			stackTrace = string(debug.Stack())
+			panicObj = rec
+		}
+	}()
+
+	res = fn(pctx)
+	return res, "", nil
+}

ginext/response.go

@@ -0,0 +1,216 @@
+package ginext
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+)
+
+type headerval struct {
+	Key string
+	Val string
+}
+
+type HTTPResponse interface {
+	Write(g *gin.Context)
+	WithHeader(k string, v string) HTTPResponse
+}
+
+type jsonHTTPResponse struct {
+	statusCode int
+	data       any
+	headers    []headerval
+}
+
+func (j jsonHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true})
+}
+
+func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type emptyHTTPResponse struct {
+	statusCode int
+	headers    []headerval
+}
+
+func (j emptyHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	g.Status(j.statusCode)
+}
+
+func (j emptyHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type textHTTPResponse struct {
+	statusCode int
+	data       string
+	headers    []headerval
+}
+
+func (j textHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	g.String(j.statusCode, "%s", j.data)
+}
+
+func (j textHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type dataHTTPResponse struct {
+	statusCode  int
+	data        []byte
+	contentType string
+	headers     []headerval
+}
+
+func (j dataHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	g.Data(j.statusCode, j.contentType, j.data)
+}
+
+func (j dataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type fileHTTPResponse struct {
+	mimetype string
+	filepath string
+	filename *string
+	headers  []headerval
+}
+
+func (j fileHTTPResponse) Write(g *gin.Context) {
+	g.Header("Content-Type", j.mimetype) // if we don't set it here gin does weird file-sniffing later...
+	if j.filename != nil {
+		g.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", *j.filename))
+
+	}
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	g.File(j.filepath)
+}
+
+func (j fileHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type downloadDataHTTPResponse struct {
+	statusCode int
+	mimetype   string
+	data       []byte
+	filename   *string
+	headers    []headerval
+}
+
+func (j downloadDataHTTPResponse) Write(g *gin.Context) {
+	g.Header("Content-Type", j.mimetype) // if we don't set it here gin does weird file-sniffing later...
+	if j.filename != nil {
+		g.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", *j.filename))
+
+	}
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	g.Data(j.statusCode, j.mimetype, j.data)
+}
+
+func (j downloadDataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type redirectHTTPResponse struct {
+	statusCode int
+	url        string
+	headers    []headerval
+}
+
+func (j redirectHTTPResponse) Write(g *gin.Context) {
+	g.Redirect(j.statusCode, j.url)
+}
+
+func (j redirectHTTPResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+type jsonAPIErrResponse struct {
+	err     *exerr.ExErr
+	headers []headerval
+}
+
+func (j jsonAPIErrResponse) Write(g *gin.Context) {
+	j.err.Output(g)
+}
+
+func (j jsonAPIErrResponse) WithHeader(k string, v string) HTTPResponse {
+	j.headers = append(j.headers, headerval{k, v})
+	return j
+}
+
+func Status(sc int) HTTPResponse {
+	return &emptyHTTPResponse{statusCode: sc}
+}
+
+func JSON(sc int, data any) HTTPResponse {
+	return &jsonHTTPResponse{statusCode: sc, data: data}
+}
+
+func Data(sc int, contentType string, data []byte) HTTPResponse {
+	return &dataHTTPResponse{statusCode: sc, contentType: contentType, data: data}
+}
+
+func Text(sc int, data string) HTTPResponse {
+	return &textHTTPResponse{statusCode: sc, data: data}
+}
+
+func File(mimetype string, filepath string) HTTPResponse {
+	return &fileHTTPResponse{mimetype: mimetype, filepath: filepath}
+}
+
+func Download(mimetype string, filepath string, filename string) HTTPResponse {
+	return &fileHTTPResponse{mimetype: mimetype, filepath: filepath, filename: &filename}
+}
+
+func DownloadData(status int, mimetype string, filename string, data []byte) HTTPResponse {
+	return &downloadDataHTTPResponse{statusCode: status, mimetype: mimetype, data: data, filename: &filename}
+}
+
+func Redirect(sc int, newURL string) HTTPResponse {
+	return &redirectHTTPResponse{statusCode: sc, url: newURL}
+}
+
+func Error(e error) HTTPResponse {
+	return &jsonAPIErrResponse{
+		err: exerr.FromError(e),
+	}
+}
+
+func ErrWrap(e error, errorType exerr.ErrorType, msg string) HTTPResponse {
+	return &jsonAPIErrResponse{
+		err: exerr.FromError(exerr.Wrap(e, msg).WithType(errorType).Build()),
+	}
+}
+
+func NotImplemented() HTTPResponse {
+	return Error(exerr.New(exerr.TypeNotImplemented, "").Build())
+}

ginext/routes.go

@@ -0,0 +1,211 @@
+package ginext
+
+import (
+	"github.com/gin-gonic/gin"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"net/http"
+	"path"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+var anyMethods = []string{
+	http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch,
+	http.MethodHead, http.MethodOptions, http.MethodDelete, http.MethodConnect,
+	http.MethodTrace,
+}
+
+type GinRoutesWrapper struct {
+	wrapper        *GinWrapper
+	routes         gin.IRouter
+	absPath        string
+	defaultHandler []gin.HandlerFunc
+}
+
+type GinRouteBuilder struct {
+	routes *GinRoutesWrapper
+
+	method   string
+	relPath  string
+	absPath  string
+	handlers []gin.HandlerFunc
+}
+
+func (w *GinWrapper) Routes() *GinRoutesWrapper {
+	return &GinRoutesWrapper{
+		wrapper:        w,
+		routes:         w.engine,
+		absPath:        "",
+		defaultHandler: make([]gin.HandlerFunc, 0),
+	}
+}
+
+func (w *GinRoutesWrapper) Group(relativePath string) *GinRoutesWrapper {
+	return &GinRoutesWrapper{
+		wrapper:        w.wrapper,
+		routes:         w.routes.Group(relativePath),
+		defaultHandler: langext.ArrCopy(w.defaultHandler),
+		absPath:        joinPaths(w.absPath, relativePath),
+	}
+}
+
+func (w *GinRoutesWrapper) Use(middleware ...gin.HandlerFunc) *GinRoutesWrapper {
+	defHandler := langext.ArrCopy(w.defaultHandler)
+	defHandler = append(defHandler, middleware...)
+	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+}
+
+func (w *GinRoutesWrapper) GET(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodGet, relativePath)
+}
+
+func (w *GinRoutesWrapper) POST(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodPost, relativePath)
+}
+
+func (w *GinRoutesWrapper) DELETE(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodDelete, relativePath)
+}
+
+func (w *GinRoutesWrapper) PATCH(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodPatch, relativePath)
+}
+
+func (w *GinRoutesWrapper) PUT(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodPut, relativePath)
+}
+
+func (w *GinRoutesWrapper) OPTIONS(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodOptions, relativePath)
+}
+
+func (w *GinRoutesWrapper) HEAD(relativePath string) *GinRouteBuilder {
+	return w._route(http.MethodHead, relativePath)
+}
+
+func (w *GinRoutesWrapper) COUNT(relativePath string) *GinRouteBuilder {
+	return w._route("COUNT", relativePath)
+}
+
+func (w *GinRoutesWrapper) Any(relativePath string) *GinRouteBuilder {
+	return w._route("*", relativePath)
+}
+
+func (w *GinRoutesWrapper) _route(method string, relativePath string) *GinRouteBuilder {
+	return &GinRouteBuilder{
+		routes:   w,
+		method:   method,
+		relPath:  relativePath,
+		absPath:  joinPaths(w.absPath, relativePath),
+		handlers: langext.ArrCopy(w.defaultHandler),
+	}
+}
+
+func (w *GinRouteBuilder) Use(middleware ...gin.HandlerFunc) *GinRouteBuilder {
+	w.handlers = append(w.handlers, middleware...)
+	return w
+}
+
+func (w *GinRouteBuilder) Handle(handler WHandlerFunc) {
+
+	if w.routes.wrapper.bufferBody {
+		arr := make([]gin.HandlerFunc, 0, len(w.handlers)+1)
+		arr = append(arr, BodyBuffer)
+		arr = append(arr, w.handlers...)
+		w.handlers = arr
+	}
+
+	middlewareNames := langext.ArrMap(w.handlers, func(v gin.HandlerFunc) string { return nameOfFunction(v) })
+	handlerName := nameOfFunction(handler)
+
+	w.handlers = append(w.handlers, Wrap(w.routes.wrapper, handler))
+
+	methodName := w.method
+
+	if w.method == "*" {
+		methodName = "ANY"
+		for _, method := range anyMethods {
+			w.routes.routes.Handle(method, w.relPath, w.handlers...)
+		}
+	} else {
+		w.routes.routes.Handle(w.method, w.relPath, w.handlers...)
+	}
+
+	w.routes.wrapper.routeSpecs = append(w.routes.wrapper.routeSpecs, ginRouteSpec{
+		Method:      methodName,
+		URL:         w.absPath,
+		Middlewares: middlewareNames,
+		Handler:     handlerName,
+	})
+}
+
+func (w *GinWrapper) NoRoute(handler WHandlerFunc) {
+
+	handlers := make([]gin.HandlerFunc, 0)
+
+	if w.bufferBody {
+		handlers = append(handlers, BodyBuffer)
+	}
+
+	middlewareNames := langext.ArrMap(handlers, func(v gin.HandlerFunc) string { return nameOfFunction(v) })
+	handlerName := nameOfFunction(handler)
+
+	handlers = append(handlers, Wrap(w, handler))
+
+	w.engine.NoRoute(handlers...)
+
+	w.routeSpecs = append(w.routeSpecs, ginRouteSpec{
+		Method:      "ANY",
+		URL:         "[NO_ROUTE]",
+		Middlewares: middlewareNames,
+		Handler:     handlerName,
+	})
+}
+
+func nameOfFunction(f any) string {
+
+	fname := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+
+	split := strings.Split(fname, "/")
+	if len(split) == 0 {
+		return ""
+	}
+
+	fname = split[len(split)-1]
+
+	// https://stackoverflow.com/a/32925345/1761622
+	if strings.HasSuffix(fname, "-fm") {
+		fname = fname[:len(fname)-len("-fm")]
+	}
+
+	suffix := rext.W(regexp.MustCompile(`\.func[0-9]+(?:\.[0-9]+)*$`))
+
+	if match, ok := suffix.MatchFirst(fname); ok {
+		fname = fname[:len(fname)-match.FullMatch().Length()]
+	}
+
+	return fname
+}
+
+// joinPaths is copied verbatim from gin@v1.9.1/gin.go
+func joinPaths(absolutePath, relativePath string) string {
+	if relativePath == "" {
+		return absolutePath
+	}
+
+	finalPath := path.Join(absolutePath, relativePath)
+	if lastChar(relativePath) == '/' && lastChar(finalPath) != '/' {
+		return finalPath + "/"
+	}
+	return finalPath
+}
+
+func lastChar(str string) uint8 {
+	if str == "" {
+		panic("The length of the string can't be 0")
+	}
+	return str[len(str)-1]
+}

go.mod

@@ -3,6 +3,48 @@ module gogs.mikescher.com/BlackForestBytes/goext
 go 1.19
 
 require (
-	golang.org/x/sys v0.1.0
-	golang.org/x/term v0.1.0
+	github.com/gin-gonic/gin v1.9.1
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/rs/xid v1.5.0
+	github.com/rs/zerolog v1.30.0
+	go.mongodb.org/mongo-driver v1.12.1
+	golang.org/x/crypto v0.12.0
+	golang.org/x/sys v0.11.0
+	golang.org/x/term v0.11.0
+)
+
+require (
+	github.com/bytedance/sonic v1.10.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
+	github.com/chenzhuoyu/iasm v0.9.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.15.1 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.9 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	golang.org/x/arch v0.4.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,4 +1,223 @@
+github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
+github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
+github.com/bytedance/sonic v1.10.0-rc2 h1:oDfRZ+4m6AYCOC0GFeOCeYqvBmucy1isvouS2K0cPzo=
+github.com/bytedance/sonic v1.10.0-rc2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.10.0-rc3 h1:uNSnscRapXTwUgTyOF0GVljYD08p9X/Lbr9MweSV3V0=
+github.com/bytedance/sonic v1.10.0-rc3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk=
+github.com/bytedance/sonic v1.10.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
+github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
+github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
+github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
+github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo=
+github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
+github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=
+github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw=
+github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM=
+github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
+github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
+github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
+github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
+github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
+github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
+github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
+github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
+github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
+github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
+go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
+go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
+go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
+go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.4.0 h1:A8WCeEWhLwPBKNbFi5Wv5UTCBx5zzubnXDlMOFAzFMc=
+golang.org/x/arch v0.4.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
+golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
+golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

goextVersion.go

@@ -0,0 +1,5 @@
+package goext
+
+const GoextVersion = "0.0.247"
+
+const GoextVersionTimestamp = "2023-08-21T14:23:44+0200"

gojson/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

gojson/README.md

@@ -0,0 +1,12 @@
+
+
+JSON serializer which serializes nil-Arrays as `[]` and nil-maps als `{}`.
+
+Idea from: https://github.com/homelight/json
+
+Forked from https://github.com/golang/go/tree/547e8e22fe565d65d1fd4d6e71436a5a855447b0/src/encoding/json ( tag go1.20.2 )
+
+Added:
+
+ - `MarshalSafeCollections()` method
+ - `Encoder.nilSafeSlices` and `Encoder.nilSafeMaps` fields

gojson/example_marshaling_test.go

@@ -0,0 +1,73 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+)
+
+type Animal int
+
+const (
+	Unknown Animal = iota
+	Gopher
+	Zebra
+)
+
+func (a *Animal) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	switch strings.ToLower(s) {
+	default:
+		*a = Unknown
+	case "gopher":
+		*a = Gopher
+	case "zebra":
+		*a = Zebra
+	}
+
+	return nil
+}
+
+func (a Animal) MarshalJSON() ([]byte, error) {
+	var s string
+	switch a {
+	default:
+		s = "unknown"
+	case Gopher:
+		s = "gopher"
+	case Zebra:
+		s = "zebra"
+	}
+
+	return json.Marshal(s)
+}
+
+func Example_customMarshalJSON() {
+	blob := `["gopher","armadillo","zebra","unknown","gopher","bee","gopher","zebra"]`
+	var zoo []Animal
+	if err := json.Unmarshal([]byte(blob), &zoo); err != nil {
+		log.Fatal(err)
+	}
+
+	census := make(map[Animal]int)
+	for _, animal := range zoo {
+		census[animal] += 1
+	}
+
+	fmt.Printf("Zoo Census:\n* Gophers: %d\n* Zebras:  %d\n* Unknown: %d\n",
+		census[Gopher], census[Zebra], census[Unknown])
+
+	// Output:
+	// Zoo Census:
+	// * Gophers: 3
+	// * Zebras:  2
+	// * Unknown: 3
+}

gojson/example_test.go

@@ -0,0 +1,310 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func ExampleMarshal() {
+	type ColorGroup struct {
+		ID     int
+		Name   string
+		Colors []string
+	}
+	group := ColorGroup{
+		ID:     1,
+		Name:   "Reds",
+		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
+	}
+	b, err := json.Marshal(group)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+	// Output:
+	// {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]}
+}
+
+func ExampleUnmarshal() {
+	var jsonBlob = []byte(`[
+	{"Name": "Platypus", "Order": "Monotremata"},
+	{"Name": "Quoll",    "Order": "Dasyuromorphia"}
+]`)
+	type Animal struct {
+		Name  string
+		Order string
+	}
+	var animals []Animal
+	err := json.Unmarshal(jsonBlob, &animals)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	fmt.Printf("%+v", animals)
+	// Output:
+	// [{Name:Platypus Order:Monotremata} {Name:Quoll Order:Dasyuromorphia}]
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder() {
+	const jsonStream = `
+	{"Name": "Ed", "Text": "Knock knock."}
+	{"Name": "Sam", "Text": "Who's there?"}
+	{"Name": "Ed", "Text": "Go fmt."}
+	{"Name": "Sam", "Text": "Go fmt who?"}
+	{"Name": "Ed", "Text": "Go fmt yourself!"}
+`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		var m Message
+		if err := dec.Decode(&m); err == io.EOF {
+			break
+		} else if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%s: %s\n", m.Name, m.Text)
+	}
+	// Output:
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+}
+
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder_Token() {
+	const jsonStream = `
+	{"Message": "Hello", "Array": [1, 2, 3], "Null": null, "Number": 1.234}
+`
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		t, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%T: %v", t, t)
+		if dec.More() {
+			fmt.Printf(" (more)")
+		}
+		fmt.Printf("\n")
+	}
+	// Output:
+	// json.Delim: { (more)
+	// string: Message (more)
+	// string: Hello (more)
+	// string: Array (more)
+	// json.Delim: [ (more)
+	// float64: 1 (more)
+	// float64: 2 (more)
+	// float64: 3
+	// json.Delim: ] (more)
+	// string: Null (more)
+	// <nil>: <nil> (more)
+	// string: Number (more)
+	// float64: 1.234
+	// json.Delim: }
+}
+
+// This example uses a Decoder to decode a streaming array of JSON objects.
+func ExampleDecoder_Decode_stream() {
+	const jsonStream = `
+	[
+		{"Name": "Ed", "Text": "Knock knock."},
+		{"Name": "Sam", "Text": "Who's there?"},
+		{"Name": "Ed", "Text": "Go fmt."},
+		{"Name": "Sam", "Text": "Go fmt who?"},
+		{"Name": "Ed", "Text": "Go fmt yourself!"}
+	]
+`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+
+	// read open bracket
+	t, err := dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// while the array contains values
+	for dec.More() {
+		var m Message
+		// decode an array value (Message)
+		err := dec.Decode(&m)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("%v: %v\n", m.Name, m.Text)
+	}
+
+	// read closing bracket
+	t, err = dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// Output:
+	// json.Delim: [
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+	// json.Delim: ]
+}
+
+// This example uses RawMessage to delay parsing part of a JSON message.
+func ExampleRawMessage_unmarshal() {
+	type Color struct {
+		Space string
+		Point json.RawMessage // delay parsing until we know the color space
+	}
+	type RGB struct {
+		R uint8
+		G uint8
+		B uint8
+	}
+	type YCbCr struct {
+		Y  uint8
+		Cb int8
+		Cr int8
+	}
+
+	var j = []byte(`[
+	{"Space": "YCbCr", "Point": {"Y": 255, "Cb": 0, "Cr": -10}},
+	{"Space": "RGB",   "Point": {"R": 98, "G": 218, "B": 255}}
+]`)
+	var colors []Color
+	err := json.Unmarshal(j, &colors)
+	if err != nil {
+		log.Fatalln("error:", err)
+	}
+
+	for _, c := range colors {
+		var dst any
+		switch c.Space {
+		case "RGB":
+			dst = new(RGB)
+		case "YCbCr":
+			dst = new(YCbCr)
+		}
+		err := json.Unmarshal(c.Point, dst)
+		if err != nil {
+			log.Fatalln("error:", err)
+		}
+		fmt.Println(c.Space, dst)
+	}
+	// Output:
+	// YCbCr &{255 0 -10}
+	// RGB &{98 218 255}
+}
+
+// This example uses RawMessage to use a precomputed JSON during marshal.
+func ExampleRawMessage_marshal() {
+	h := json.RawMessage(`{"precomputed": true}`)
+
+	c := struct {
+		Header *json.RawMessage `json:"header"`
+		Body   string           `json:"body"`
+	}{Header: &h, Body: "Hello Gophers!"}
+
+	b, err := json.MarshalIndent(&c, "", "\t")
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	os.Stdout.Write(b)
+
+	// Output:
+	// {
+	// 	"header": {
+	// 		"precomputed": true
+	// 	},
+	// 	"body": "Hello Gophers!"
+	// }
+}
+
+func ExampleIndent() {
+	type Road struct {
+		Name   string
+		Number int
+	}
+	roads := []Road{
+		{"Diamond Fork", 29},
+		{"Sheep Creek", 51},
+	}
+
+	b, err := json.Marshal(roads)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var out bytes.Buffer
+	json.Indent(&out, b, "=", "\t")
+	out.WriteTo(os.Stdout)
+	// Output:
+	// [
+	// =	{
+	// =		"Name": "Diamond Fork",
+	// =		"Number": 29
+	// =	},
+	// =	{
+	// =		"Name": "Sheep Creek",
+	// =		"Number": 51
+	// =	}
+	// =]
+}
+
+func ExampleMarshalIndent() {
+	data := map[string]int{
+		"a": 1,
+		"b": 2,
+	}
+
+	b, err := json.MarshalIndent(data, "<prefix>", "<indent>")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(b))
+	// Output:
+	// {
+	// <prefix><indent>"a": 1,
+	// <prefix><indent>"b": 2
+	// <prefix>}
+}
+
+func ExampleValid() {
+	goodJSON := `{"example": 1}`
+	badJSON := `{"example":2:]}}`
+
+	fmt.Println(json.Valid([]byte(goodJSON)), json.Valid([]byte(badJSON)))
+	// Output:
+	// true false
+}
+
+func ExampleHTMLEscape() {
+	var out bytes.Buffer
+	json.HTMLEscape(&out, []byte(`{"Name":"<b>HTML content</b>"}`))
+	out.WriteTo(os.Stdout)
+	// Output:
+	//{"Name":"\u003cb\u003eHTML content\u003c/b\u003e"}
+}

gojson/example_text_marshaling_test.go

@@ -0,0 +1,67 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+)
+
+type Size int
+
+const (
+	Unrecognized Size = iota
+	Small
+	Large
+)
+
+func (s *Size) UnmarshalText(text []byte) error {
+	switch strings.ToLower(string(text)) {
+	default:
+		*s = Unrecognized
+	case "small":
+		*s = Small
+	case "large":
+		*s = Large
+	}
+	return nil
+}
+
+func (s Size) MarshalText() ([]byte, error) {
+	var name string
+	switch s {
+	default:
+		name = "unrecognized"
+	case Small:
+		name = "small"
+	case Large:
+		name = "large"
+	}
+	return []byte(name), nil
+}
+
+func Example_textMarshalJSON() {
+	blob := `["small","regular","large","unrecognized","small","normal","small","large"]`
+	var inventory []Size
+	if err := json.Unmarshal([]byte(blob), &inventory); err != nil {
+		log.Fatal(err)
+	}
+
+	counts := make(map[Size]int)
+	for _, size := range inventory {
+		counts[size] += 1
+	}
+
+	fmt.Printf("Inventory Counts:\n* Small:        %d\n* Large:        %d\n* Unrecognized: %d\n",
+		counts[Small], counts[Large], counts[Unrecognized])
+
+	// Output:
+	// Inventory Counts:
+	// * Small:        3
+	// * Large:        2
+	// * Unrecognized: 3
+}

gojson/fold.go

@@ -0,0 +1,141 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+const (
+	caseMask     = ^byte(0x20) // Mask to ignore case in ASCII.
+	kelvin       = '\u212a'
+	smallLongEss = '\u017f'
+)
+
+// foldFunc returns one of four different case folding equivalence
+// functions, from most general (and slow) to fastest:
+//
+// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8
+// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S')
+// 3) asciiEqualFold, no special, but includes non-letters (including _)
+// 4) simpleLetterEqualFold, no specials, no non-letters.
+//
+// The letters S and K are special because they map to 3 runes, not just 2:
+//   - S maps to s and to U+017F 'ſ' Latin small letter long s
+//   - k maps to K and to U+212A 'K' Kelvin sign
+//
+// See https://play.golang.org/p/tTxjOc0OGo
+//
+// The returned function is specialized for matching against s and
+// should only be given s. It's not curried for performance reasons.
+func foldFunc(s []byte) func(s, t []byte) bool {
+	nonLetter := false
+	special := false // special letter
+	for _, b := range s {
+		if b >= utf8.RuneSelf {
+			return bytes.EqualFold
+		}
+		upper := b & caseMask
+		if upper < 'A' || upper > 'Z' {
+			nonLetter = true
+		} else if upper == 'K' || upper == 'S' {
+			// See above for why these letters are special.
+			special = true
+		}
+	}
+	if special {
+		return equalFoldRight
+	}
+	if nonLetter {
+		return asciiEqualFold
+	}
+	return simpleLetterEqualFold
+}
+
+// equalFoldRight is a specialization of bytes.EqualFold when s is
+// known to be all ASCII (including punctuation), but contains an 's',
+// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t.
+// See comments on foldFunc.
+func equalFoldRight(s, t []byte) bool {
+	for _, sb := range s {
+		if len(t) == 0 {
+			return false
+		}
+		tb := t[0]
+		if tb < utf8.RuneSelf {
+			if sb != tb {
+				sbUpper := sb & caseMask
+				if 'A' <= sbUpper && sbUpper <= 'Z' {
+					if sbUpper != tb&caseMask {
+						return false
+					}
+				} else {
+					return false
+				}
+			}
+			t = t[1:]
+			continue
+		}
+		// sb is ASCII and t is not. t must be either kelvin
+		// sign or long s; sb must be s, S, k, or K.
+		tr, size := utf8.DecodeRune(t)
+		switch sb {
+		case 's', 'S':
+			if tr != smallLongEss {
+				return false
+			}
+		case 'k', 'K':
+			if tr != kelvin {
+				return false
+			}
+		default:
+			return false
+		}
+		t = t[size:]
+
+	}
+	return len(t) == 0
+}
+
+// asciiEqualFold is a specialization of bytes.EqualFold for use when
+// s is all ASCII (but may contain non-letters) and contains no
+// special-folding letters.
+// See comments on foldFunc.
+func asciiEqualFold(s, t []byte) bool {
+	if len(s) != len(t) {
+		return false
+	}
+	for i, sb := range s {
+		tb := t[i]
+		if sb == tb {
+			continue
+		}
+		if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') {
+			if sb&caseMask != tb&caseMask {
+				return false
+			}
+		} else {
+			return false
+		}
+	}
+	return true
+}
+
+// simpleLetterEqualFold is a specialization of bytes.EqualFold for
+// use when s is all ASCII letters (no underscores, etc) and also
+// doesn't contain 'k', 'K', 's', or 'S'.
+// See comments on foldFunc.
+func simpleLetterEqualFold(s, t []byte) bool {
+	if len(s) != len(t) {
+		return false
+	}
+	for i, b := range s {
+		if b&caseMask != t[i]&caseMask {
+			return false
+		}
+	}
+	return true
+}

gojson/fold_test.go

@@ -0,0 +1,110 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+var foldTests = []struct {
+	fn   func(s, t []byte) bool
+	s, t string
+	want bool
+}{
+	{equalFoldRight, "", "", true},
+	{equalFoldRight, "a", "a", true},
+	{equalFoldRight, "", "a", false},
+	{equalFoldRight, "a", "", false},
+	{equalFoldRight, "a", "A", true},
+	{equalFoldRight, "AB", "ab", true},
+	{equalFoldRight, "AB", "ac", false},
+	{equalFoldRight, "sbkKc", "ſbKKc", true},
+	{equalFoldRight, "SbKkc", "ſbKKc", true},
+	{equalFoldRight, "SbKkc", "ſbKK", false},
+	{equalFoldRight, "e", "é", false},
+	{equalFoldRight, "s", "S", true},
+
+	{simpleLetterEqualFold, "", "", true},
+	{simpleLetterEqualFold, "abc", "abc", true},
+	{simpleLetterEqualFold, "abc", "ABC", true},
+	{simpleLetterEqualFold, "abc", "ABCD", false},
+	{simpleLetterEqualFold, "abc", "xxx", false},
+
+	{asciiEqualFold, "a_B", "A_b", true},
+	{asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent
+}
+
+func TestFold(t *testing.T) {
+	for i, tt := range foldTests {
+		if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want {
+			t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want)
+		}
+		truth := strings.EqualFold(tt.s, tt.t)
+		if truth != tt.want {
+			t.Errorf("strings.EqualFold doesn't agree with case %d", i)
+		}
+	}
+}
+
+func TestFoldAgainstUnicode(t *testing.T) {
+	var buf1, buf2 []byte
+	var runes []rune
+	for i := 0x20; i <= 0x7f; i++ {
+		runes = append(runes, rune(i))
+	}
+	runes = append(runes, kelvin, smallLongEss)
+
+	funcs := []struct {
+		name   string
+		fold   func(s, t []byte) bool
+		letter bool // must be ASCII letter
+		simple bool // must be simple ASCII letter (not 'S' or 'K')
+	}{
+		{
+			name: "equalFoldRight",
+			fold: equalFoldRight,
+		},
+		{
+			name:   "asciiEqualFold",
+			fold:   asciiEqualFold,
+			simple: true,
+		},
+		{
+			name:   "simpleLetterEqualFold",
+			fold:   simpleLetterEqualFold,
+			simple: true,
+			letter: true,
+		},
+	}
+
+	for _, ff := range funcs {
+		for _, r := range runes {
+			if r >= utf8.RuneSelf {
+				continue
+			}
+			if ff.letter && !isASCIILetter(byte(r)) {
+				continue
+			}
+			if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') {
+				continue
+			}
+			for _, r2 := range runes {
+				buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x')
+				buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x')
+				want := bytes.EqualFold(buf1, buf2)
+				if got := ff.fold(buf1, buf2); got != want {
+					t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)
+				}
+			}
+		}
+	}
+}
+
+func isASCIILetter(b byte) bool {
+	return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z')
+}

gojson/fuzz.go

@@ -0,0 +1,42 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build gofuzz
+
+package json
+
+import (
+	"fmt"
+)
+
+func Fuzz(data []byte) (score int) {
+	for _, ctor := range []func() any{
+		func() any { return new(any) },
+		func() any { return new(map[string]any) },
+		func() any { return new([]any) },
+	} {
+		v := ctor()
+		err := Unmarshal(data, v)
+		if err != nil {
+			continue
+		}
+		score = 1
+
+		m, err := Marshal(v)
+		if err != nil {
+			fmt.Printf("v=%#v\n", v)
+			panic(err)
+		}
+
+		u := ctor()
+		err = Unmarshal(m, u)
+		if err != nil {
+			fmt.Printf("v=%#v\n", v)
+			fmt.Printf("m=%s\n", m)
+			panic(err)
+		}
+	}
+
+	return
+}

gojson/fuzz_test.go

@@ -0,0 +1,83 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func FuzzUnmarshalJSON(f *testing.F) {
+	f.Add([]byte(`{
+"object": {
+	"slice": [
+		1,
+		2.0,
+		"3",
+		[4],
+		{5: {}}
+	]
+},
+"slice": [[]],
+"string": ":)",
+"int": 1e5,
+"float": 3e-9"
+}`))
+
+	f.Fuzz(func(t *testing.T, b []byte) {
+		for _, typ := range []func() interface{}{
+			func() interface{} { return new(interface{}) },
+			func() interface{} { return new(map[string]interface{}) },
+			func() interface{} { return new([]interface{}) },
+		} {
+			i := typ()
+			if err := Unmarshal(b, i); err != nil {
+				return
+			}
+
+			encoded, err := Marshal(i)
+			if err != nil {
+				t.Fatalf("failed to marshal: %s", err)
+			}
+
+			if err := Unmarshal(encoded, i); err != nil {
+				t.Fatalf("failed to roundtrip: %s", err)
+			}
+		}
+	})
+}
+
+func FuzzDecoderToken(f *testing.F) {
+	f.Add([]byte(`{
+"object": {
+	"slice": [
+		1,
+		2.0,
+		"3",
+		[4],
+		{5: {}}
+	]
+},
+"slice": [[]],
+"string": ":)",
+"int": 1e5,
+"float": 3e-9"
+}`))
+
+	f.Fuzz(func(t *testing.T, b []byte) {
+		r := bytes.NewReader(b)
+		d := NewDecoder(r)
+		for {
+			_, err := d.Token()
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return
+			}
+		}
+	})
+}

gojson/gionic.go

@@ -0,0 +1,44 @@
+package json
+
+import (
+	"net/http"
+)
+
+// Render interface is copied from github.com/gin-gonic/gin@v1.8.1/render/render.go
+type Render interface {
+	// Render writes data with custom ContentType.
+	Render(http.ResponseWriter) error
+	// WriteContentType writes custom ContentType.
+	WriteContentType(w http.ResponseWriter)
+}
+
+type GoJsonRender struct {
+	Data          any
+	NilSafeSlices bool
+	NilSafeMaps   bool
+	Indent        *IndentOpt
+}
+
+func (r GoJsonRender) Render(w http.ResponseWriter) error {
+	header := w.Header()
+	if val := header["Content-Type"]; len(val) == 0 {
+		header["Content-Type"] = []string{"application/json; charset=utf-8"}
+	}
+
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent)
+	if err != nil {
+		panic(err)
+	}
+	_, err = w.Write(jsonBytes)
+	if err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+func (r GoJsonRender) WriteContentType(w http.ResponseWriter) {
+	header := w.Header()
+	if val := header["Content-Type"]; len(val) == 0 {
+		header["Content-Type"] = []string{"application/json; charset=utf-8"}
+	}
+}

gojson/indent.go

@@ -0,0 +1,143 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+)
+
+// Compact appends to dst the JSON-encoded src with
+// insignificant space characters elided.
+func Compact(dst *bytes.Buffer, src []byte) error {
+	return compact(dst, src, false)
+}
+
+func compact(dst *bytes.Buffer, src []byte, escape bool) error {
+	origLen := dst.Len()
+	scan := newScanner()
+	defer freeScanner(scan)
+	start := 0
+	for i, c := range src {
+		if escape && (c == '<' || c == '>' || c == '&') {
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			dst.WriteString(`\u00`)
+			dst.WriteByte(hex[c>>4])
+			dst.WriteByte(hex[c&0xF])
+			start = i + 1
+		}
+		// Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9).
+		if escape && c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 {
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			dst.WriteString(`\u202`)
+			dst.WriteByte(hex[src[i+2]&0xF])
+			start = i + 3
+		}
+		v := scan.step(scan, c)
+		if v >= scanSkipSpace {
+			if v == scanError {
+				break
+			}
+			if start < i {
+				dst.Write(src[start:i])
+			}
+			start = i + 1
+		}
+	}
+	if scan.eof() == scanError {
+		dst.Truncate(origLen)
+		return scan.err
+	}
+	if start < len(src) {
+		dst.Write(src[start:])
+	}
+	return nil
+}
+
+func newline(dst *bytes.Buffer, prefix, indent string, depth int) {
+	dst.WriteByte('\n')
+	dst.WriteString(prefix)
+	for i := 0; i < depth; i++ {
+		dst.WriteString(indent)
+	}
+}
+
+// Indent appends to dst an indented form of the JSON-encoded src.
+// Each element in a JSON object or array begins on a new,
+// indented line beginning with prefix followed by one or more
+// copies of indent according to the indentation nesting.
+// The data appended to dst does not begin with the prefix nor
+// any indentation, to make it easier to embed inside other formatted JSON data.
+// Although leading space characters (space, tab, carriage return, newline)
+// at the beginning of src are dropped, trailing space characters
+// at the end of src are preserved and copied to dst.
+// For example, if src has no trailing spaces, neither will dst;
+// if src ends in a trailing newline, so will dst.
+func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
+	origLen := dst.Len()
+	scan := newScanner()
+	defer freeScanner(scan)
+	needIndent := false
+	depth := 0
+	for _, c := range src {
+		scan.bytes++
+		v := scan.step(scan, c)
+		if v == scanSkipSpace {
+			continue
+		}
+		if v == scanError {
+			break
+		}
+		if needIndent && v != scanEndObject && v != scanEndArray {
+			needIndent = false
+			depth++
+			newline(dst, prefix, indent, depth)
+		}
+
+		// Emit semantically uninteresting bytes
+		// (in particular, punctuation in strings) unmodified.
+		if v == scanContinue {
+			dst.WriteByte(c)
+			continue
+		}
+
+		// Add spacing around real punctuation.
+		switch c {
+		case '{', '[':
+			// delay indent so that empty object and array are formatted as {} and [].
+			needIndent = true
+			dst.WriteByte(c)
+
+		case ',':
+			dst.WriteByte(c)
+			newline(dst, prefix, indent, depth)
+
+		case ':':
+			dst.WriteByte(c)
+			dst.WriteByte(' ')
+
+		case '}', ']':
+			if needIndent {
+				// suppress indent in empty object/array
+				needIndent = false
+			} else {
+				depth--
+				newline(dst, prefix, indent, depth)
+			}
+			dst.WriteByte(c)
+
+		default:
+			dst.WriteByte(c)
+		}
+	}
+	if scan.eof() == scanError {
+		dst.Truncate(origLen)
+		return scan.err
+	}
+	return nil
+}

gojson/number_test.go

@@ -0,0 +1,118 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"regexp"
+	"testing"
+)
+
+func TestNumberIsValid(t *testing.T) {
+	// From: https://stackoverflow.com/a/13340826
+	var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
+
+	validTests := []string{
+		"0",
+		"-0",
+		"1",
+		"-1",
+		"0.1",
+		"-0.1",
+		"1234",
+		"-1234",
+		"12.34",
+		"-12.34",
+		"12E0",
+		"12E1",
+		"12e34",
+		"12E-0",
+		"12e+1",
+		"12e-34",
+		"-12E0",
+		"-12E1",
+		"-12e34",
+		"-12E-0",
+		"-12e+1",
+		"-12e-34",
+		"1.2E0",
+		"1.2E1",
+		"1.2e34",
+		"1.2E-0",
+		"1.2e+1",
+		"1.2e-34",
+		"-1.2E0",
+		"-1.2E1",
+		"-1.2e34",
+		"-1.2E-0",
+		"-1.2e+1",
+		"-1.2e-34",
+		"0E0",
+		"0E1",
+		"0e34",
+		"0E-0",
+		"0e+1",
+		"0e-34",
+		"-0E0",
+		"-0E1",
+		"-0e34",
+		"-0E-0",
+		"-0e+1",
+		"-0e-34",
+	}
+
+	for _, test := range validTests {
+		if !isValidNumber(test) {
+			t.Errorf("%s should be valid", test)
+		}
+
+		var f float64
+		if err := Unmarshal([]byte(test), &f); err != nil {
+			t.Errorf("%s should be valid but Unmarshal failed: %v", test, err)
+		}
+
+		if !jsonNumberRegexp.MatchString(test) {
+			t.Errorf("%s should be valid but regexp does not match", test)
+		}
+	}
+
+	invalidTests := []string{
+		"",
+		"invalid",
+		"1.0.1",
+		"1..1",
+		"-1-2",
+		"012a42",
+		"01.2",
+		"012",
+		"12E12.12",
+		"1e2e3",
+		"1e+-2",
+		"1e--23",
+		"1e",
+		"e1",
+		"1e+",
+		"1ea",
+		"1a",
+		"1.a",
+		"1.",
+		"01",
+		"1.e1",
+	}
+
+	for _, test := range invalidTests {
+		if isValidNumber(test) {
+			t.Errorf("%s should be invalid", test)
+		}
+
+		var f float64
+		if err := Unmarshal([]byte(test), &f); err == nil {
+			t.Errorf("%s should be invalid but unmarshal wrote %v", test, f)
+		}
+
+		if jsonNumberRegexp.MatchString(test) {
+			t.Errorf("%s should be invalid but matches regexp", test)
+		}
+	}
+}

gojson/scanner.go

@@ -0,0 +1,610 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+// JSON value parser state machine.
+// Just about at the limit of what is reasonable to write by hand.
+// Some parts are a bit tedious, but overall it nicely factors out the
+// otherwise common code from the multiple scanning functions
+// in this package (Compact, Indent, checkValid, etc).
+//
+// This file starts with two simple examples using the scanner
+// before diving into the scanner itself.
+
+import (
+	"strconv"
+	"sync"
+)
+
+// Valid reports whether data is a valid JSON encoding.
+func Valid(data []byte) bool {
+	scan := newScanner()
+	defer freeScanner(scan)
+	return checkValid(data, scan) == nil
+}
+
+// checkValid verifies that data is valid JSON-encoded data.
+// scan is passed in for use by checkValid to avoid an allocation.
+// checkValid returns nil or a SyntaxError.
+func checkValid(data []byte, scan *scanner) error {
+	scan.reset()
+	for _, c := range data {
+		scan.bytes++
+		if scan.step(scan, c) == scanError {
+			return scan.err
+		}
+	}
+	if scan.eof() == scanError {
+		return scan.err
+	}
+	return nil
+}
+
+// A SyntaxError is a description of a JSON syntax error.
+// Unmarshal will return a SyntaxError if the JSON can't be parsed.
+type SyntaxError struct {
+	msg    string // description of error
+	Offset int64  // error occurred after reading Offset bytes
+}
+
+func (e *SyntaxError) Error() string { return e.msg }
+
+// A scanner is a JSON scanning state machine.
+// Callers call scan.reset and then pass bytes in one at a time
+// by calling scan.step(&scan, c) for each byte.
+// The return value, referred to as an opcode, tells the
+// caller about significant parsing events like beginning
+// and ending literals, objects, and arrays, so that the
+// caller can follow along if it wishes.
+// The return value scanEnd indicates that a single top-level
+// JSON value has been completed, *before* the byte that
+// just got passed in.  (The indication must be delayed in order
+// to recognize the end of numbers: is 123 a whole value or
+// the beginning of 12345e+6?).
+type scanner struct {
+	// The step is a func to be called to execute the next transition.
+	// Also tried using an integer constant and a single func
+	// with a switch, but using the func directly was 10% faster
+	// on a 64-bit Mac Mini, and it's nicer to read.
+	step func(*scanner, byte) int
+
+	// Reached end of top-level value.
+	endTop bool
+
+	// Stack of what we're in the middle of - array values, object keys, object values.
+	parseState []int
+
+	// Error that happened, if any.
+	err error
+
+	// total bytes consumed, updated by decoder.Decode (and deliberately
+	// not set to zero by scan.reset)
+	bytes int64
+}
+
+var scannerPool = sync.Pool{
+	New: func() any {
+		return &scanner{}
+	},
+}
+
+func newScanner() *scanner {
+	scan := scannerPool.Get().(*scanner)
+	// scan.reset by design doesn't set bytes to zero
+	scan.bytes = 0
+	scan.reset()
+	return scan
+}
+
+func freeScanner(scan *scanner) {
+	// Avoid hanging on to too much memory in extreme cases.
+	if len(scan.parseState) > 1024 {
+		scan.parseState = nil
+	}
+	scannerPool.Put(scan)
+}
+
+// These values are returned by the state transition functions
+// assigned to scanner.state and the method scanner.eof.
+// They give details about the current state of the scan that
+// callers might be interested to know about.
+// It is okay to ignore the return value of any particular
+// call to scanner.state: if one call returns scanError,
+// every subsequent call will return scanError too.
+const (
+	// Continue.
+	scanContinue     = iota // uninteresting byte
+	scanBeginLiteral        // end implied by next result != scanContinue
+	scanBeginObject         // begin object
+	scanObjectKey           // just finished object key (string)
+	scanObjectValue         // just finished non-last object value
+	scanEndObject           // end object (implies scanObjectValue if possible)
+	scanBeginArray          // begin array
+	scanArrayValue          // just finished array value
+	scanEndArray            // end array (implies scanArrayValue if possible)
+	scanSkipSpace           // space byte; can skip; known to be last "continue" result
+
+	// Stop.
+	scanEnd   // top-level value ended *before* this byte; known to be first "stop" result
+	scanError // hit an error, scanner.err.
+)
+
+// These values are stored in the parseState stack.
+// They give the current state of a composite value
+// being scanned. If the parser is inside a nested value
+// the parseState describes the nested state, outermost at entry 0.
+const (
+	parseObjectKey   = iota // parsing object key (before colon)
+	parseObjectValue        // parsing object value (after colon)
+	parseArrayValue         // parsing array value
+)
+
+// This limits the max nesting depth to prevent stack overflow.
+// This is permitted by https://tools.ietf.org/html/rfc7159#section-9
+const maxNestingDepth = 10000
+
+// reset prepares the scanner for use.
+// It must be called before calling s.step.
+func (s *scanner) reset() {
+	s.step = stateBeginValue
+	s.parseState = s.parseState[0:0]
+	s.err = nil
+	s.endTop = false
+}
+
+// eof tells the scanner that the end of input has been reached.
+// It returns a scan status just as s.step does.
+func (s *scanner) eof() int {
+	if s.err != nil {
+		return scanError
+	}
+	if s.endTop {
+		return scanEnd
+	}
+	s.step(s, ' ')
+	if s.endTop {
+		return scanEnd
+	}
+	if s.err == nil {
+		s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
+	}
+	return scanError
+}
+
+// pushParseState pushes a new parse state p onto the parse stack.
+// an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned.
+func (s *scanner) pushParseState(c byte, newParseState int, successState int) int {
+	s.parseState = append(s.parseState, newParseState)
+	if len(s.parseState) <= maxNestingDepth {
+		return successState
+	}
+	return s.error(c, "exceeded max depth")
+}
+
+// popParseState pops a parse state (already obtained) off the stack
+// and updates s.step accordingly.
+func (s *scanner) popParseState() {
+	n := len(s.parseState) - 1
+	s.parseState = s.parseState[0:n]
+	if n == 0 {
+		s.step = stateEndTop
+		s.endTop = true
+	} else {
+		s.step = stateEndValue
+	}
+}
+
+func isSpace(c byte) bool {
+	return c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n')
+}
+
+// stateBeginValueOrEmpty is the state after reading `[`.
+func stateBeginValueOrEmpty(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == ']' {
+		return stateEndValue(s, c)
+	}
+	return stateBeginValue(s, c)
+}
+
+// stateBeginValue is the state at the beginning of the input.
+func stateBeginValue(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	switch c {
+	case '{':
+		s.step = stateBeginStringOrEmpty
+		return s.pushParseState(c, parseObjectKey, scanBeginObject)
+	case '[':
+		s.step = stateBeginValueOrEmpty
+		return s.pushParseState(c, parseArrayValue, scanBeginArray)
+	case '"':
+		s.step = stateInString
+		return scanBeginLiteral
+	case '-':
+		s.step = stateNeg
+		return scanBeginLiteral
+	case '0': // beginning of 0.123
+		s.step = state0
+		return scanBeginLiteral
+	case 't': // beginning of true
+		s.step = stateT
+		return scanBeginLiteral
+	case 'f': // beginning of false
+		s.step = stateF
+		return scanBeginLiteral
+	case 'n': // beginning of null
+		s.step = stateN
+		return scanBeginLiteral
+	}
+	if '1' <= c && c <= '9' { // beginning of 1234.5
+		s.step = state1
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of value")
+}
+
+// stateBeginStringOrEmpty is the state after reading `{`.
+func stateBeginStringOrEmpty(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '}' {
+		n := len(s.parseState)
+		s.parseState[n-1] = parseObjectValue
+		return stateEndValue(s, c)
+	}
+	return stateBeginString(s, c)
+}
+
+// stateBeginString is the state after reading `{"key": value,`.
+func stateBeginString(s *scanner, c byte) int {
+	if isSpace(c) {
+		return scanSkipSpace
+	}
+	if c == '"' {
+		s.step = stateInString
+		return scanBeginLiteral
+	}
+	return s.error(c, "looking for beginning of object key string")
+}
+
+// stateEndValue is the state after completing a value,
+// such as after reading `{}` or `true` or `["x"`.
+func stateEndValue(s *scanner, c byte) int {
+	n := len(s.parseState)
+	if n == 0 {
+		// Completed top-level before the current byte.
+		s.step = stateEndTop
+		s.endTop = true
+		return stateEndTop(s, c)
+	}
+	if isSpace(c) {
+		s.step = stateEndValue
+		return scanSkipSpace
+	}
+	ps := s.parseState[n-1]
+	switch ps {
+	case parseObjectKey:
+		if c == ':' {
+			s.parseState[n-1] = parseObjectValue
+			s.step = stateBeginValue
+			return scanObjectKey
+		}
+		return s.error(c, "after object key")
+	case parseObjectValue:
+		if c == ',' {
+			s.parseState[n-1] = parseObjectKey
+			s.step = stateBeginString
+			return scanObjectValue
+		}
+		if c == '}' {
+			s.popParseState()
+			return scanEndObject
+		}
+		return s.error(c, "after object key:value pair")
+	case parseArrayValue:
+		if c == ',' {
+			s.step = stateBeginValue
+			return scanArrayValue
+		}
+		if c == ']' {
+			s.popParseState()
+			return scanEndArray
+		}
+		return s.error(c, "after array element")
+	}
+	return s.error(c, "")
+}
+
+// stateEndTop is the state after finishing the top-level value,
+// such as after reading `{}` or `[1,2,3]`.
+// Only space characters should be seen now.
+func stateEndTop(s *scanner, c byte) int {
+	if !isSpace(c) {
+		// Complain about non-space byte on next call.
+		s.error(c, "after top-level value")
+	}
+	return scanEnd
+}
+
+// stateInString is the state after reading `"`.
+func stateInString(s *scanner, c byte) int {
+	if c == '"' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	if c == '\\' {
+		s.step = stateInStringEsc
+		return scanContinue
+	}
+	if c < 0x20 {
+		return s.error(c, "in string literal")
+	}
+	return scanContinue
+}
+
+// stateInStringEsc is the state after reading `"\` during a quoted string.
+func stateInStringEsc(s *scanner, c byte) int {
+	switch c {
+	case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
+		s.step = stateInString
+		return scanContinue
+	case 'u':
+		s.step = stateInStringEscU
+		return scanContinue
+	}
+	return s.error(c, "in string escape code")
+}
+
+// stateInStringEscU is the state after reading `"\u` during a quoted string.
+func stateInStringEscU(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU1
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
+func stateInStringEscU1(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU12
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
+func stateInStringEscU12(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInStringEscU123
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
+func stateInStringEscU123(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
+		s.step = stateInString
+		return scanContinue
+	}
+	// numbers
+	return s.error(c, "in \\u hexadecimal character escape")
+}
+
+// stateNeg is the state after reading `-` during a number.
+func stateNeg(s *scanner, c byte) int {
+	if c == '0' {
+		s.step = state0
+		return scanContinue
+	}
+	if '1' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return s.error(c, "in numeric literal")
+}
+
+// state1 is the state after reading a non-zero integer during a number,
+// such as after reading `1` or `100` but not `0`.
+func state1(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = state1
+		return scanContinue
+	}
+	return state0(s, c)
+}
+
+// state0 is the state after reading `0` during a number.
+func state0(s *scanner, c byte) int {
+	if c == '.' {
+		s.step = stateDot
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateDot is the state after reading the integer and decimal point in a number,
+// such as after reading `1.`.
+func stateDot(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateDot0
+		return scanContinue
+	}
+	return s.error(c, "after decimal point in numeric literal")
+}
+
+// stateDot0 is the state after reading the integer, decimal point, and subsequent
+// digits of a number, such as after reading `3.14`.
+func stateDot0(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	if c == 'e' || c == 'E' {
+		s.step = stateE
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateE is the state after reading the mantissa and e in a number,
+// such as after reading `314e` or `0.314e`.
+func stateE(s *scanner, c byte) int {
+	if c == '+' || c == '-' {
+		s.step = stateESign
+		return scanContinue
+	}
+	return stateESign(s, c)
+}
+
+// stateESign is the state after reading the mantissa, e, and sign in a number,
+// such as after reading `314e-` or `0.314e+`.
+func stateESign(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		s.step = stateE0
+		return scanContinue
+	}
+	return s.error(c, "in exponent of numeric literal")
+}
+
+// stateE0 is the state after reading the mantissa, e, optional sign,
+// and at least one digit of the exponent in a number,
+// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
+func stateE0(s *scanner, c byte) int {
+	if '0' <= c && c <= '9' {
+		return scanContinue
+	}
+	return stateEndValue(s, c)
+}
+
+// stateT is the state after reading `t`.
+func stateT(s *scanner, c byte) int {
+	if c == 'r' {
+		s.step = stateTr
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'r')")
+}
+
+// stateTr is the state after reading `tr`.
+func stateTr(s *scanner, c byte) int {
+	if c == 'u' {
+		s.step = stateTru
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'u')")
+}
+
+// stateTru is the state after reading `tru`.
+func stateTru(s *scanner, c byte) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal true (expecting 'e')")
+}
+
+// stateF is the state after reading `f`.
+func stateF(s *scanner, c byte) int {
+	if c == 'a' {
+		s.step = stateFa
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'a')")
+}
+
+// stateFa is the state after reading `fa`.
+func stateFa(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateFal
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'l')")
+}
+
+// stateFal is the state after reading `fal`.
+func stateFal(s *scanner, c byte) int {
+	if c == 's' {
+		s.step = stateFals
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 's')")
+}
+
+// stateFals is the state after reading `fals`.
+func stateFals(s *scanner, c byte) int {
+	if c == 'e' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal false (expecting 'e')")
+}
+
+// stateN is the state after reading `n`.
+func stateN(s *scanner, c byte) int {
+	if c == 'u' {
+		s.step = stateNu
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'u')")
+}
+
+// stateNu is the state after reading `nu`.
+func stateNu(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateNul
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateNul is the state after reading `nul`.
+func stateNul(s *scanner, c byte) int {
+	if c == 'l' {
+		s.step = stateEndValue
+		return scanContinue
+	}
+	return s.error(c, "in literal null (expecting 'l')")
+}
+
+// stateError is the state after reaching a syntax error,
+// such as after reading `[1}` or `5.1.2`.
+func stateError(s *scanner, c byte) int {
+	return scanError
+}
+
+// error records an error and switches to the error state.
+func (s *scanner) error(c byte, context string) int {
+	s.step = stateError
+	s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
+	return scanError
+}
+
+// quoteChar formats c as a quoted character literal.
+func quoteChar(c byte) string {
+	// special cases - different from quoted strings
+	if c == '\'' {
+		return `'\''`
+	}
+	if c == '"' {
+		return `'"'`
+	}
+
+	// use quoted string with different quotation marks
+	s := strconv.Quote(string(c))
+	return "'" + s[1:len(s)-1] + "'"
+}

gojson/scanner_test.go

@@ -0,0 +1,301 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"math"
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+var validTests = []struct {
+	data string
+	ok   bool
+}{
+	{`foo`, false},
+	{`}{`, false},
+	{`{]`, false},
+	{`{}`, true},
+	{`{"foo":"bar"}`, true},
+	{`{"foo":"bar","bar":{"baz":["qux"]}}`, true},
+}
+
+func TestValid(t *testing.T) {
+	for _, tt := range validTests {
+		if ok := Valid([]byte(tt.data)); ok != tt.ok {
+			t.Errorf("Valid(%#q) = %v, want %v", tt.data, ok, tt.ok)
+		}
+	}
+}
+
+// Tests of simple examples.
+
+type example struct {
+	compact string
+	indent  string
+}
+
+var examples = []example{
+	{`1`, `1`},
+	{`{}`, `{}`},
+	{`[]`, `[]`},
+	{`{"":2}`, "{\n\t\"\": 2\n}"},
+	{`[3]`, "[\n\t3\n]"},
+	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
+	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
+	{ex1, ex1i},
+	{"{\"\":\"<>&\u2028\u2029\"}", "{\n\t\"\": \"<>&\u2028\u2029\"\n}"}, // See golang.org/issue/34070
+}
+
+var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`
+
+var ex1i = `[
+	true,
+	false,
+	null,
+	"x",
+	1,
+	1.5,
+	0,
+	-5e+2
+]`
+
+func TestCompact(t *testing.T) {
+	var buf bytes.Buffer
+	for _, tt := range examples {
+		buf.Reset()
+		if err := Compact(&buf, []byte(tt.compact)); err != nil {
+			t.Errorf("Compact(%#q): %v", tt.compact, err)
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
+		}
+
+		buf.Reset()
+		if err := Compact(&buf, []byte(tt.indent)); err != nil {
+			t.Errorf("Compact(%#q): %v", tt.indent, err)
+			continue
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
+		}
+	}
+}
+
+func TestCompactSeparators(t *testing.T) {
+	// U+2028 and U+2029 should be escaped inside strings.
+	// They should not appear outside strings.
+	tests := []struct {
+		in, compact string
+	}{
+		{"{\"\u2028\": 1}", "{\"\u2028\":1}"},
+		{"{\"\u2029\" :2}", "{\"\u2029\":2}"},
+	}
+	for _, tt := range tests {
+		var buf bytes.Buffer
+		if err := Compact(&buf, []byte(tt.in)); err != nil {
+			t.Errorf("Compact(%q): %v", tt.in, err)
+		} else if s := buf.String(); s != tt.compact {
+			t.Errorf("Compact(%q) = %q, want %q", tt.in, s, tt.compact)
+		}
+	}
+}
+
+func TestIndent(t *testing.T) {
+	var buf bytes.Buffer
+	for _, tt := range examples {
+		buf.Reset()
+		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
+			t.Errorf("Indent(%#q): %v", tt.indent, err)
+		} else if s := buf.String(); s != tt.indent {
+			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
+		}
+
+		buf.Reset()
+		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
+			t.Errorf("Indent(%#q): %v", tt.compact, err)
+			continue
+		} else if s := buf.String(); s != tt.indent {
+			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
+		}
+	}
+}
+
+// Tests of a large random structure.
+
+func TestCompactBig(t *testing.T) {
+	initBig()
+	var buf bytes.Buffer
+	if err := Compact(&buf, jsonBig); err != nil {
+		t.Fatalf("Compact: %v", err)
+	}
+	b := buf.Bytes()
+	if !bytes.Equal(b, jsonBig) {
+		t.Error("Compact(jsonBig) != jsonBig")
+		diff(t, b, jsonBig)
+		return
+	}
+}
+
+func TestIndentBig(t *testing.T) {
+	t.Parallel()
+	initBig()
+	var buf bytes.Buffer
+	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
+		t.Fatalf("Indent1: %v", err)
+	}
+	b := buf.Bytes()
+	if len(b) == len(jsonBig) {
+		// jsonBig is compact (no unnecessary spaces);
+		// indenting should make it bigger
+		t.Fatalf("Indent(jsonBig) did not get bigger")
+	}
+
+	// should be idempotent
+	var buf1 bytes.Buffer
+	if err := Indent(&buf1, b, "", "\t"); err != nil {
+		t.Fatalf("Indent2: %v", err)
+	}
+	b1 := buf1.Bytes()
+	if !bytes.Equal(b1, b) {
+		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
+		diff(t, b1, b)
+		return
+	}
+
+	// should get back to original
+	buf1.Reset()
+	if err := Compact(&buf1, b); err != nil {
+		t.Fatalf("Compact: %v", err)
+	}
+	b1 = buf1.Bytes()
+	if !bytes.Equal(b1, jsonBig) {
+		t.Error("Compact(Indent(jsonBig)) != jsonBig")
+		diff(t, b1, jsonBig)
+		return
+	}
+}
+
+type indentErrorTest struct {
+	in  string
+	err error
+}
+
+var indentErrorTests = []indentErrorTest{
+	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
+	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
+}
+
+func TestIndentErrors(t *testing.T) {
+	for i, tt := range indentErrorTests {
+		slice := make([]uint8, 0)
+		buf := bytes.NewBuffer(slice)
+		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
+			if !reflect.DeepEqual(err, tt.err) {
+				t.Errorf("#%d: Indent: %#v", i, err)
+				continue
+			}
+		}
+	}
+}
+
+func diff(t *testing.T, a, b []byte) {
+	for i := 0; ; i++ {
+		if i >= len(a) || i >= len(b) || a[i] != b[i] {
+			j := i - 10
+			if j < 0 {
+				j = 0
+			}
+			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
+			return
+		}
+	}
+}
+
+func trim(b []byte) []byte {
+	if len(b) > 20 {
+		return b[0:20]
+	}
+	return b
+}
+
+// Generate a random JSON object.
+
+var jsonBig []byte
+
+func initBig() {
+	n := 10000
+	if testing.Short() {
+		n = 100
+	}
+	b, err := Marshal(genValue(n))
+	if err != nil {
+		panic(err)
+	}
+	jsonBig = b
+}
+
+func genValue(n int) any {
+	if n > 1 {
+		switch rand.Intn(2) {
+		case 0:
+			return genArray(n)
+		case 1:
+			return genMap(n)
+		}
+	}
+	switch rand.Intn(3) {
+	case 0:
+		return rand.Intn(2) == 0
+	case 1:
+		return rand.NormFloat64()
+	case 2:
+		return genString(30)
+	}
+	panic("unreachable")
+}
+
+func genString(stddev float64) string {
+	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
+	c := make([]rune, n)
+	for i := range c {
+		f := math.Abs(rand.NormFloat64()*64 + 32)
+		if f > 0x10ffff {
+			f = 0x10ffff
+		}
+		c[i] = rune(f)
+	}
+	return string(c)
+}
+
+func genArray(n int) []any {
+	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
+	if f > n {
+		f = n
+	}
+	if f < 1 {
+		f = 1
+	}
+	x := make([]any, f)
+	for i := range x {
+		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
+	}
+	return x
+}
+
+func genMap(n int) map[string]any {
+	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
+	if f > n {
+		f = n
+	}
+	if n > 0 && f == 0 {
+		f = 1
+	}
+	x := make(map[string]any)
+	for i := 0; i < f; i++ {
+		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
+	}
+	return x
+}

gojson/stream.go

@@ -0,0 +1,524 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+// A Decoder reads and decodes JSON values from an input stream.
+type Decoder struct {
+	r       io.Reader
+	buf     []byte
+	d       decodeState
+	scanp   int   // start of unread data in buf
+	scanned int64 // amount of data already scanned
+	scan    scanner
+	err     error
+
+	tokenState int
+	tokenStack []int
+}
+
+// NewDecoder returns a new decoder that reads from r.
+//
+// The decoder introduces its own buffering and may
+// read data from r beyond the JSON values requested.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{r: r}
+}
+
+// UseNumber causes the Decoder to unmarshal a number into an interface{} as a
+// Number instead of as a float64.
+func (dec *Decoder) UseNumber() { dec.d.useNumber = true }
+
+// DisallowUnknownFields causes the Decoder to return an error when the destination
+// is a struct and the input contains object keys which do not match any
+// non-ignored, exported fields in the destination.
+func (dec *Decoder) DisallowUnknownFields() { dec.d.disallowUnknownFields = true }
+
+// Decode reads the next JSON-encoded value from its
+// input and stores it in the value pointed to by v.
+//
+// See the documentation for Unmarshal for details about
+// the conversion of JSON into a Go value.
+func (dec *Decoder) Decode(v any) error {
+	if dec.err != nil {
+		return dec.err
+	}
+
+	if err := dec.tokenPrepareForDecode(); err != nil {
+		return err
+	}
+
+	if !dec.tokenValueAllowed() {
+		return &SyntaxError{msg: "not at beginning of value", Offset: dec.InputOffset()}
+	}
+
+	// Read whole value into buffer.
+	n, err := dec.readValue()
+	if err != nil {
+		return err
+	}
+	dec.d.init(dec.buf[dec.scanp : dec.scanp+n])
+	dec.scanp += n
+
+	// Don't save err from unmarshal into dec.err:
+	// the connection is still usable since we read a complete JSON
+	// object from it before the error happened.
+	err = dec.d.unmarshal(v)
+
+	// fixup token streaming state
+	dec.tokenValueEnd()
+
+	return err
+}
+
+// Buffered returns a reader of the data remaining in the Decoder's
+// buffer. The reader is valid until the next call to Decode.
+func (dec *Decoder) Buffered() io.Reader {
+	return bytes.NewReader(dec.buf[dec.scanp:])
+}
+
+// readValue reads a JSON value into dec.buf.
+// It returns the length of the encoding.
+func (dec *Decoder) readValue() (int, error) {
+	dec.scan.reset()
+
+	scanp := dec.scanp
+	var err error
+Input:
+	// help the compiler see that scanp is never negative, so it can remove
+	// some bounds checks below.
+	for scanp >= 0 {
+
+		// Look in the buffer for a new value.
+		for ; scanp < len(dec.buf); scanp++ {
+			c := dec.buf[scanp]
+			dec.scan.bytes++
+			switch dec.scan.step(&dec.scan, c) {
+			case scanEnd:
+				// scanEnd is delayed one byte so we decrement
+				// the scanner bytes count by 1 to ensure that
+				// this value is correct in the next call of Decode.
+				dec.scan.bytes--
+				break Input
+			case scanEndObject, scanEndArray:
+				// scanEnd is delayed one byte.
+				// We might block trying to get that byte from src,
+				// so instead invent a space byte.
+				if stateEndValue(&dec.scan, ' ') == scanEnd {
+					scanp++
+					break Input
+				}
+			case scanError:
+				dec.err = dec.scan.err
+				return 0, dec.scan.err
+			}
+		}
+
+		// Did the last read have an error?
+		// Delayed until now to allow buffer scan.
+		if err != nil {
+			if err == io.EOF {
+				if dec.scan.step(&dec.scan, ' ') == scanEnd {
+					break Input
+				}
+				if nonSpace(dec.buf) {
+					err = io.ErrUnexpectedEOF
+				}
+			}
+			dec.err = err
+			return 0, err
+		}
+
+		n := scanp - dec.scanp
+		err = dec.refill()
+		scanp = dec.scanp + n
+	}
+	return scanp - dec.scanp, nil
+}
+
+func (dec *Decoder) refill() error {
+	// Make room to read more into the buffer.
+	// First slide down data already consumed.
+	if dec.scanp > 0 {
+		dec.scanned += int64(dec.scanp)
+		n := copy(dec.buf, dec.buf[dec.scanp:])
+		dec.buf = dec.buf[:n]
+		dec.scanp = 0
+	}
+
+	// Grow buffer if not large enough.
+	const minRead = 512
+	if cap(dec.buf)-len(dec.buf) < minRead {
+		newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead)
+		copy(newBuf, dec.buf)
+		dec.buf = newBuf
+	}
+
+	// Read. Delay error for next iteration (after scan).
+	n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
+	dec.buf = dec.buf[0 : len(dec.buf)+n]
+
+	return err
+}
+
+func nonSpace(b []byte) bool {
+	for _, c := range b {
+		if !isSpace(c) {
+			return true
+		}
+	}
+	return false
+}
+
+// An Encoder writes JSON values to an output stream.
+type Encoder struct {
+	w             io.Writer
+	err           error
+	escapeHTML    bool
+	nilSafeSlices bool
+	nilSafeMaps   bool
+
+	indentBuf    *bytes.Buffer
+	indentPrefix string
+	indentValue  string
+}
+
+// NewEncoder returns a new encoder that writes to w.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{w: w, escapeHTML: true}
+}
+
+// Encode writes the JSON encoding of v to the stream,
+// followed by a newline character.
+//
+// See the documentation for Marshal for details about the
+// conversion of Go values to JSON.
+func (enc *Encoder) Encode(v any) error {
+	if enc.err != nil {
+		return enc.err
+	}
+
+	e := newEncodeState()
+	defer encodeStatePool.Put(e)
+
+	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
+	if err != nil {
+		return err
+	}
+
+	// Terminate each value with a newline.
+	// This makes the output look a little nicer
+	// when debugging, and some kind of space
+	// is required if the encoded value was a number,
+	// so that the reader knows there aren't more
+	// digits coming.
+	e.WriteByte('\n')
+
+	b := e.Bytes()
+	if enc.indentPrefix != "" || enc.indentValue != "" {
+		if enc.indentBuf == nil {
+			enc.indentBuf = new(bytes.Buffer)
+		}
+		enc.indentBuf.Reset()
+		err = Indent(enc.indentBuf, b, enc.indentPrefix, enc.indentValue)
+		if err != nil {
+			return err
+		}
+		b = enc.indentBuf.Bytes()
+	}
+	if _, err = enc.w.Write(b); err != nil {
+		enc.err = err
+	}
+	return err
+}
+
+// SetIndent instructs the encoder to format each subsequent encoded
+// value as if indented by the package-level function Indent(dst, src, prefix, indent).
+// Calling SetIndent("", "") disables indentation.
+func (enc *Encoder) SetIndent(prefix, indent string) {
+	enc.indentPrefix = prefix
+	enc.indentValue = indent
+}
+
+// SetNilSafeCollection specifies whether to represent nil slices and maps as
+// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
+func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
+	enc.nilSafeSlices = nilSafeSlices
+	enc.nilSafeMaps = nilSafeMaps
+}
+
+// SetEscapeHTML specifies whether problematic HTML characters
+// should be escaped inside JSON quoted strings.
+// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e
+// to avoid certain safety problems that can arise when embedding JSON in HTML.
+//
+// In non-HTML settings where the escaping interferes with the readability
+// of the output, SetEscapeHTML(false) disables this behavior.
+func (enc *Encoder) SetEscapeHTML(on bool) {
+	enc.escapeHTML = on
+}
+
+// RawMessage is a raw encoded JSON value.
+// It implements Marshaler and Unmarshaler and can
+// be used to delay JSON decoding or precompute a JSON encoding.
+type RawMessage []byte
+
+// MarshalJSON returns m as the JSON encoding of m.
+func (m RawMessage) MarshalJSON() ([]byte, error) {
+	if m == nil {
+		return []byte("null"), nil
+	}
+	return m, nil
+}
+
+// UnmarshalJSON sets *m to a copy of data.
+func (m *RawMessage) UnmarshalJSON(data []byte) error {
+	if m == nil {
+		return errors.New("json.RawMessage: UnmarshalJSON on nil pointer")
+	}
+	*m = append((*m)[0:0], data...)
+	return nil
+}
+
+var _ Marshaler = (*RawMessage)(nil)
+var _ Unmarshaler = (*RawMessage)(nil)
+
+// A Token holds a value of one of these types:
+//
+//	Delim, for the four JSON delimiters [ ] { }
+//	bool, for JSON booleans
+//	float64, for JSON numbers
+//	Number, for JSON numbers
+//	string, for JSON string literals
+//	nil, for JSON null
+type Token any
+
+const (
+	tokenTopValue = iota
+	tokenArrayStart
+	tokenArrayValue
+	tokenArrayComma
+	tokenObjectStart
+	tokenObjectKey
+	tokenObjectColon
+	tokenObjectValue
+	tokenObjectComma
+)
+
+// advance tokenstate from a separator state to a value state
+func (dec *Decoder) tokenPrepareForDecode() error {
+	// Note: Not calling peek before switch, to avoid
+	// putting peek into the standard Decode path.
+	// peek is only called when using the Token API.
+	switch dec.tokenState {
+	case tokenArrayComma:
+		c, err := dec.peek()
+		if err != nil {
+			return err
+		}
+		if c != ',' {
+			return &SyntaxError{"expected comma after array element", dec.InputOffset()}
+		}
+		dec.scanp++
+		dec.tokenState = tokenArrayValue
+	case tokenObjectColon:
+		c, err := dec.peek()
+		if err != nil {
+			return err
+		}
+		if c != ':' {
+			return &SyntaxError{"expected colon after object key", dec.InputOffset()}
+		}
+		dec.scanp++
+		dec.tokenState = tokenObjectValue
+	}
+	return nil
+}
+
+func (dec *Decoder) tokenValueAllowed() bool {
+	switch dec.tokenState {
+	case tokenTopValue, tokenArrayStart, tokenArrayValue, tokenObjectValue:
+		return true
+	}
+	return false
+}
+
+func (dec *Decoder) tokenValueEnd() {
+	switch dec.tokenState {
+	case tokenArrayStart, tokenArrayValue:
+		dec.tokenState = tokenArrayComma
+	case tokenObjectValue:
+		dec.tokenState = tokenObjectComma
+	}
+}
+
+// A Delim is a JSON array or object delimiter, one of [ ] { or }.
+type Delim rune
+
+func (d Delim) String() string {
+	return string(d)
+}
+
+// Token returns the next JSON token in the input stream.
+// At the end of the input stream, Token returns nil, io.EOF.
+//
+// Token guarantees that the delimiters [ ] { } it returns are
+// properly nested and matched: if Token encounters an unexpected
+// delimiter in the input, it will return an error.
+//
+// The input stream consists of basic JSON values—bool, string,
+// number, and null—along with delimiters [ ] { } of type Delim
+// to mark the start and end of arrays and objects.
+// Commas and colons are elided.
+func (dec *Decoder) Token() (Token, error) {
+	for {
+		c, err := dec.peek()
+		if err != nil {
+			return nil, err
+		}
+		switch c {
+		case '[':
+			if !dec.tokenValueAllowed() {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
+			dec.tokenState = tokenArrayStart
+			return Delim('['), nil
+
+		case ']':
+			if dec.tokenState != tokenArrayStart && dec.tokenState != tokenArrayComma {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
+			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
+			dec.tokenValueEnd()
+			return Delim(']'), nil
+
+		case '{':
+			if !dec.tokenValueAllowed() {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
+			dec.tokenState = tokenObjectStart
+			return Delim('{'), nil
+
+		case '}':
+			if dec.tokenState != tokenObjectStart && dec.tokenState != tokenObjectComma {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
+			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
+			dec.tokenValueEnd()
+			return Delim('}'), nil
+
+		case ':':
+			if dec.tokenState != tokenObjectColon {
+				return dec.tokenError(c)
+			}
+			dec.scanp++
+			dec.tokenState = tokenObjectValue
+			continue
+
+		case ',':
+			if dec.tokenState == tokenArrayComma {
+				dec.scanp++
+				dec.tokenState = tokenArrayValue
+				continue
+			}
+			if dec.tokenState == tokenObjectComma {
+				dec.scanp++
+				dec.tokenState = tokenObjectKey
+				continue
+			}
+			return dec.tokenError(c)
+
+		case '"':
+			if dec.tokenState == tokenObjectStart || dec.tokenState == tokenObjectKey {
+				var x string
+				old := dec.tokenState
+				dec.tokenState = tokenTopValue
+				err := dec.Decode(&x)
+				dec.tokenState = old
+				if err != nil {
+					return nil, err
+				}
+				dec.tokenState = tokenObjectColon
+				return x, nil
+			}
+			fallthrough
+
+		default:
+			if !dec.tokenValueAllowed() {
+				return dec.tokenError(c)
+			}
+			var x any
+			if err := dec.Decode(&x); err != nil {
+				return nil, err
+			}
+			return x, nil
+		}
+	}
+}
+
+func (dec *Decoder) tokenError(c byte) (Token, error) {
+	var context string
+	switch dec.tokenState {
+	case tokenTopValue:
+		context = " looking for beginning of value"
+	case tokenArrayStart, tokenArrayValue, tokenObjectValue:
+		context = " looking for beginning of value"
+	case tokenArrayComma:
+		context = " after array element"
+	case tokenObjectKey:
+		context = " looking for beginning of object key string"
+	case tokenObjectColon:
+		context = " after object key"
+	case tokenObjectComma:
+		context = " after object key:value pair"
+	}
+	return nil, &SyntaxError{"invalid character " + quoteChar(c) + context, dec.InputOffset()}
+}
+
+// More reports whether there is another element in the
+// current array or object being parsed.
+func (dec *Decoder) More() bool {
+	c, err := dec.peek()
+	return err == nil && c != ']' && c != '}'
+}
+
+func (dec *Decoder) peek() (byte, error) {
+	var err error
+	for {
+		for i := dec.scanp; i < len(dec.buf); i++ {
+			c := dec.buf[i]
+			if isSpace(c) {
+				continue
+			}
+			dec.scanp = i
+			return c, nil
+		}
+		// buffer has been scanned, now report any error
+		if err != nil {
+			return 0, err
+		}
+		err = dec.refill()
+	}
+}
+
+// InputOffset returns the input stream byte offset of the current decoder position.
+// The offset gives the location of the end of the most recently returned token
+// and the beginning of the next token.
+func (dec *Decoder) InputOffset() int64 {
+	return dec.scanned + int64(dec.scanp)
+}

gojson/stream_test.go

@@ -0,0 +1,539 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"runtime/debug"
+	"strings"
+	"testing"
+)
+
+// Test values for the stream test.
+// One of each JSON kind.
+var streamTest = []any{
+	0.1,
+	"hello",
+	nil,
+	true,
+	false,
+	[]any{"a", "b", "c"},
+	map[string]any{"K": "Kelvin", "ß": "long s"},
+	3.14, // another value to make sure something can follow map
+}
+
+var streamEncoded = `0.1
+"hello"
+null
+true
+false
+["a","b","c"]
+{"ß":"long s","K":"Kelvin"}
+3.14
+`
+
+func TestEncoder(t *testing.T) {
+	for i := 0; i <= len(streamTest); i++ {
+		var buf strings.Builder
+		enc := NewEncoder(&buf)
+		// Check that enc.SetIndent("", "") turns off indentation.
+		enc.SetIndent(">", ".")
+		enc.SetIndent("", "")
+		for j, v := range streamTest[0:i] {
+			if err := enc.Encode(v); err != nil {
+				t.Fatalf("encode #%d: %v", j, err)
+			}
+		}
+		if have, want := buf.String(), nlines(streamEncoded, i); have != want {
+			t.Errorf("encoding %d items: mismatch", i)
+			diff(t, []byte(have), []byte(want))
+			break
+		}
+	}
+}
+
+func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
+	// Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test.
+	percent := debug.SetGCPercent(-1)
+	defer debug.SetGCPercent(percent)
+
+	// Trigger an error in Marshal with cyclic data.
+	type Dummy struct {
+		Name string
+		Next *Dummy
+	}
+	dummy := Dummy{Name: "Dummy"}
+	dummy.Next = &dummy
+
+	var buf bytes.Buffer
+	enc := NewEncoder(&buf)
+	if err := enc.Encode(dummy); err == nil {
+		t.Errorf("Encode(dummy) == nil; want error")
+	}
+
+	type Data struct {
+		A string
+		I int
+	}
+	data := Data{A: "a", I: 1}
+	if err := enc.Encode(data); err != nil {
+		t.Errorf("Marshal(%v) = %v", data, err)
+	}
+
+	var data2 Data
+	if err := Unmarshal(buf.Bytes(), &data2); err != nil {
+		t.Errorf("Unmarshal(%v) = %v", data2, err)
+	}
+	if data2 != data {
+		t.Errorf("expect: %v, but get: %v", data, data2)
+	}
+}
+
+var streamEncodedIndent = `0.1
+"hello"
+null
+true
+false
+[
+>."a",
+>."b",
+>."c"
+>]
+{
+>."ß": "long s",
+>."K": "Kelvin"
+>}
+3.14
+`
+
+func TestEncoderIndent(t *testing.T) {
+	var buf strings.Builder
+	enc := NewEncoder(&buf)
+	enc.SetIndent(">", ".")
+	for _, v := range streamTest {
+		enc.Encode(v)
+	}
+	if have, want := buf.String(), streamEncodedIndent; have != want {
+		t.Error("indented encoding mismatch")
+		diff(t, []byte(have), []byte(want))
+	}
+}
+
+type strMarshaler string
+
+func (s strMarshaler) MarshalJSON() ([]byte, error) {
+	return []byte(s), nil
+}
+
+type strPtrMarshaler string
+
+func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) {
+	return []byte(*s), nil
+}
+
+func TestEncoderSetEscapeHTML(t *testing.T) {
+	var c C
+	var ct CText
+	var tagStruct struct {
+		Valid   int `json:"<>&#! "`
+		Invalid int `json:"\\"`
+	}
+
+	// This case is particularly interesting, as we force the encoder to
+	// take the address of the Ptr field to use its MarshalJSON method. This
+	// is why the '&' is important.
+	marshalerStruct := &struct {
+		NonPtr strMarshaler
+		Ptr    strPtrMarshaler
+	}{`"<str>"`, `"<str>"`}
+
+	// https://golang.org/issue/34154
+	stringOption := struct {
+		Bar string `json:"bar,string"`
+	}{`<html>foobar</html>`}
+
+	for _, tt := range []struct {
+		name       string
+		v          any
+		wantEscape string
+		want       string
+	}{
+		{"c", c, `"\u003c\u0026\u003e"`, `"<&>"`},
+		{"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`},
+		{`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`},
+		{
+			"tagStruct", tagStruct,
+			`{"\u003c\u003e\u0026#! ":0,"Invalid":0}`,
+			`{"<>&#! ":0,"Invalid":0}`,
+		},
+		{
+			`"<str>"`, marshalerStruct,
+			`{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`,
+			`{"NonPtr":"<str>","Ptr":"<str>"}`,
+		},
+		{
+			"stringOption", stringOption,
+			`{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
+			`{"bar":"\"<html>foobar</html>\""}`,
+		},
+	} {
+		var buf strings.Builder
+		enc := NewEncoder(&buf)
+		if err := enc.Encode(tt.v); err != nil {
+			t.Errorf("Encode(%s): %s", tt.name, err)
+			continue
+		}
+		if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
+			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape)
+		}
+		buf.Reset()
+		enc.SetEscapeHTML(false)
+		if err := enc.Encode(tt.v); err != nil {
+			t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
+			continue
+		}
+		if got := strings.TrimSpace(buf.String()); got != tt.want {
+			t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q",
+				tt.name, got, tt.want)
+		}
+	}
+}
+
+func TestDecoder(t *testing.T) {
+	for i := 0; i <= len(streamTest); i++ {
+		// Use stream without newlines as input,
+		// just to stress the decoder even more.
+		// Our test input does not include back-to-back numbers.
+		// Otherwise stripping the newlines would
+		// merge two adjacent JSON values.
+		var buf bytes.Buffer
+		for _, c := range nlines(streamEncoded, i) {
+			if c != '\n' {
+				buf.WriteRune(c)
+			}
+		}
+		out := make([]any, i)
+		dec := NewDecoder(&buf)
+		for j := range out {
+			if err := dec.Decode(&out[j]); err != nil {
+				t.Fatalf("decode #%d/%d: %v", j, i, err)
+			}
+		}
+		if !reflect.DeepEqual(out, streamTest[0:i]) {
+			t.Errorf("decoding %d items: mismatch", i)
+			for j := range out {
+				if !reflect.DeepEqual(out[j], streamTest[j]) {
+					t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j])
+				}
+			}
+			break
+		}
+	}
+}
+
+func TestDecoderBuffered(t *testing.T) {
+	r := strings.NewReader(`{"Name": "Gopher"} extra `)
+	var m struct {
+		Name string
+	}
+	d := NewDecoder(r)
+	err := d.Decode(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if m.Name != "Gopher" {
+		t.Errorf("Name = %q; want Gopher", m.Name)
+	}
+	rest, err := io.ReadAll(d.Buffered())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g, w := string(rest), " extra "; g != w {
+		t.Errorf("Remaining = %q; want %q", g, w)
+	}
+}
+
+func nlines(s string, n int) string {
+	if n <= 0 {
+		return ""
+	}
+	for i, c := range s {
+		if c == '\n' {
+			if n--; n == 0 {
+				return s[0 : i+1]
+			}
+		}
+	}
+	return s
+}
+
+func TestRawMessage(t *testing.T) {
+	var data struct {
+		X  float64
+		Id RawMessage
+		Y  float32
+	}
+	const raw = `["\u0056",null]`
+	const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
+	err := Unmarshal([]byte(msg), &data)
+	if err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+	if string([]byte(data.Id)) != raw {
+		t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw)
+	}
+	b, err := Marshal(&data)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if string(b) != msg {
+		t.Fatalf("Marshal: have %#q want %#q", b, msg)
+	}
+}
+
+func TestNullRawMessage(t *testing.T) {
+	var data struct {
+		X     float64
+		Id    RawMessage
+		IdPtr *RawMessage
+		Y     float32
+	}
+	const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}`
+	err := Unmarshal([]byte(msg), &data)
+	if err != nil {
+		t.Fatalf("Unmarshal: %v", err)
+	}
+	if want, got := "null", string(data.Id); want != got {
+		t.Fatalf("Raw mismatch: have %q, want %q", got, want)
+	}
+	if data.IdPtr != nil {
+		t.Fatalf("Raw pointer mismatch: have non-nil, want nil")
+	}
+	b, err := Marshal(&data)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+	if string(b) != msg {
+		t.Fatalf("Marshal: have %#q want %#q", b, msg)
+	}
+}
+
+var blockingTests = []string{
+	`{"x": 1}`,
+	`[1, 2, 3]`,
+}
+
+func TestBlocking(t *testing.T) {
+	for _, enc := range blockingTests {
+		r, w := net.Pipe()
+		go w.Write([]byte(enc))
+		var val any
+
+		// If Decode reads beyond what w.Write writes above,
+		// it will block, and the test will deadlock.
+		if err := NewDecoder(r).Decode(&val); err != nil {
+			t.Errorf("decoding %s: %v", enc, err)
+		}
+		r.Close()
+		w.Close()
+	}
+}
+
+type tokenStreamCase struct {
+	json      string
+	expTokens []any
+}
+
+type decodeThis struct {
+	v any
+}
+
+var tokenStreamCases = []tokenStreamCase{
+	// streaming token cases
+	{json: `10`, expTokens: []any{float64(10)}},
+	{json: ` [10] `, expTokens: []any{
+		Delim('['), float64(10), Delim(']')}},
+	{json: ` [false,10,"b"] `, expTokens: []any{
+		Delim('['), false, float64(10), "b", Delim(']')}},
+	{json: `{ "a": 1 }`, expTokens: []any{
+		Delim('{'), "a", float64(1), Delim('}')}},
+	{json: `{"a": 1, "b":"3"}`, expTokens: []any{
+		Delim('{'), "a", float64(1), "b", "3", Delim('}')}},
+	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{
+		Delim('['),
+		Delim('{'), "a", float64(1), Delim('}'),
+		Delim('{'), "a", float64(2), Delim('}'),
+		Delim(']')}},
+	{json: `{"obj": {"a": 1}}`, expTokens: []any{
+		Delim('{'), "obj", Delim('{'), "a", float64(1), Delim('}'),
+		Delim('}')}},
+	{json: `{"obj": [{"a": 1}]}`, expTokens: []any{
+		Delim('{'), "obj", Delim('['),
+		Delim('{'), "a", float64(1), Delim('}'),
+		Delim(']'), Delim('}')}},
+
+	// streaming tokens with intermittent Decode()
+	{json: `{ "a": 1 }`, expTokens: []any{
+		Delim('{'), "a",
+		decodeThis{float64(1)},
+		Delim('}')}},
+	{json: ` [ { "a" : 1 } ] `, expTokens: []any{
+		Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		Delim(']')}},
+	{json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{
+		Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		decodeThis{map[string]any{"a": float64(2)}},
+		Delim(']')}},
+	{json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []any{
+		Delim('{'), "obj", Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		Delim(']'), Delim('}')}},
+
+	{json: `{"obj": {"a": 1}}`, expTokens: []any{
+		Delim('{'), "obj",
+		decodeThis{map[string]any{"a": float64(1)}},
+		Delim('}')}},
+	{json: `{"obj": [{"a": 1}]}`, expTokens: []any{
+		Delim('{'), "obj",
+		decodeThis{[]any{
+			map[string]any{"a": float64(1)},
+		}},
+		Delim('}')}},
+	{json: ` [{"a": 1} {"a": 2}] `, expTokens: []any{
+		Delim('['),
+		decodeThis{map[string]any{"a": float64(1)}},
+		decodeThis{&SyntaxError{"expected comma after array element", 11}},
+	}},
+	{json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []any{
+		Delim('{'), strings.Repeat("a", 513),
+		decodeThis{&SyntaxError{"expected colon after object key", 518}},
+	}},
+	{json: `{ "\a" }`, expTokens: []any{
+		Delim('{'),
+		&SyntaxError{"invalid character 'a' in string escape code", 3},
+	}},
+	{json: ` \a`, expTokens: []any{
+		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
+	}},
+}
+
+func TestDecodeInStream(t *testing.T) {
+	for ci, tcase := range tokenStreamCases {
+
+		dec := NewDecoder(strings.NewReader(tcase.json))
+		for i, etk := range tcase.expTokens {
+
+			var tk any
+			var err error
+
+			if dt, ok := etk.(decodeThis); ok {
+				etk = dt.v
+				err = dec.Decode(&tk)
+			} else {
+				tk, err = dec.Token()
+			}
+			if experr, ok := etk.(error); ok {
+				if err == nil || !reflect.DeepEqual(err, experr) {
+					t.Errorf("case %v: Expected error %#v in %q, but was %#v", ci, experr, tcase.json, err)
+				}
+				break
+			} else if err == io.EOF {
+				t.Errorf("case %v: Unexpected EOF in %q", ci, tcase.json)
+				break
+			} else if err != nil {
+				t.Errorf("case %v: Unexpected error '%#v' in %q", ci, err, tcase.json)
+				break
+			}
+			if !reflect.DeepEqual(tk, etk) {
+				t.Errorf(`case %v: %q @ %v expected %T(%v) was %T(%v)`, ci, tcase.json, i, etk, etk, tk, tk)
+				break
+			}
+		}
+	}
+}
+
+// Test from golang.org/issue/11893
+func TestHTTPDecoding(t *testing.T) {
+	const raw = `{ "foo": "bar" }`
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(raw))
+	}))
+	defer ts.Close()
+	res, err := http.Get(ts.URL)
+	if err != nil {
+		log.Fatalf("GET failed: %v", err)
+	}
+	defer res.Body.Close()
+
+	foo := struct {
+		Foo string
+	}{}
+
+	d := NewDecoder(res.Body)
+	err = d.Decode(&foo)
+	if err != nil {
+		t.Fatalf("Decode: %v", err)
+	}
+	if foo.Foo != "bar" {
+		t.Errorf("decoded %q; want \"bar\"", foo.Foo)
+	}
+
+	// make sure we get the EOF the second time
+	err = d.Decode(&foo)
+	if err != io.EOF {
+		t.Errorf("err = %v; want io.EOF", err)
+	}
+}
+
+func TestEncoderSetNilSafeCollection(t *testing.T) {
+	var (
+		nilSlice  []interface{}
+		pNilSlice *[]interface{}
+		nilMap    map[string]interface{}
+		pNilMap   *map[string]interface{}
+	)
+	for _, tt := range []struct {
+		name        string
+		v           interface{}
+		want        string
+		rescuedWant string
+	}{
+		{"nilSlice", nilSlice, "null", "[]"},
+		{"nonNilSlice", []interface{}{}, "[]", "[]"},
+		{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
+		{"pNilSlice", pNilSlice, "null", "null"},
+		{"nilMap", nilMap, "null", "{}"},
+		{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
+		{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
+		{"pNilMap", pNilMap, "null", "null"},
+	} {
+		var buf bytes.Buffer
+		enc := NewEncoder(&buf)
+		if err := enc.Encode(tt.v); err != nil {
+			t.Fatalf("Encode(%s): %s", tt.name, err)
+		}
+		if got := strings.TrimSpace(buf.String()); got != tt.want {
+			t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
+		}
+		buf.Reset()
+		enc.SetNilSafeCollection(true, true)
+		if err := enc.Encode(tt.v); err != nil {
+			t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
+		}
+		if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
+			t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
+				tt.name, got, tt.want)
+		}
+	}
+}

gojson/tables.go

@@ -0,0 +1,218 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import "unicode/utf8"
+
+// safeSet holds the value true if the ASCII character with the given array
+// position can be represented inside a JSON string without any further
+// escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), and the backslash character ("\").
+var safeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      true,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      true,
+	'=':      true,
+	'>':      true,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}
+
+// htmlSafeSet holds the value true if the ASCII character with the given
+// array position can be safely represented inside a JSON string, embedded
+// inside of HTML <script> tags, without any additional escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), the backslash character ("\"), HTML opening and closing
+// tags ("<" and ">"), and the ampersand ("&").
+var htmlSafeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      false,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      false,
+	'=':      true,
+	'>':      false,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}

gojson/tagkey_test.go

@@ -0,0 +1,120 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"testing"
+)
+
+type basicLatin2xTag struct {
+	V string `json:"$%-/"`
+}
+
+type basicLatin3xTag struct {
+	V string `json:"0123456789"`
+}
+
+type basicLatin4xTag struct {
+	V string `json:"ABCDEFGHIJKLMO"`
+}
+
+type basicLatin5xTag struct {
+	V string `json:"PQRSTUVWXYZ_"`
+}
+
+type basicLatin6xTag struct {
+	V string `json:"abcdefghijklmno"`
+}
+
+type basicLatin7xTag struct {
+	V string `json:"pqrstuvwxyz"`
+}
+
+type miscPlaneTag struct {
+	V string `json:"色は匂へど"`
+}
+
+type percentSlashTag struct {
+	V string `json:"text/html%"` // https://golang.org/issue/2718
+}
+
+type punctuationTag struct {
+	V string `json:"!#$%&()*+-./:;<=>?@[]^_{|}~ "` // https://golang.org/issue/3546
+}
+
+type dashTag struct {
+	V string `json:"-,"`
+}
+
+type emptyTag struct {
+	W string
+}
+
+type misnamedTag struct {
+	X string `jsom:"Misnamed"`
+}
+
+type badFormatTag struct {
+	Y string `:"BadFormat"`
+}
+
+type badCodeTag struct {
+	Z string `json:" !\"#&'()*+,."`
+}
+
+type spaceTag struct {
+	Q string `json:"With space"`
+}
+
+type unicodeTag struct {
+	W string `json:"Ελλάδα"`
+}
+
+var structTagObjectKeyTests = []struct {
+	raw   any
+	value string
+	key   string
+}{
+	{basicLatin2xTag{"2x"}, "2x", "$%-/"},
+	{basicLatin3xTag{"3x"}, "3x", "0123456789"},
+	{basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"},
+	{basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"},
+	{basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"},
+	{basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"},
+	{miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"},
+	{dashTag{"foo"}, "foo", "-"},
+	{emptyTag{"Pour Moi"}, "Pour Moi", "W"},
+	{misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"},
+	{badFormatTag{"Orfevre"}, "Orfevre", "Y"},
+	{badCodeTag{"Reliable Man"}, "Reliable Man", "Z"},
+	{percentSlashTag{"brut"}, "brut", "text/html%"},
+	{punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:;<=>?@[]^_{|}~ "},
+	{spaceTag{"Perreddu"}, "Perreddu", "With space"},
+	{unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"},
+}
+
+func TestStructTagObjectKey(t *testing.T) {
+	for _, tt := range structTagObjectKeyTests {
+		b, err := Marshal(tt.raw)
+		if err != nil {
+			t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err)
+		}
+		var f any
+		err = Unmarshal(b, &f)
+		if err != nil {
+			t.Fatalf("Unmarshal(%#q) failed: %v", b, err)
+		}
+		for i, v := range f.(map[string]any) {
+			switch i {
+			case tt.key:
+				if s, ok := v.(string); !ok || s != tt.value {
+					t.Fatalf("Unexpected value: %#q, want %v", s, tt.value)
+				}
+			default:
+				t.Fatalf("Unexpected key: %#q, from %#q", i, b)
+			}
+		}
+	}
+}

gojson/tags.go

@@ -0,0 +1,38 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"strings"
+)
+
+// tagOptions is the string following a comma in a struct field's "json"
+// tag, or the empty string. It does not include the leading comma.
+type tagOptions string
+
+// parseTag splits a struct field's json tag into its name and
+// comma-separated options.
+func parseTag(tag string) (string, tagOptions) {
+	tag, opt, _ := strings.Cut(tag, ",")
+	return tag, tagOptions(opt)
+}
+
+// Contains reports whether a comma-separated list of options
+// contains a particular substr flag. substr must be surrounded by a
+// string boundary or commas.
+func (o tagOptions) Contains(optionName string) bool {
+	if len(o) == 0 {
+		return false
+	}
+	s := string(o)
+	for s != "" {
+		var name string
+		name, s, _ = strings.Cut(s, ",")
+		if name == optionName {
+			return true
+		}
+	}
+	return false
+}

gojson/tags_test.go

@@ -0,0 +1,28 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+	"testing"
+)
+
+func TestTagParsing(t *testing.T) {
+	name, opts := parseTag("field,foobar,foo")
+	if name != "field" {
+		t.Fatalf("name = %q, want field", name)
+	}
+	for _, tt := range []struct {
+		opt  string
+		want bool
+	}{
+		{"foobar", true},
+		{"foo", true},
+		{"bar", false},
+	} {
+		if opts.Contains(tt.opt) != tt.want {
+			t.Errorf("Contains(%q) = %v", tt.opt, !tt.want)
+		}
+	}
+}

langext/array.go

@@ -1,6 +1,8 @@
 package langext
 
 import (
+	"errors"
+	"fmt"
 	"reflect"
 )
 
@@ -70,7 +72,73 @@ func ArrEqualsExact[T comparable](arr1 []T, arr2 []T) bool {
 	return true
 }
 
-func ArrAll(arr interface{}, fn func(int) bool) bool {
+func ArrAll[T any](arr []T, fn func(T) bool) bool {
+	for _, av := range arr {
+		if !fn(av) {
+			return false
+		}
+	}
+	return true
+}
+
+func ArrAllErr[T any](arr []T, fn func(T) (bool, error)) (bool, error) {
+	for _, av := range arr {
+		v, err := fn(av)
+		if err != nil {
+			return false, err
+		}
+		if !v {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func ArrNone[T any](arr []T, fn func(T) bool) bool {
+	for _, av := range arr {
+		if fn(av) {
+			return false
+		}
+	}
+	return true
+}
+
+func ArrNoneErr[T any](arr []T, fn func(T) (bool, error)) (bool, error) {
+	for _, av := range arr {
+		v, err := fn(av)
+		if err != nil {
+			return false, err
+		}
+		if v {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func ArrAny[T any](arr []T, fn func(T) bool) bool {
+	for _, av := range arr {
+		if fn(av) {
+			return true
+		}
+	}
+	return false
+}
+
+func ArrAnyErr[T any](arr []T, fn func(T) (bool, error)) (bool, error) {
+	for _, av := range arr {
+		v, err := fn(av)
+		if err != nil {
+			return false, err
+		}
+		if v {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func ArrIdxAll(arr any, fn func(int) bool) bool {
 	av := reflect.ValueOf(arr)
 	for i := 0; i < av.Len(); i++ {
 		if !fn(i) {
@@ -80,7 +148,7 @@ func ArrAll(arr interface{}, fn func(int) bool) bool {
 	return true
 }
 
-func ArrAllErr(arr interface{}, fn func(int) (bool, error)) (bool, error) {
+func ArrIdxAllErr(arr any, fn func(int) (bool, error)) (bool, error) {
 	av := reflect.ValueOf(arr)
 	for i := 0; i < av.Len(); i++ {
 		v, err := fn(i)
@@ -94,7 +162,7 @@ func ArrAllErr(arr interface{}, fn func(int) (bool, error)) (bool, error) {
 	return true, nil
 }
 
-func ArrNone(arr interface{}, fn func(int) bool) bool {
+func ArrIdxNone(arr any, fn func(int) bool) bool {
 	av := reflect.ValueOf(arr)
 	for i := 0; i < av.Len(); i++ {
 		if fn(i) {
@@ -104,7 +172,7 @@ func ArrNone(arr interface{}, fn func(int) bool) bool {
 	return true
 }
 
-func ArrNoneErr(arr interface{}, fn func(int) (bool, error)) (bool, error) {
+func ArrIdxNoneErr(arr any, fn func(int) (bool, error)) (bool, error) {
 	av := reflect.ValueOf(arr)
 	for i := 0; i < av.Len(); i++ {
 		v, err := fn(i)
@@ -118,7 +186,7 @@ func ArrNoneErr(arr interface{}, fn func(int) (bool, error)) (bool, error) {
 	return true, nil
 }
 
-func ArrAny(arr interface{}, fn func(int) bool) bool {
+func ArrIdxAny(arr any, fn func(int) bool) bool {
 	av := reflect.ValueOf(arr)
 	for i := 0; i < av.Len(); i++ {
 		if fn(i) {
@@ -128,7 +196,7 @@ func ArrAny(arr interface{}, fn func(int) bool) bool {
 	return false
 }
 
-func ArrAnyErr(arr interface{}, fn func(int) (bool, error)) (bool, error) {
+func ArrIdxAnyErr(arr any, fn func(int) (bool, error)) (bool, error) {
 	av := reflect.ValueOf(arr)
 	for i := 0; i < av.Len(); i++ {
 		v, err := fn(i)
@@ -142,7 +210,7 @@ func ArrAnyErr(arr interface{}, fn func(int) (bool, error)) (bool, error) {
 	return false, nil
 }
 
-func ArrFirst[T comparable](arr []T, comp func(v T) bool) (T, bool) {
+func ArrFirst[T any](arr []T, comp func(v T) bool) (T, bool) {
 	for _, v := range arr {
 		if comp(v) {
 			return v, true
@@ -151,7 +219,16 @@ func ArrFirst[T comparable](arr []T, comp func(v T) bool) (T, bool) {
 	return *new(T), false
 }
 
-func ArrLast[T comparable](arr []T, comp func(v T) bool) (T, bool) {
+func ArrFirstOrNil[T any](arr []T, comp func(v T) bool) *T {
+	for _, v := range arr {
+		if comp(v) {
+			return Ptr(v)
+		}
+	}
+	return nil
+}
+
+func ArrLast[T any](arr []T, comp func(v T) bool) (T, bool) {
 	found := false
 	result := *new(T)
 	for _, v := range arr {
@@ -163,6 +240,22 @@ func ArrLast[T comparable](arr []T, comp func(v T) bool) (T, bool) {
 	return result, found
 }
 
+func ArrLastOrNil[T any](arr []T, comp func(v T) bool) *T {
+	found := false
+	result := *new(T)
+	for _, v := range arr {
+		if comp(v) {
+			found = true
+			result = v
+		}
+	}
+	if found {
+		return Ptr(result)
+	} else {
+		return nil
+	}
+}
+
 func ArrFirstIndex[T comparable](arr []T, needle T) int {
 	for i, v := range arr {
 		if v == needle {
@@ -199,6 +292,66 @@ func ArrMap[T1 any, T2 any](arr []T1, conv func(v T1) T2) []T2 {
 	return r
 }
 
+func MapMap[TK comparable, TV any, TR any](inmap map[TK]TV, conv func(k TK, v TV) TR) []TR {
+	r := make([]TR, 0, len(inmap))
+	for k, v := range inmap {
+		r = append(r, conv(k, v))
+	}
+	return r
+}
+
+func MapMapErr[TK comparable, TV any, TR any](inmap map[TK]TV, conv func(k TK, v TV) (TR, error)) ([]TR, error) {
+	r := make([]TR, 0, len(inmap))
+	for k, v := range inmap {
+		elem, err := conv(k, v)
+		if err != nil {
+			return nil, err
+		}
+		r = append(r, elem)
+	}
+	return r, nil
+}
+
+func ArrMapExt[T1 any, T2 any](arr []T1, conv func(idx int, v T1) T2) []T2 {
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i] = conv(i, v)
+	}
+	return r
+}
+
+func ArrMapErr[T1 any, T2 any](arr []T1, conv func(v T1) (T2, error)) ([]T2, error) {
+	var err error
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i], err = conv(v)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return r, nil
+}
+
+func ArrFilterMap[T1 any, T2 any](arr []T1, filter func(v T1) bool, conv func(v T1) T2) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v := range arr {
+		if filter(v) {
+			r = append(r, conv(v))
+		}
+	}
+	return r
+}
+
+func ArrFilter[T any](arr []T, filter func(v T) bool) []T {
+	r := make([]T, 0, len(arr))
+	for _, v := range arr {
+		if filter(v) {
+			r = append(r, v)
+		}
+	}
+	return r
+}
+
 func ArrSum[T NumberConstraint](arr []T) T {
 	var r T = 0
 	for _, v := range arr {
@@ -206,3 +359,123 @@ func ArrSum[T NumberConstraint](arr []T) T {
 	}
 	return r
 }
+
+func ArrFlatten[T1 any, T2 any](arr []T1, conv func(v T1) []T2) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v1 := range arr {
+		r = append(r, conv(v1)...)
+	}
+	return r
+}
+
+func ArrFlattenDirect[T1 any](arr [][]T1) []T1 {
+	r := make([]T1, 0, len(arr))
+	for _, v1 := range arr {
+		r = append(r, v1...)
+	}
+	return r
+}
+
+func ArrCastToAny[T1 any](arr []T1) []any {
+	r := make([]any, len(arr))
+	for i, v := range arr {
+		r[i] = any(v)
+	}
+	return r
+}
+
+func ArrCastSafe[T1 any, T2 any](arr []T1) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v := range arr {
+		if vcast, ok := any(v).(T2); ok {
+			r = append(r, vcast)
+		}
+	}
+	return r
+}
+
+func ArrCastErr[T1 any, T2 any](arr []T1) ([]T2, error) {
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		if vcast, ok := any(v).(T2); ok {
+			r[i] = vcast
+		} else {
+			return nil, errors.New(fmt.Sprintf("Cannot cast element %d of type %T to type %v", i, v, *new(T2)))
+		}
+	}
+	return r, nil
+}
+
+func ArrCastPanic[T1 any, T2 any](arr []T1) []T2 {
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		if vcast, ok := any(v).(T2); ok {
+			r[i] = vcast
+		} else {
+			panic(fmt.Sprintf("Cannot cast element %d of type %T to type %v", i, v, *new(T2)))
+		}
+	}
+	return r
+}
+
+func ArrConcat[T any](arr ...[]T) []T {
+	c := 0
+	for _, v := range arr {
+		c += len(v)
+	}
+	r := make([]T, c)
+	i := 0
+	for _, av := range arr {
+		for _, v := range av {
+			r[i] = v
+			i++
+		}
+	}
+	return r
+}
+
+// ArrCopy does a shallow copy of the 'in' array
+func ArrCopy[T any](in []T) []T {
+	out := make([]T, len(in))
+	copy(out, in)
+	return out
+}
+
+func ArrRemove[T comparable](arr []T, needle T) []T {
+	idx := ArrFirstIndex(arr, needle)
+	if idx >= 0 {
+		return append(arr[:idx], arr[idx+1:]...)
+	}
+	return arr
+}
+
+func ArrExcept[T comparable](arr []T, needles ...T) []T {
+	r := make([]T, 0, len(arr))
+	rmlist := ArrToSet(needles)
+	for _, v := range arr {
+		if _, ok := rmlist[v]; !ok {
+			r = append(r, v)
+		}
+	}
+	return r
+}
+
+func ArrayToInterface[T any](t []T) []interface{} {
+	res := make([]interface{}, 0, len(t))
+	for i, _ := range t {
+		res = append(res, t[i])
+	}
+	return res
+}
+
+func JoinString(arr []string, delimiter string) string {
+	str := ""
+	for i, v := range arr {
+		str += v
+		if i < len(arr)-1 {
+			str += delimiter
+		}
+	}
+
+	return str
+}

langext/array_test.go

@@ -0,0 +1,12 @@
+package langext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestJoinString(t *testing.T) {
+	ids := []string{"1", "2", "3"}
+	res := JoinString(ids, ",")
+	tst.AssertEqual(t, res, "1,2,3")
+}

langext/base58.go

@@ -0,0 +1,178 @@
+package langext
+
+import (
+	"bytes"
+	"errors"
+	"math/big"
+)
+
+// shamelessly stolen from https://github.com/btcsuite/
+
+type B58Encoding struct {
+	bigRadix     [11]*big.Int
+	bigRadix10   *big.Int
+	alphabet     string
+	alphabetIdx0 byte
+	b58          [256]byte
+}
+
+var Base58DefaultEncoding = newBase58Encoding("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
+var Base58FlickrEncoding = newBase58Encoding("123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ")
+var Base58RippleEncoding = newBase58Encoding("rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz")
+var Base58BitcoinEncoding = newBase58Encoding("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
+
+func newBase58Encoding(alphabet string) *B58Encoding {
+	bigRadix10 := big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58 * 58 * 58 * 58)
+	enc := &B58Encoding{
+		alphabet:     alphabet,
+		alphabetIdx0: '1',
+		bigRadix: [...]*big.Int{
+			big.NewInt(0),
+			big.NewInt(58),
+			big.NewInt(58 * 58),
+			big.NewInt(58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58 * 58 * 58),
+			bigRadix10,
+		},
+		bigRadix10: bigRadix10,
+	}
+
+	b58 := make([]byte, 0, 256)
+
+	for i := byte(0); i < 32; i++ {
+		for j := byte(0); j < 8; j++ {
+
+			b := i*8 + j
+
+			idx := bytes.IndexByte([]byte(alphabet), b)
+			if idx == -1 {
+				b58 = append(b58, 255)
+			} else {
+				b58 = append(b58, byte(idx))
+			}
+
+		}
+	}
+
+	enc.b58 = *((*[256]byte)(b58))
+
+	return enc
+}
+
+func (enc *B58Encoding) EncodeString(src string) (string, error) {
+	v, err := enc.Encode([]byte(src))
+	if err != nil {
+		return "", err
+	}
+	return string(v), nil
+}
+
+func (enc *B58Encoding) Encode(src []byte) ([]byte, error) {
+	x := new(big.Int)
+	x.SetBytes(src)
+
+	// maximum length of output is log58(2^(8*len(b))) == len(b) * 8 / log(58)
+	maxlen := int(float64(len(src))*1.365658237309761) + 1
+	answer := make([]byte, 0, maxlen)
+	mod := new(big.Int)
+	for x.Sign() > 0 {
+		// Calculating with big.Int is slow for each iteration.
+		//    x, mod = x / 58, x % 58
+		//
+		// Instead we can try to do as much calculations on int64.
+		//    x, mod = x / 58^10, x % 58^10
+		//
+		// Which will give us mod, which is 10 digit base58 number.
+		// We'll loop that 10 times to convert to the answer.
+
+		x.DivMod(x, enc.bigRadix10, mod)
+		if x.Sign() == 0 {
+			// When x = 0, we need to ensure we don't add any extra zeros.
+			m := mod.Int64()
+			for m > 0 {
+				answer = append(answer, enc.alphabet[m%58])
+				m /= 58
+			}
+		} else {
+			m := mod.Int64()
+			for i := 0; i < 10; i++ {
+				answer = append(answer, enc.alphabet[m%58])
+				m /= 58
+			}
+		}
+	}
+
+	// leading zero bytes
+	for _, i := range src {
+		if i != 0 {
+			break
+		}
+		answer = append(answer, enc.alphabetIdx0)
+	}
+
+	// reverse
+	alen := len(answer)
+	for i := 0; i < alen/2; i++ {
+		answer[i], answer[alen-1-i] = answer[alen-1-i], answer[i]
+	}
+
+	return answer, nil
+}
+
+func (enc *B58Encoding) DecodeString(src string) (string, error) {
+	v, err := enc.Decode([]byte(src))
+	if err != nil {
+		return "", err
+	}
+	return string(v), nil
+}
+
+func (enc *B58Encoding) Decode(src []byte) ([]byte, error) {
+	answer := big.NewInt(0)
+	scratch := new(big.Int)
+
+	for t := src; len(t) > 0; {
+		n := len(t)
+		if n > 10 {
+			n = 10
+		}
+
+		total := uint64(0)
+		for _, v := range t[:n] {
+			if v > 255 {
+				return []byte{}, errors.New("invalid char in input")
+			}
+
+			tmp := enc.b58[v]
+			if tmp == 255 {
+				return []byte{}, errors.New("invalid char in input")
+			}
+			total = total*58 + uint64(tmp)
+		}
+
+		answer.Mul(answer, enc.bigRadix[n])
+		scratch.SetUint64(total)
+		answer.Add(answer, scratch)
+
+		t = t[n:]
+	}
+
+	tmpval := answer.Bytes()
+
+	var numZeros int
+	for numZeros = 0; numZeros < len(src); numZeros++ {
+		if src[numZeros] != enc.alphabetIdx0 {
+			break
+		}
+	}
+	flen := numZeros + len(tmpval)
+	val := make([]byte, flen)
+	copy(val[numZeros:], tmpval)
+
+	return val, nil
+}

langext/base58_test.go

@@ -0,0 +1,62 @@
+package langext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func _encStr(t *testing.T, enc *B58Encoding, v string) string {
+	v, err := enc.EncodeString(v)
+	if err != nil {
+		t.Error(err)
+	}
+	return v
+}
+
+func _decStr(t *testing.T, enc *B58Encoding, v string) string {
+	v, err := enc.DecodeString(v)
+	if err != nil {
+		t.Error(err)
+	}
+	return v
+}
+
+func TestBase58DefaultEncoding(t *testing.T) {
+	tst.AssertEqual(t, _encStr(t, Base58DefaultEncoding, "Hello"), "9Ajdvzr")
+	tst.AssertEqual(t, _encStr(t, Base58DefaultEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX")
+}
+
+func TestBase58DefaultDecoding(t *testing.T) {
+	tst.AssertEqual(t, _decStr(t, Base58DefaultEncoding, "9Ajdvzr"), "Hello")
+	tst.AssertEqual(t, _decStr(t, Base58DefaultEncoding, "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestBase58RippleEncoding(t *testing.T) {
+	tst.AssertEqual(t, _encStr(t, Base58RippleEncoding, "Hello"), "9wjdvzi")
+	tst.AssertEqual(t, _encStr(t, Base58RippleEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "h3as3SMcJu26nokqPxhkUVCnd3Qwdgbd4Cp3gfReYrsFi7N44bMy11jqnGj1iJHEnzeZCq1huJM7JHifVbi7hXB7ZpEA9DVtqt894reXucNWSNZ26XVaAhy1GSWqGdFeYTJCrMdDzTg3vCcQV55CJjZX")
+}
+
+func TestBase58RippleDecoding(t *testing.T) {
+	tst.AssertEqual(t, _decStr(t, Base58RippleEncoding, "9wjdvzi"), "Hello")
+	tst.AssertEqual(t, _decStr(t, Base58RippleEncoding, "h3as3SMcJu26nokqPxhkUVCnd3Qwdgbd4Cp3gfReYrsFi7N44bMy11jqnGj1iJHEnzeZCq1huJM7JHifVbi7hXB7ZpEA9DVtqt894reXucNWSNZ26XVaAhy1GSWqGdFeYTJCrMdDzTg3vCcQV55CJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestBase58BitcoinEncoding(t *testing.T) {
+	tst.AssertEqual(t, _encStr(t, Base58BitcoinEncoding, "Hello"), "9Ajdvzr")
+	tst.AssertEqual(t, _encStr(t, Base58BitcoinEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX")
+}
+
+func TestBase58BitcoinDecoding(t *testing.T) {
+	tst.AssertEqual(t, _decStr(t, Base58BitcoinEncoding, "9Ajdvzr"), "Hello")
+	tst.AssertEqual(t, _decStr(t, Base58BitcoinEncoding, "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestBase58FlickrEncoding(t *testing.T) {
+	tst.AssertEqual(t, _encStr(t, Base58FlickrEncoding, "Hello"), "9aJCVZR")
+	tst.AssertEqual(t, _encStr(t, Base58FlickrEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638rmBiUzG5NKQoX4KcuE5C8paCFACnE28F7qDx13PRtennAmYSSJQ5gJSRihf5ZDyEQS4UimtihR7uARt4wbty2fW9duTQTM9n1DwUBevreyzGwu6W4YSgrvQgCPDxsiE1mCdZsF8VEBpuHHEiJyw")
+}
+
+func TestBase58FlickrDecoding(t *testing.T) {
+	tst.AssertEqual(t, _decStr(t, Base58FlickrEncoding, "9aJCVZR"), "Hello")
+	tst.AssertEqual(t, _decStr(t, Base58FlickrEncoding, "48638rmBiUzG5NKQoX4KcuE5C8paCFACnE28F7qDx13PRtennAmYSSJQ5gJSRihf5ZDyEQS4UimtihR7uARt4wbty2fW9duTQTM9n1DwUBevreyzGwu6W4YSgrvQgCPDxsiE1mCdZsF8VEBpuHHEiJyw"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}

langext/bool.go

@@ -15,3 +15,35 @@ func Conditional[T any](v bool, resTrue T, resFalse T) T {
 		return resFalse
 	}
 }
+
+func ConditionalFn00[T any](v bool, resTrue T, resFalse T) T {
+	if v {
+		return resTrue
+	} else {
+		return resFalse
+	}
+}
+
+func ConditionalFn10[T any](v bool, resTrue func() T, resFalse T) T {
+	if v {
+		return resTrue()
+	} else {
+		return resFalse
+	}
+}
+
+func ConditionalFn01[T any](v bool, resTrue T, resFalse func() T) T {
+	if v {
+		return resTrue
+	} else {
+		return resFalse()
+	}
+}
+
+func ConditionalFn11[T any](v bool, resTrue func() T, resFalse func() T) T {
+	if v {
+		return resTrue()
+	} else {
+		return resFalse()
+	}
+}

langext/coalesce.go

@@ -60,3 +60,12 @@ func CoalesceStringer(s fmt.Stringer, def string) string {
 		return s.String()
 	}
 }
+
+func SafeCast[T any](v any, def T) T {
+	switch r := v.(type) {
+	case T:
+		return r
+	default:
+		return def
+	}
+}

langext/compare.go

@@ -31,16 +31,16 @@ func CompareIntArr(arr1 []int, arr2 []int) bool {
 	return false
 }
 
-func CompareArr[T OrderedConstraint](arr1 []T, arr2 []T) bool {
+func CompareArr[T OrderedConstraint](arr1 []T, arr2 []T) int {
 
 	for i := 0; i < len(arr1) || i < len(arr2); i++ {
 
 		if i < len(arr1) && i < len(arr2) {
 
 			if arr1[i] < arr2[i] {
-				return true
+				return -1
 			} else if arr1[i] > arr2[i] {
-				return false
+				return +2
 			} else {
 				continue
 			}
@@ -49,15 +49,55 @@ func CompareArr[T OrderedConstraint](arr1 []T, arr2 []T) bool {
 
 		if i < len(arr1) {
 
-			return true
+			return +1
 
 		} else { // if i < len(arr2)
 
-			return false
+			return -1
 
 		}
 
 	}
 
-	return false
+	return 0
+}
+
+func CompareString(a, b string) int {
+	if a == b {
+		return 0
+	}
+	if a < b {
+		return -1
+	}
+	return +1
+}
+
+func CompareInt(a, b int) int {
+	if a == b {
+		return 0
+	}
+	if a < b {
+		return -1
+	}
+	return +1
+}
+
+func CompareInt64(a, b int64) int {
+	if a == b {
+		return 0
+	}
+	if a < b {
+		return -1
+	}
+	return +1
+}
+
+func Compare[T OrderedConstraint](a, b T) int {
+	if a == b {
+		return 0
+	}
+	if a < b {
+		return -1
+	}
+	return +1
 }

langext/maps.go

@@ -1,5 +1,10 @@
 package langext
 
+type MapEntry[T comparable, V any] struct {
+	Key   T
+	Value V
+}
+
 func MapKeyArr[T comparable, V any](v map[T]V) []T {
 	result := make([]T, 0, len(v))
 	for k := range v {
@@ -7,3 +12,54 @@ func MapKeyArr[T comparable, V any](v map[T]V) []T {
 	}
 	return result
 }
+
+func MapValueArr[T comparable, V any](v map[T]V) []V {
+	result := make([]V, 0, len(v))
+	for _, mv := range v {
+		result = append(result, mv)
+	}
+	return result
+}
+
+func ArrToMap[T comparable, V any](a []V, keyfunc func(V) T) map[T]V {
+	result := make(map[T]V, len(a))
+	for _, v := range a {
+		result[keyfunc(v)] = v
+	}
+	return result
+}
+
+func ArrToSet[T comparable](a []T) map[T]bool {
+	result := make(map[T]bool, len(a))
+	for _, v := range a {
+		result[v] = true
+	}
+	return result
+}
+
+func MapToArr[T comparable, V any](v map[T]V) []MapEntry[T, V] {
+	result := make([]MapEntry[T, V], 0, len(v))
+	for mk, mv := range v {
+		result = append(result, MapEntry[T, V]{
+			Key:   mk,
+			Value: mv,
+		})
+	}
+	return result
+}
+
+func CopyMap[K comparable, V any](a map[K]V) map[K]V {
+	result := make(map[K]V, len(a))
+	for k, v := range a {
+		result[k] = v
+	}
+	return result
+}
+
+func ForceMap[K comparable, V any](v map[K]V) map[K]V {
+	if v == nil {
+		return make(map[K]V, 0)
+	} else {
+		return v
+	}
+}

langext/panic.go

@@ -0,0 +1,71 @@
+package langext
+
+type PanicWrappedErr struct {
+	panic any
+}
+
+func (p PanicWrappedErr) Error() string {
+	return "A panic occured"
+}
+
+func (p PanicWrappedErr) ReoveredObj() any {
+	return p.panic
+}
+
+func RunPanicSafe(fn func()) (err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = PanicWrappedErr{panic: rec}
+		}
+	}()
+
+	fn()
+
+	return nil
+}
+
+func RunPanicSafeR1(fn func() error) (err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = PanicWrappedErr{panic: rec}
+		}
+	}()
+
+	return fn()
+}
+
+func RunPanicSafeR2[T1 any](fn func() (T1, error)) (r1 T1, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			r1 = *new(T1)
+			err = PanicWrappedErr{panic: rec}
+		}
+	}()
+
+	return fn()
+}
+
+func RunPanicSafeR3[T1 any, T2 any](fn func() (T1, T2, error)) (r1 T1, r2 T2, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			r1 = *new(T1)
+			r2 = *new(T2)
+			err = PanicWrappedErr{panic: rec}
+		}
+	}()
+
+	return fn()
+}
+
+func RunPanicSafeR4[T1 any, T2 any, T3 any](fn func() (T1, T2, T3, error)) (r1 T1, r2 T2, r3 T3, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			r1 = *new(T1)
+			r2 = *new(T2)
+			r3 = *new(T3)
+			err = PanicWrappedErr{panic: rec}
+		}
+	}()
+
+	return fn()
+}

langext/pointer.go

@@ -4,6 +4,12 @@ import (
 	"reflect"
 )
 
+// PTrue := &true
+var PTrue = Ptr(true)
+
+// PFalse := &false
+var PFalse = Ptr(false)
+
 func Ptr[T any](v T) *T {
 	return &v
 }
@@ -34,3 +40,7 @@ func IsNil(i interface{}) bool {
 	}
 	return false
 }
+
+func PtrEquals[T comparable](v1 *T, v2 *T) bool {
+	return (v1 == nil && v2 == nil) || (v1 != nil && v2 != nil && *v1 == *v2)
+}

langext/sort.go

@@ -0,0 +1,57 @@
+package langext
+
+import "sort"
+
+func Sort[T OrderedConstraint](arr []T) {
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+}
+
+func SortStable[T OrderedConstraint](arr []T) {
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+}
+
+func IsSorted[T OrderedConstraint](arr []T) bool {
+	return sort.SliceIsSorted(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+}
+
+func SortSlice[T any](arr []T, less func(v1, v2 T) bool) {
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+}
+
+func SortSliceStable[T any](arr []T, less func(v1, v2 T) bool) {
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+}
+
+func IsSliceSorted[T any](arr []T, less func(v1, v2 T) bool) bool {
+	return sort.SliceIsSorted(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+}
+
+func SortBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+}
+
+func SortByStable[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+}
+
+func IsSortedBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+}

langext/string.go

@@ -107,3 +107,11 @@ func NumToStringOpt[V IntConstraint](v *V, fallback string) string {
 		return fmt.Sprintf("%d", v)
 	}
 }
+
+func StrRepeat(val string, count int) string {
+	r := ""
+	for i := 0; i < count; i++ {
+		r += val
+	}
+	return r
+}

langext/uuid.go

@@ -41,6 +41,14 @@ func NewHexUUID() (string, error) {
 	return string(dst), nil
 }
 
+func MustHexUUID() string {
+	v, err := NewHexUUID()
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
 func NewUpperHexUUID() (string, error) {
 	uuid, err := NewUUID()
 	if err != nil {
@@ -64,6 +72,14 @@ func NewUpperHexUUID() (string, error) {
 	return strings.ToUpper(string(dst)), nil
 }
 
+func MustUpperHexUUID() string {
+	v, err := NewUpperHexUUID()
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
 func NewRawHexUUID() (string, error) {
 	uuid, err := NewUUID()
 	if err != nil {
@@ -83,6 +99,14 @@ func NewRawHexUUID() (string, error) {
 	return strings.ToUpper(string(dst)), nil
 }
 
+func MustRawHexUUID() string {
+	v, err := NewRawHexUUID()
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
 func NewBracesUUID() (string, error) {
 	uuid, err := NewUUID()
 	if err != nil {
@@ -108,6 +132,14 @@ func NewBracesUUID() (string, error) {
 	return strings.ToUpper(string(dst)), nil
 }
 
+func MustBracesUUID() string {
+	v, err := NewBracesUUID()
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
 func NewParensUUID() (string, error) {
 	uuid, err := NewUUID()
 	if err != nil {
@@ -132,3 +164,11 @@ func NewParensUUID() (string, error) {
 
 	return strings.ToUpper(string(dst)), nil
 }
+
+func MustParensUUID() string {
+	v, err := NewParensUUID()
+	if err != nil {
+		panic(err)
+	}
+	return v
+}

mathext/math.go

@@ -22,6 +22,31 @@ func Max[T langext.OrderedConstraint](v1 T, v2 T) T {
 	}
 }
 
+func Max3[T langext.OrderedConstraint](v1 T, v2 T, v3 T) T {
+	result := v1
+	if v2 > result {
+		result = v2
+	}
+	if v3 > result {
+		result = v3
+	}
+	return result
+}
+
+func Max4[T langext.OrderedConstraint](v1 T, v2 T, v3 T, v4 T) T {
+	result := v1
+	if v2 > result {
+		result = v2
+	}
+	if v3 > result {
+		result = v3
+	}
+	if v4 > result {
+		result = v4
+	}
+	return result
+}
+
 func Min[T langext.OrderedConstraint](v1 T, v2 T) T {
 	if v1 < v2 {
 		return v1
@@ -30,6 +55,31 @@ func Min[T langext.OrderedConstraint](v1 T, v2 T) T {
 	}
 }
 
+func Min3[T langext.OrderedConstraint](v1 T, v2 T, v3 T) T {
+	result := v1
+	if v2 < result {
+		result = v2
+	}
+	if v3 < result {
+		result = v3
+	}
+	return result
+}
+
+func Min4[T langext.OrderedConstraint](v1 T, v2 T, v3 T, v4 T) T {
+	result := v1
+	if v2 < result {
+		result = v2
+	}
+	if v3 < result {
+		result = v3
+	}
+	if v4 < result {
+		result = v4
+	}
+	return result
+}
+
 func Abs[T langext.NumberConstraint](v T) T {
 	if v < 0 {
 		return -v

mongoext/pipeline.go

@@ -0,0 +1,49 @@
+package mongoext
+
+import (
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+// FixTextSearchPipeline moves {$match:{$text:{$search}}} entries to the front of the pipeline (otherwise its an mongo error)
+func FixTextSearchPipeline(pipeline mongo.Pipeline) mongo.Pipeline {
+
+	dget := func(v bson.D, k string) (bson.M, bool) {
+		for _, e := range v {
+			if e.Key == k {
+				if mv, ok := e.Value.(bson.M); ok {
+					return mv, true
+				}
+			}
+		}
+		return nil, false
+	}
+	mget := func(v bson.M, k string) (bson.M, bool) {
+		for ekey, eval := range v {
+			if ekey == k {
+				if mv, ok := eval.(bson.M); ok {
+					return mv, true
+				}
+			}
+		}
+		return nil, false
+	}
+
+	result := make([]bson.D, 0, len(pipeline))
+
+	for _, entry := range pipeline {
+
+		if v0, ok := dget(entry, "$match"); ok {
+			if v1, ok := mget(v0, "$text"); ok {
+				if _, ok := v1["$search"]; ok {
+					result = append([]bson.D{entry}, result...)
+					continue
+				}
+			}
+		}
+
+		result = append(result, entry)
+	}
+
+	return result
+}