v0.0.644 exerr.DeregisterListener

(cherry picked from commit c0443af63b)

.claude/settings.json

@@ -0,0 +1,44 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(make:*)",
+
+      "Bash(mkdir:*)",
+
+      "Bash(go build:*)",
+      "Bash(go test:*)",
+      "Bash(go get:*)",
+      "Bash(go mod:*)",
+      "Bash(go clean:*)",
+      "Bash(go doc:*)",
+
+      "Bash(grep:*)",
+      "Bash(find:*)",
+      "Bash(rg:*)",
+      "Bash(base64:*)",
+      "Bash(sed:*)",
+      "Bash(ls:*)",
+
+      "Bash(curl:*)",
+
+      "Bash(timeout 60s go test -v -count=1 ./...)",
+      "Bash(timeout 60s go test -v -count=1 ./tests/integration/...)",
+      "Bash(timeout 60s go test:*)",
+      "Bash(timeout 300 make test)",
+      "Bash(timeout 30s go test ./tests/integration -run:*)",
+
+      "Bash(done)",
+      "Bash(awk:*)",
+
+      "WebFetch(domain:platform.openai.com)"
+    ],
+    "deny": [
+    ],
+    "defaultMode": "acceptEdits"
+  },
+  "env": {
+    "CLAUDE_CODE_ENABLE_TELEMETRY": "0",
+    "DISABLE_ERROR_REPORTING": "1",
+    "DISABLE_TELEMETRY": "1"
+  }
+}

.gitea/workflows/tests.yml

@@ -0,0 +1,42 @@
+
+# https://docs.gitea.com/next/usage/actions/quickstart
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
+
+name: Build Docker and Deploy
+run-name: "[test]: ${{ github.event.head_commit.message }}"
+
+on:
+  push:
+    branches:
+      - '*'
+      - '**'
+
+
+jobs:
+  run_tests:
+    name: Run goext test-suite  
+    runs-on: bfb-cicd-latest
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Setup go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: '${{ gitea.workspace }}/go.mod'
+
+      - name: Setup packages
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: curl python3
+          version: 1.0
+
+      - name: go version
+        run: go version
+
+      - name: Run tests
+        run: cd "${{ gitea.workspace }}" && make test
+
+

.gitignore

@@ -1,4 +1,6 @@
 
+.claude-queue
+
 ##########################################################################
 
 .idea/**/workspace.xml

.idea/.gitignore

@@ -6,3 +6,5 @@
 # Datasource local storage ignored files
 /dataSources/
 /dataSources.local.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

.idea/copilot.data.migration.agent.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AgentMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

.idea/copilot.data.migration.ask.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AskMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

.idea/copilot.data.migration.ask2agent.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Ask2AgentMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

.idea/copilot.data.migration.edit.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="EditMigrationStateService">
+    <option name="migrationStatus" value="COMPLETED" />
+  </component>
+</project>

.idea/go.imports.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoImports">
+    <option name="excludedPackages">
+      <array>
+        <option value="github.com/pkg/errors" />
+        <option value="golang.org/x/net/context" />
+      </array>
+    </option>
+  </component>
+</project>

.idea/goext.iml

@@ -1,6 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="WEB_MODULE" version="4">
-  <component name="Go" enabled="true" />
+  <component name="Go" enabled="true">
+    <buildTags>
+      <option name="goVersion" value="1.19" />
+    </buildTags>
+  </component>
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
     <orderEntry type="inheritedJdk" />

.idea/golinter.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoLinterSettings">
+    <option name="checkGoLinterExe" value="false" />
+  </component>
+</project>

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +1,7 @@
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="GoMixedReceiverTypes" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
     <inspection_tool class="LanguageDetectionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
     <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
       <option name="processCode" value="true" />

.idea/sqldialects.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/sq/sq_test.go" dialect="SQLite" />
+  </component>
+</project>

Makefile

@@ -3,7 +3,15 @@ run:
 	echo "This is a library - can't be run" && false
 
 test:
-	go test ./...
+	# go test ./...
+	which gotestsum || go install gotest.tools/gotestsum@latest
+	gotestsum --format "testname" -- -tags="timetzdata sqlite_fts5 sqlite_foreign_keys" "./..."
+
+test-in-docker:
+	tag="goext_temp_test_image:$(shell uuidgen | tr -d '-')";        \
+	docker build --tag $$tag . -f .gitea/workflows/Dockerfile_tests; \
+	docker run --rm $$tag;                                           \
+	docker rmi $$tag
 
 version:
 	_data/version.sh

README.md

@@ -5,4 +5,111 @@ A collection of general & useful library methods
 
 This should not have any heavy dependencies (gin, mongo, etc) and add missing basic language features...
 
-Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
+Potentially needs `export GOPRIVATE="git.blackforestbytes.com"`
+
+
+## Packages:
+
+| Name        | Maintainer | Description                                                                                                   |
+|-------------|------------|---------------------------------------------------------------------------------------------------------------|
+| langext     | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
+| mathext     | Mike       | Utility/Helper functions for math                                                                             |
+| cryptext    | Mike       | Utility/Helper functions for encryption                                                                       |
+| syncext     | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
+| dataext     | Mike       | Various useful data structures                                                                                |
+| zipext      | Mike       | Utility for zip/gzip/tar etc                                                                                  |
+| reflectext  | Mike       | Utility for golang reflection                                                                                 |
+| fsext       | Mike       | Utility for filesytem access                                                                                  |
+| ctxext      | Mike       | Utility for context.Context                                                                                   |
+|             |            |                                                                                                               |
+| mongoext    | Mike       | Utility/Helper functions for mongodb (kinda abandoned)                                                        |
+| cursortoken | Mike       | MongoDB cursortoken implementation                                                                            |
+| pagination  | Mike       | Pagination implementation                                                                                     |
+|             |            |                                                                                                               |
+| ginext      | Mike       | gin wrapper                                                                                                   |
+| wsw         | Mike       | websocket wrapper                                                                                             |
+|             |            |                                                                                                               |
+| totpext     | Mike       | Implementation of TOTP (2-Factor-Auth)                                                                        |
+| termext     | Mike       | Utilities for terminals (mostly color output)                                                                 |
+| confext     | Mike       | Parses environment configuration into structs                                                                 |
+| cmdext      | Mike       | Runner for external commands/processes                                                                        |
+| excelext    | Mike       | Build Excel files                                                                                             |
+|             |            |                                                                                                               |
+| sq          | Mike       | Utility functions for sql based databases (primarily sqlite)                                                  |
+| tst         | Mike       | Utility functions for unit tests                                                                              |
+|             |            |                                                                                                               |
+| rfctime     | Mike       | Classes for time seriallization, with different marshallign method for mongo and json                         |
+| gojson      | Mike       | Same interface for marshalling/unmarshalling as go/json, except with proper serialization of null arrays/maps |
+|             |            |                                                                                                               |
+| bfcodegen   | Mike       | Various codegen tools (run via go generate)                                                                   |
+|             |            |                                                                                                               |
+| rext        | Mike       | Regex Wrapper, wraps regexp with a better interface                                                           |
+| wmo         | Mike       | Mongo Wrapper, wraps mongodb with a better interface                                                          |
+|             |            |                                                                                                               |
+| scn         | Mike       | SimpleCloudNotifier                                                                                           |
+|             |            |                                                                                                               |
+
+
+
+## Usage:
+
+### exerr
+
+ - see **mongoext/builder.go** for full info
+
+Short summary:
+ - An better error package with metadata, listener, api-output and error-traces
+ - Initialize with `exerr.Init()`
+ - *Never* return `err` direct, always use exerr.Wrap(err, "...") - add metadata where applicable
+ - at the end either Print(), Fatal() or Output() your error (print = stdout, fatal = panic, output = json API response)
+ - You can add listeners with exerr.RegisterListener(), and save the full errors to a db or smth
+
+### wmo
+
+ - A typed wrapper around the official mongo-go-driver
+ - Use `wmo.W[...](...)` to wrap the collections and type-ify them
+ - The new collections have all the usual methods, but types
+ - Also they have List() and Paginate() methods for paginated listings (witehr with a cursortoken or page/limit)
+ - Register additional hooks with `WithDecodeFunc`, `WithUnmarshalHook`, `WithMarshalHook`, `WithModifyingPipeline`, `WithModifyingPipelineFunc`
+ - List(), Paginate(), etc support filter interfaces
+   - Rule(s) of thumb: 
+     - filter the results in the filter interface
+     - sort the results in the sort function of the filter interface
+     - add joins ($lookup's) in the `WithModifyingPipelineFunc`/`WithModifyingPipeline`
+
+#### ginext
+
+ - A wrapper around gin-gonic/gin
+ - create the gin engine with `ginext.NewEngine`
+ - Add routes with `engine.Routes()...`
+   - `.Use(..)` adds a middleware
+   - `.Group(..)` adds a group
+   - `.Get().Handle(..)` adds a handler
+ - Handler return values (in contract to ginext) - values implement the `ginext.HTTPResponse` interface
+ - Every handler starts with something like:
+```go 
+func (handler Handler) CommunityMetricsValues(pctx ginext.PreContext) ginext.HTTPResponse {
+    type communityURI struct {
+        Version     string             `uri:"version"`
+        CommunityID models.CommunityID `uri:"cid"`
+    }
+    type body struct {
+        UserID  models.UserID  `json:"userID"`
+        EventID models.EventID `json:"eventID"`
+    }
+
+    var u uri
+    var b body
+    ctx, gctx, httpErr := pctx.URI(&u).Body(&b).Start() // can have more unmarshaller, like header, form, etc
+    if httpErr != nil {
+    	return *httpErr
+    }
+    defer ctx.Cancel()
+
+    // do stuff
+}
+```
+
+#### sq
+
+ - TODO (like mongoext for sqlite/sql databases)

TODO.md

@@ -0,0 +1,9 @@
+
+
+ - cronext
+
+ - rfctime.HMSTimeOnly
+ - rfctime.NanoTimeOnly
+
+ - remove sqlx dependency from sq  (unmaintained, and mostly superseeded by our own stuff?)
+ - Move DBLogger and DBPreprocessor to sq

_data/version.sh

@@ -7,6 +7,29 @@ set -o pipefail  # Return value of a pipeline is the value of the last (rightmos
 IFS=$'\n\t'      # Set $IFS to only newline and tab.
 
 
+function black() { echo -e "\x1B[30m $1 \x1B[0m"; }
+function red()   { echo -e "\x1B[31m $1 \x1B[0m"; }
+function green() { echo -e "\x1B[32m $1 \x1B[0m"; }
+function yellow(){ echo -e "\x1B[33m $1 \x1B[0m"; }
+function blue()  { echo -e "\x1B[34m $1 \x1B[0m"; }
+function purple(){ echo -e "\x1B[35m $1 \x1B[0m"; }
+function cyan()  { echo -e "\x1B[36m $1 \x1B[0m"; }
+function white() { echo -e "\x1B[37m $1 \x1B[0m"; }
+
+if [ "$( git rev-parse --abbrev-ref HEAD )" != "master" ]; then
+  >&2 red "[ERROR] Can only create versions of <master>"
+  exit 1
+fi
+
+echo ""
+echo -n "Insert optional commit message: "
+read commitMessage
+echo ""
+
+git pull --ff
+
+go get -u ./...
+
 curr_vers=$(git describe --tags --abbrev=0 | sed 's/v//g')
 
 next_ver=$(echo "$curr_vers" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{if(length($NF+1)>length($NF))$(NF-1)++; $NF=sprintf("%0*d", length($NF), ($NF+1)%(10^length($NF))); print}')
@@ -16,9 +39,22 @@ echo "> Current Version: ${curr_vers}"
 echo "> Next    Version: ${next_ver}"
 echo ""
 
+printf "package goext\n\nconst GoextVersion = \"%s\"\n\nconst GoextVersionTimestamp = \"%s\"\n" "${next_ver}" "$( date +"%Y-%m-%dT%H:%M:%S%z" )" > "goextVersion.go"
+
 git add --verbose .
 
-git commit -a -m "v${next_ver}"
+msg="v${next_ver}"
+
+if [[ "$commitMessage" != "" ]]; then
+  msg="${msg} ${commitMessage}"
+fi
+
+
+if [ $# -gt 0 ]; then
+  msg="$1"
+fi
+
+git commit -a -m "${msg}"
 
 git tag "v${next_ver}"
 

bfcodegen/csid-generate.go

@@ -0,0 +1,198 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/cryptext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/rext"
+	"go/format"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type CSIDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+	Prefix       string
+}
+
+type CSIDGenOptions struct {
+	DebugOutput *bool
+}
+
+var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
+
+var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed csid-generate.template
+var templateCSIDGenerateText string
+
+func GenerateCharsetIDSpecs(sourceDir string, destFile string, opt CSIDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexCSIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	var newChecksumStr strings.Builder
+	newChecksumStr.WriteString(goext.GoextVersion)
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr.WriteString("\n" + f.Name() + "\t" + cryptext.BytesSha256(content))
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr.String()))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[CSIDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[CSIDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
+		if err != nil {
+			return err
+		}
+
+		if debugOutput {
+			fmt.Printf("\n")
+		}
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	fdata, err := format.Source([]byte(fmtCSIDOutput(newChecksum, allIDs, pkgname)))
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(destFile, fdata, 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processCSIDFile(basedir string, fn string, debugOutput bool) ([]CSIDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexCSIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexCSIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := CSIDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+				Prefix:       match.GroupByName("prefix").Value(),
+			}
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtCSIDOutput(cs string, ids []CSIDDef, pkgname string) string {
+	templ := template.Must(template.New("csid-generate").Parse(templateCSIDGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/csid-generate.template

@@ -0,0 +1,194 @@
+// Code generated by csid-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "crypto/rand"
+import "crypto/sha256"
+import "fmt"
+import "github.com/go-playground/validator/v10"
+import "github.com/rs/zerolog/log"
+import "git.blackforestbytes.com/BlackForestBytes/goext/exerr"
+import "git.blackforestbytes.com/BlackForestBytes/goext/langext"
+import "git.blackforestbytes.com/BlackForestBytes/goext/rext"
+import "math/big"
+import "reflect"
+import "regexp"
+import "strings"
+
+const ChecksumCharsetIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+const idlen = 24
+
+const checklen = 1
+
+const idCharset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+const idCharsetLen = len(idCharset)
+
+var charSetReverseMap = generateCharsetMap()
+
+const ({{range .IDs}}
+	prefix{{.Name}} = "{{.Prefix}}" {{end}}
+)
+
+var ({{range .IDs}}
+	regex{{.Name}} = generateRegex(prefix{{.Name}}) {{end}}
+)
+
+func generateRegex(prefix string) rext.Regex {
+	return rext.W(regexp.MustCompile(fmt.Sprintf("^%s[%s]{%d}[%s]{%d}$", prefix, idCharset, idlen-len(prefix)-checklen, idCharset, checklen)))
+}
+
+func generateCharsetMap() []int {
+	result := make([]int, 128)
+	for i := 0; i < len(result); i++ {
+		result[i] = -1
+	}
+	for idx, chr := range idCharset {
+		result[int(chr)] = idx
+	}
+	return result
+}
+
+func generateID(prefix string) string {
+	k := ""
+	csMax := big.NewInt(int64(idCharsetLen))
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		v, err := rand.Int(rand.Reader, csMax)
+		if err != nil {
+			panic(err)
+		}
+		v64 := v.Int64()
+		k += string(idCharset[v64])
+		checksum = (checksum + int(v64)) % (idCharsetLen)
+	}
+	checkstr := string(idCharset[checksum%idCharsetLen])
+	return prefix + k + checkstr
+}
+
+func generateIDFromSeed(prefix string, seed string) string {
+	h := sha256.New()
+
+	iddata := ""
+	for len(iddata) < idlen-len(prefix)-checklen {
+		h.Write([]byte(seed))
+		bs := h.Sum(nil)
+		iddata += langext.NewAnyBaseConverter(idCharset).Encode(bs)
+	}
+
+	checksum := 0
+	for i := 0; i < idlen-len(prefix)-checklen; i++ {
+		ichr := int(iddata[i])
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	return prefix + iddata[:(idlen-len(prefix)-checklen)] + checkstr
+}
+
+func validateID(prefix string, value string) error {
+	if len(value) != idlen {
+		return exerr.New(exerr.TypeInvalidCSID, "id has the wrong length").Str("value", value).Build()
+	}
+
+	if !strings.HasPrefix(value, prefix) {
+		return exerr.New(exerr.TypeInvalidCSID, "id is missing the correct prefix").Str("value", value).Str("prefix", prefix).Build()
+	}
+
+	checksum := 0
+	for i := len(prefix); i < len(value)-checklen; i++ {
+		ichr := int(value[i])
+		if ichr < 0 || ichr >= len(charSetReverseMap) || charSetReverseMap[ichr] == -1 {
+			return exerr.New(exerr.TypeInvalidCSID, "id contains invalid characters").Str("value", value).Build()
+		}
+		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)
+	}
+
+	checkstr := string(idCharset[checksum%idCharsetLen])
+
+	if !strings.HasSuffix(value, checkstr) {
+		return exerr.New(exerr.TypeInvalidCSID, "id checkstring is invalid").Str("value", value).Str("checkstr", checkstr).Build()
+	}
+
+	return nil
+}
+
+func getRawData(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[len(prefix) : idlen-checklen]
+}
+
+func getCheckString(prefix string, value string) string {
+	if len(value) != idlen {
+		return ""
+	}
+	return value[idlen-checklen:]
+}
+
+func ValidateEntityID(vfl validator.FieldLevel) bool {
+	if !vfl.Field().CanInterface() {
+		log.Error().Msgf("Failed to validate EntityID (cannot interface ?!?)")
+		return false
+	}
+
+	ifvalue := vfl.Field().Interface()
+
+	if value1, ok := ifvalue.(EntityID); ok {
+
+		if vfl.Field().Type().Kind() == reflect.Pointer && langext.IsNil(value1) {
+			return true
+		}
+
+		if err := value1.Valid(); err != nil {
+			log.Debug().Msgf("Failed to validate EntityID '%s' (%s)", value1.String(), err.Error())
+			return false
+		} else {
+			return true
+		}
+
+	} else {
+		log.Error().Msgf("Failed to validate EntityID (wrong type: %T)", ifvalue)
+		return false
+	}
+}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(generateID(prefix{{.Name}}))
+}
+
+func (id {{.Name}}) Valid() error {
+	return validateID(prefix{{.Name}}, string(id))
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) Prefix() string {
+	return prefix{{.Name}}
+}
+
+func (id {{.Name}}) Raw() string {
+	return getRawData(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) CheckString() string {
+	return getCheckString(prefix{{.Name}}, string(id))
+}
+
+func (id {{.Name}}) IsZero() bool {
+	return id == ""
+}
+
+func (id {{.Name}}) Regex() rext.Regex {
+	return regex{{.Name}}
+}
+
+{{end}}

bfcodegen/csid-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext/cmdext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var CSIDExampleModels1 []byte
+
+func TestGenerateCSIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, CSIDExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/csid_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/enum-generate.go

@@ -0,0 +1,374 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/cryptext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/rext"
+	"go/format"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type EnumDefVal struct {
+	VarName     string
+	Value       string
+	Description *string
+	Data        *map[string]any
+	RawComment  *string
+}
+
+type EnumDef struct {
+	File         string
+	FileRelative string
+	EnumTypeName string
+	Type         string
+	Values       []EnumDefVal
+}
+
+type EnumGenOptions struct {
+	DebugOutput *bool
+	GoFormat    *bool
+}
+
+var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
+
+var rexEnumValueDef = rext.W(regexp.MustCompile(`^\s*(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*=\s*(?P<value>("[/@A-Za-z0-9_:\s\-.]*"|[0-9]+))\s*(//(?P<comm>.*))?.*$`))
+
+var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed enum-generate.template
+var templateEnumGenerateText string
+
+func GenerateEnumSpecs(sourceDir string, destFile string, opt EnumGenOptions) error {
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexEnumChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, langext.Coalesce(opt.GoFormat, true), langext.Coalesce(opt.DebugOutput, false))
+	if err != nil {
+		return err
+	}
+
+	if !changed {
+		return nil
+	}
+
+	err = os.WriteFile(destFile, []byte(gocode), 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool, debugOutput bool) (string, string, bool, error) {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return "", "", false, err
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	var newChecksumStr strings.Builder
+	newChecksumStr.WriteString(goext.GoextVersion)
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return "", "", false, err
+		}
+		newChecksumStr.WriteString("\n" + f.Name() + "\t" + cryptext.BytesSha256(content))
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr.String()))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[EnumGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[EnumGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return "", oldChecksum, false, nil
+	}
+
+	allEnums := make([]EnumDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
+		if err != nil {
+			return "", "", false, err
+		}
+
+		if debugOutput {
+			fmt.Printf("\n")
+		}
+
+		allEnums = append(allEnums, fileEnums...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return "", "", false, errors.New("no package name found in any file")
+	}
+
+	rdata := fmtEnumOutput(newChecksum, allEnums, pkgname)
+
+	if !gofmt {
+		return rdata, newChecksum, true, nil
+	}
+
+	fdata, err := format.Source([]byte(rdata))
+	if err != nil {
+		return "", "", false, err
+	}
+
+	return string(fdata), newChecksum, true, nil
+}
+
+func processEnumFile(basedir string, fn string, debugOutput bool) ([]EnumDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	enums := make([]EnumDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexEnumPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexEnumDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := EnumDef{
+				File:         fn,
+				FileRelative: rfp,
+				EnumTypeName: match.GroupByName("name").Value(),
+				Type:         match.GroupByName("type").Value(),
+				Values:       make([]EnumDefVal, 0),
+			}
+			enums = append(enums, def)
+
+			if debugOutput {
+				fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+			}
+		}
+
+		if match, ok := rexEnumValueDef.MatchFirst(line); ok {
+			typename := match.GroupByName("type").Value()
+
+			comment := match.GroupByNameOrEmpty("comm").ValueOrNil()
+			var descr *string = nil
+			var data *map[string]any = nil
+			if comment != nil {
+				comment = new(strings.TrimSpace(*comment))
+				if strings.HasPrefix(*comment, "{") {
+					if v, ok := tryParseDataComment(*comment); ok {
+						data = &v
+						if anyDataDescr, ok := v["description"]; ok {
+							if dataDescr, ok := anyDataDescr.(string); ok {
+								descr = &dataDescr
+							}
+						}
+					} else {
+						descr = comment
+					}
+				} else {
+					descr = comment
+				}
+			}
+
+			def := EnumDefVal{
+				VarName:     match.GroupByName("name").Value(),
+				Value:       match.GroupByName("value").Value(),
+				RawComment:  comment,
+				Description: descr,
+				Data:        data,
+			}
+
+			found := false
+			for i, v := range enums {
+				if v.EnumTypeName == typename {
+					enums[i].Values = append(enums[i].Values, def)
+					found = true
+
+					if debugOutput {
+						if def.Description != nil {
+							fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
+						} else {
+							fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+						}
+					}
+					break
+				}
+			}
+			if !found {
+				if debugOutput {
+					fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				}
+			}
+		}
+	}
+
+	return enums, pkgname, nil
+}
+
+func tryParseDataComment(s string) (map[string]any, bool) {
+
+	r := make(map[string]any)
+
+	err := json.Unmarshal([]byte(s), &r)
+	if err != nil {
+		return nil, false
+	}
+
+	for _, v := range r {
+
+		rv := reflect.ValueOf(v)
+
+		if rv.Kind() == reflect.Pointer && rv.IsNil() {
+			continue
+		}
+		if rv.Kind() == reflect.Bool {
+			continue
+		}
+		if rv.Kind() == reflect.String {
+			continue
+		}
+		if rv.Kind() == reflect.Int64 {
+			continue
+		}
+		if rv.Kind() == reflect.Float64 {
+			continue
+		}
+
+		return nil, false
+	}
+
+	return r, true
+}
+
+func fmtEnumOutput(cs string, enums []EnumDef, pkgname string) string {
+
+	templ := template.New("enum-generate")
+
+	templ = templ.Funcs(template.FuncMap{
+		"boolToStr": func(b bool) string { return langext.Conditional(b, "true", "false") },
+		"deref":     func(v *string) string { return *v },
+		"trimSpace": func(str string) string { return strings.TrimSpace(str) },
+		"hasStr":    func(v EnumDef) bool { return v.Type == "string" },
+		"hasDescr": func(v EnumDef) bool {
+			return langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Description != nil })
+		},
+		"hasData": func(v EnumDef) bool {
+			return len(v.Values) > 0 && langext.ArrAll(v.Values, func(val EnumDefVal) bool { return val.Data != nil })
+		},
+		"gostr": func(v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"goobj": func(name string, v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"godatakey": func(v string) string {
+			return strings.ToUpper(v[0:1]) + v[1:]
+		},
+		"godatavalue": func(v any) string {
+			return fmt.Sprintf("%#+v", v)
+		},
+		"godatatype": func(v any) string {
+			return fmt.Sprintf("%T", v)
+		},
+		"mapindex": func(v map[string]any, k string) any {
+			return v[k]
+		},
+		"generalDataKeys": func(v EnumDef) map[string]string {
+			r0 := make(map[string]int)
+
+			for _, eval := range v.Values {
+				for k := range *eval.Data {
+					if ctr, ok := r0[k]; ok {
+						r0[k] = ctr + 1
+					} else {
+						r0[k] = 1
+					}
+				}
+			}
+
+			r1 := langext.MapToArr(r0)
+			r2 := langext.ArrFilter(r1, func(p langext.MapEntry[string, int]) bool { return p.Value == len(v.Values) })
+			r3 := langext.ArrMap(r2, func(p langext.MapEntry[string, int]) string { return p.Key })
+			r4 := langext.ArrToKVMap(r3, func(p string) string { return p }, func(p string) string { return fmt.Sprintf("%T", (*v.Values[0].Data)[p]) })
+
+			return r4
+		},
+	})
+
+	templ = template.Must(templ.Parse(templateEnumGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"Enums":        enums,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/enum-generate.template

@@ -0,0 +1,177 @@
+// Code generated by enum-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "git.blackforestbytes.com/BlackForestBytes/goext/langext"
+import "git.blackforestbytes.com/BlackForestBytes/goext/enums"
+
+const ChecksumEnumGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{ $pkgname  := .PkgName }}
+
+{{range .Enums}}
+
+{{ $hasStr   := ( . | hasStr   ) }}
+{{ $hasDescr := ( . | hasDescr ) }}
+{{ $hasData  := ( . | hasData  ) }}
+
+// ================================ {{.EnumTypeName}} ================================
+//
+// File:       {{.FileRelative}}
+// StringEnum: {{$hasStr   | boolToStr}}
+// DescrEnum:  {{$hasDescr | boolToStr}}
+// DataEnum:   {{$hasData  | boolToStr}}
+//
+
+{{ $typename := .EnumTypeName }}
+{{ $enumdef  := . }}
+
+var __{{.EnumTypeName}}Values = []{{.EnumTypeName}}{ {{range .Values}}
+	{{.VarName}}, {{end}}
+}
+
+{{if $hasDescr}}
+var __{{.EnumTypeName}}Descriptions = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: {{.Description | deref | trimSpace | gostr}}, {{end}}
+}
+{{end}}
+
+{{if $hasData}}
+type {{ .EnumTypeName }}Data struct { {{ range $datakey, $datatype := ($enumdef | generalDataKeys)  }}
+    {{ $datakey | godatakey }} {{ $datatype }} `json:"{{ $datakey }}"` {{ end }}
+}
+
+var __{{.EnumTypeName}}Data = map[{{.EnumTypeName}}]{{.EnumTypeName}}Data{ {{range .Values}} {{ $enumvalue := . }}
+	{{.VarName}}: {{ $typename }}Data{ {{ range $datakey, $datatype := $enumdef | generalDataKeys  }}
+	    {{ $datakey | godatakey }}: {{ (mapindex $enumvalue.Data $datakey) | godatavalue }}, {{ end }}
+	}, {{end}}
+}
+{{end}}
+
+var __{{.EnumTypeName}}Varnames = map[{{.EnumTypeName}}]string{ {{range .Values}}
+	{{.VarName}}: "{{.VarName}}", {{end}}
+}
+
+func (e {{.EnumTypeName}}) Valid() bool {
+	return langext.InArray(e, __{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func (e {{.EnumTypeName}}) ValuesAny() []any {
+	return langext.ArrCastToAny(__{{.EnumTypeName}}Values)
+}
+
+func (e {{.EnumTypeName}}) ValuesMeta() []enums.EnumMetaValue {
+	return {{.EnumTypeName}}ValuesMeta()
+}
+
+{{if $hasStr}}
+func (e {{.EnumTypeName}}) String() string {
+	return string(e)
+}
+{{end}}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) Description() string {
+	if d, ok := __{{.EnumTypeName}}Descriptions[e]; ok {
+		return d
+	}
+	return ""
+}
+{{end}}
+
+{{if $hasData}}
+func (e {{.EnumTypeName}}) Data() {{.EnumTypeName}}Data {
+	if d, ok := __{{.EnumTypeName}}Data[e]; ok {
+		return d
+	}
+	return {{.EnumTypeName}}Data{}
+}
+{{end}}
+
+func (e {{.EnumTypeName}}) VarName() string {
+	if d, ok := __{{.EnumTypeName}}Varnames[e]; ok {
+		return d
+	}
+	return ""
+}
+
+func (e {{.EnumTypeName}}) TypeName() string {
+	return "{{$typename}}"
+}
+
+func (e {{.EnumTypeName}}) PackageName() string {
+	return "{{$pkgname }}"
+}
+
+func (e {{.EnumTypeName}}) Meta() enums.EnumMetaValue {
+    {{if $hasDescr}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: new(e.Description())} {{else}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil} {{end}}
+}
+
+{{if $hasDescr}}
+func (e {{.EnumTypeName}}) DescriptionMeta() enums.EnumDescriptionMetaValue {
+    return enums.EnumDescriptionMetaValue{VarName: e.VarName(), Value: e, Description: e.Description()}
+}
+{{end}}
+
+{{if $hasData}}
+func (e {{.EnumTypeName}}) DataMeta() enums.EnumDataMetaValue {
+    return enums.EnumDataMetaValue{
+        VarName: e.VarName(),
+        Value: e,
+        {{if $hasDescr}} Description: new(e.Description()), {{else}} Description: nil, {{end}}
+        Data: map[string]any{
+            {{ range $datakey, $datatype := $enumdef | generalDataKeys  }} "{{ $datakey }}": e.Data().{{ $datakey | godatakey }},
+            {{ end }}
+        },
+    }
+}
+{{end}}
+
+func Parse{{.EnumTypeName}}(vv string) ({{.EnumTypeName}}, bool) {
+	for _, ev := range __{{.EnumTypeName}}Values {
+		if string(ev) == vv {
+			return ev, true
+		}
+	}
+	return "", false
+}
+
+func {{.EnumTypeName}}Values() []{{.EnumTypeName}} {
+	return __{{.EnumTypeName}}Values
+}
+
+func {{.EnumTypeName}}ValuesMeta() []enums.EnumMetaValue {
+	return []enums.EnumMetaValue{ {{range .Values}}
+            {{.VarName}}.Meta(), {{end}}
+	}
+}
+
+{{if $hasData}}
+func {{.EnumTypeName}}ValuesDataMeta() []enums.EnumDataMetaValue {
+	return []enums.EnumDataMetaValue{ {{range .Values}}
+            {{.VarName}}.DataMeta(), {{end}}
+	}
+}
+{{end}}
+
+{{if $hasDescr}}
+func {{.EnumTypeName}}ValuesDescriptionMeta() []enums.EnumDescriptionMetaValue {
+	return []enums.EnumDescriptionMetaValue{ {{range .Values}}
+            {{.VarName}}.DescriptionMeta(), {{end}}
+	}
+}
+{{end}}
+
+{{end}}
+
+// ================================ ================= ================================
+
+func AllPackageEnums() []enums.Enum {
+    return []enums.Enum{ {{range .Enums}}
+        {{ if gt (len .Values) 0 }} {{  $v := index .Values 0 }} {{ $v.VarName}}, {{end}} // {{ .EnumTypeName }} {{end}}
+    }
+}

bfcodegen/enum-generate_test.go

@@ -0,0 +1,91 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext/cmdext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var EnumExampleModels1 []byte
+
+//go:embed _test_example_2.tgz
+var EnumExampleModels2 []byte
+
+func TestGenerateEnumSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, EnumExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
+	tst.AssertNoErr(t, err)
+
+	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, cs1, cs2)
+	tst.AssertEqual(t, s1, s2)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(s1)
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}
+
+func TestGenerateEnumSpecsData(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, EnumExampleModels2, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true, true)
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(s1)
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/id-generate.go

@@ -0,0 +1,199 @@
+package bfcodegen
+
+import (
+	"bytes"
+	_ "embed"
+	"errors"
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/cryptext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/rext"
+	"go/format"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+type IDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+}
+
+type IDGenOptions struct {
+	DebugOutput *bool
+}
+
+var rexIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@id:type).*$`))
+
+var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+//go:embed id-generate.template
+var templateIDGenerateText string
+
+func GenerateIDSpecs(sourceDir string, destFile string, opt IDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	var newChecksumStr strings.Builder
+	newChecksumStr.WriteString(goext.GoextVersion)
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr.WriteString("\n" + f.Name() + "\t" + cryptext.BytesSha256(content))
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr.String()))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[IDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[IDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]IDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
+		if err != nil {
+			return err
+		}
+
+		if debugOutput {
+			fmt.Printf("\n")
+		}
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	fdata, err := format.Source([]byte(fmtIDOutput(newChecksum, allIDs, pkgname)))
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(destFile, fdata, 0o755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processIDFile(basedir string, fn string, debugOutput bool) ([]IDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]IDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := IDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+			}
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtIDOutput(cs string, ids []IDDef, pkgname string) string {
+	templ := template.Must(template.New("id-generate").Parse(templateIDGenerateText))
+
+	buffer := bytes.Buffer{}
+
+	anyDef := langext.ArrFirstOrNil(ids, func(def IDDef) bool { return def.Name == "AnyID" || def.Name == "AnyId" })
+
+	err := templ.Execute(&buffer, langext.H{
+		"PkgName":      pkgname,
+		"Checksum":     cs,
+		"GoextVersion": goext.GoextVersion,
+		"IDs":          ids,
+		"AnyDef":       anyDef,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return buffer.String()
+}

bfcodegen/id-generate.template

@@ -0,0 +1,58 @@
+// Code generated by id-generate.go DO NOT EDIT.
+
+package {{.PkgName}}
+
+import "go.mongodb.org/mongo-driver/v2/bson"
+import "git.blackforestbytes.com/BlackForestBytes/goext/exerr"
+import "git.blackforestbytes.com/BlackForestBytes/goext/wmo"
+
+const ChecksumIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
+
+{{range .IDs}}
+
+// ================================ {{.Name}} ({{.FileRelative}}) ================================
+
+func (i {{.Name}}) MarshalBSONValue() (byte, []byte, error) {
+	if objId, err := bson.ObjectIDFromHex(string(i)); err == nil {
+		tp, data, err := bson.MarshalValue(objId)
+		return byte(tp), data, err
+	} else {
+		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, "Failed to marshal {{.Name}}("+i.String()+") to ObjectId").Str("value", string(i)).Type("type", i).Build()
+	}
+}
+
+func (i {{.Name}}) String() string {
+	return string(i)
+}
+
+func (i {{.Name}}) ObjID() (bson.ObjectID, error) {
+	return bson.ObjectIDFromHex(string(i))
+}
+
+func (i {{.Name}}) Valid() bool {
+	_, err := bson.ObjectIDFromHex(string(i))
+	return err == nil
+}
+
+{{if ne $.AnyDef nil}}
+func (i {{.Name}}) AsAny() {{$.AnyDef.Name}} {
+	return {{$.AnyDef.Name}}(i)
+}
+
+func (i {{.Name}}) AsAnyPtr() *{{$.AnyDef.Name}} {
+    v := {{$.AnyDef.Name}}(i)
+	return &v
+}
+{{end}}
+
+func (i {{.Name}}) IsZero() bool {
+	return i == ""
+}
+
+func New{{.Name}}() {{.Name}} {
+	return {{.Name}}(bson.NewObjectID().Hex())
+}
+
+var _ wmo.MongoEntityID = (*{{.Name}})(nil)
+
+{{end}}

bfcodegen/id-generate_test.go

@@ -0,0 +1,52 @@
+package bfcodegen
+
+import (
+	_ "embed"
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext/cmdext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+//go:embed _test_example_1.tgz
+var IDExampleModels1 []byte
+
+func TestGenerateIDSpecs(t *testing.T) {
+
+	tmpFile := filepath.Join(t.TempDir(), langext.MustHexUUID()+".tgz")
+
+	tmpDir := filepath.Join(t.TempDir(), langext.MustHexUUID())
+
+	err := os.WriteFile(tmpFile, IDExampleModels1, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.Remove(tmpFile) })
+
+	err = os.Mkdir(tmpDir, 0o777)
+	tst.AssertNoErr(t, err)
+
+	t.Cleanup(func() { _ = os.RemoveAll(tmpFile) })
+
+	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", IDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", IDGenOptions{DebugOutput: langext.PTrue})
+	tst.AssertNoErr(t, err)
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+	fmt.Println("=====================================================================================================")
+	fmt.Println(string(tst.Must(os.ReadFile(tmpDir + "/id_gen.go"))(t)))
+	fmt.Println("=====================================================================================================")
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}

bfcodegen/unit_csid_test.go

@@ -0,0 +1,160 @@
+package bfcodegen
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestProcessCSIDFileSimple(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package mymodels
+
+type UserID string // @csid:type [USR]
+type OrderID string // @csid:type [ORD]
+`
+	fp := writeTestFile(t, dir, "models.go", src)
+
+	ids, pkg, err := processCSIDFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "mymodels")
+	tst.AssertEqual(t, len(ids), 2)
+	tst.AssertEqual(t, ids[0].Name, "UserID")
+	tst.AssertEqual(t, ids[0].Prefix, "USR")
+	tst.AssertEqual(t, ids[1].Name, "OrderID")
+	tst.AssertEqual(t, ids[1].Prefix, "ORD")
+	tst.AssertEqual(t, ids[0].FileRelative, "models.go")
+}
+
+func TestProcessCSIDFilePrefixMustBeUppercase(t *testing.T) {
+	dir := t.TempDir()
+
+	// lowercase prefix should not match the regex (only [A-Z0-9]{3})
+	src := `package x
+
+type FooID string // @csid:type [usr]
+`
+	fp := writeTestFile(t, dir, "x.go", src)
+
+	ids, pkg, err := processCSIDFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "x")
+	tst.AssertEqual(t, len(ids), 0)
+}
+
+func TestProcessCSIDFileGeneratedHeaderSkipped(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `// Code generated by csid-generate.go DO NOT EDIT.
+
+package x
+
+type SkipMeID string // @csid:type [SKP]
+`
+	fp := writeTestFile(t, dir, "skip.go", src)
+
+	ids, pkg, err := processCSIDFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "")
+	tst.AssertEqual(t, len(ids), 0)
+}
+
+func TestGenerateCharsetIDSpecsEndToEnd(t *testing.T) {
+	dir := t.TempDir()
+
+	src1 := `package models
+
+type EntityID string // @csid:type [ENT]
+type UserID string // @csid:type [USR]
+`
+	writeTestFile(t, dir, "a_models.go", src1)
+
+	src2 := `package models
+
+type OrderID string // @csid:type [ORD]
+`
+	writeTestFile(t, dir, "b_models.go", src2)
+
+	dest := filepath.Join(dir, "csid_gen.go")
+
+	err := GenerateCharsetIDSpecs(dir, dest, CSIDGenOptions{DebugOutput: langext.PFalse})
+	tst.AssertNoErr(t, err)
+
+	out, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+	outStr := string(out)
+
+	tst.AssertTrue(t, strings.Contains(outStr, "package models"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ChecksumCharsetIDGenerator"))
+	tst.AssertTrue(t, strings.Contains(outStr, "func NewUserID()"))
+	tst.AssertTrue(t, strings.Contains(outStr, "func NewOrderID()"))
+	tst.AssertTrue(t, strings.Contains(outStr, "func NewEntityID()"))
+	tst.AssertTrue(t, strings.Contains(outStr, `prefixUserID`) && strings.Contains(outStr, `"USR"`))
+	tst.AssertTrue(t, strings.Contains(outStr, `prefixOrderID`) && strings.Contains(outStr, `"ORD"`))
+	tst.AssertTrue(t, strings.Contains(outStr, `prefixEntityID`) && strings.Contains(outStr, `"ENT"`))
+}
+
+func TestGenerateCharsetIDSpecsIdempotentWhenUnchanged(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type SomeID string // @csid:type [SOM]
+`
+	writeTestFile(t, dir, "models.go", src)
+	dest := filepath.Join(dir, "csid_gen.go")
+
+	err := GenerateCharsetIDSpecs(dir, dest, CSIDGenOptions{})
+	tst.AssertNoErr(t, err)
+
+	content1, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+
+	err = GenerateCharsetIDSpecs(dir, dest, CSIDGenOptions{})
+	tst.AssertNoErr(t, err)
+
+	content2, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, string(content1), string(content2))
+}
+
+func TestGenerateCharsetIDSpecsErrorsWithoutPackage(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `// Code generated by csid-generate.go DO NOT EDIT.
+
+package x
+
+type SkippedID string // @csid:type [SKP]
+`
+	writeTestFile(t, dir, "z.go", src)
+	dest := filepath.Join(dir, "csid_gen.go")
+
+	err := GenerateCharsetIDSpecs(dir, dest, CSIDGenOptions{})
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestGenerateCharsetIDSpecsMissingDir(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "definitely-missing")
+	err := GenerateCharsetIDSpecs(dir, filepath.Join(dir, "csid_gen.go"), CSIDGenOptions{})
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestFmtCSIDOutputContainsAllNames(t *testing.T) {
+	ids := []CSIDDef{
+		{File: "a.go", FileRelative: "a.go", Name: "AlphaID", Prefix: "ALP"},
+		{File: "b.go", FileRelative: "b.go", Name: "BetaID", Prefix: "BET"},
+	}
+	out := fmtCSIDOutput("CHK_XYZ", ids, "models")
+	tst.AssertTrue(t, strings.Contains(out, "package models"))
+	tst.AssertTrue(t, strings.Contains(out, "CHK_XYZ"))
+	tst.AssertTrue(t, strings.Contains(out, "AlphaID"))
+	tst.AssertTrue(t, strings.Contains(out, "BetaID"))
+	tst.AssertTrue(t, strings.Contains(out, `prefixAlphaID`) && strings.Contains(out, `"ALP"`))
+	tst.AssertTrue(t, strings.Contains(out, `prefixBetaID`) && strings.Contains(out, `"BET"`))
+}

bfcodegen/unit_enum_test.go

@@ -0,0 +1,369 @@
+package bfcodegen
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestProcessEnumFileBasicStringEnum(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package mymodels
+
+type Color string // @enum:type
+
+const (
+	ColorRed   Color = "red"
+	ColorBlue  Color = "blue"
+	ColorGreen Color = "green"
+)
+`
+	fp := writeTestFile(t, dir, "color.go", src)
+
+	enums, pkg, err := processEnumFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "mymodels")
+	tst.AssertEqual(t, len(enums), 1)
+	tst.AssertEqual(t, enums[0].EnumTypeName, "Color")
+	tst.AssertEqual(t, enums[0].Type, "string")
+	tst.AssertEqual(t, len(enums[0].Values), 3)
+	tst.AssertEqual(t, enums[0].Values[0].VarName, "ColorRed")
+	tst.AssertEqual(t, enums[0].Values[0].Value, `"red"`)
+	tst.AssertEqual(t, enums[0].Values[1].VarName, "ColorBlue")
+	tst.AssertEqual(t, enums[0].Values[2].VarName, "ColorGreen")
+}
+
+func TestProcessEnumFileIntEnum(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package m
+
+type Priority int // @enum:type
+
+const (
+	PriorityLow    Priority = 0
+	PriorityMedium Priority = 1
+	PriorityHigh   Priority = 2
+)
+`
+	fp := writeTestFile(t, dir, "prio.go", src)
+
+	enums, pkg, err := processEnumFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "m")
+	tst.AssertEqual(t, len(enums), 1)
+	tst.AssertEqual(t, enums[0].EnumTypeName, "Priority")
+	tst.AssertEqual(t, enums[0].Type, "int")
+	tst.AssertEqual(t, len(enums[0].Values), 3)
+	tst.AssertEqual(t, enums[0].Values[0].Value, "0")
+	tst.AssertEqual(t, enums[0].Values[2].Value, "2")
+}
+
+func TestProcessEnumFileWithDescriptions(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package m
+
+type Status string // @enum:type
+
+const (
+	StatusActive   Status = "active"   // The active status
+	StatusInactive Status = "inactive" // The inactive status
+)
+`
+	fp := writeTestFile(t, dir, "s.go", src)
+
+	enums, _, err := processEnumFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, len(enums), 1)
+	tst.AssertEqual(t, len(enums[0].Values), 2)
+
+	v0 := enums[0].Values[0]
+	tst.AssertTrue(t, v0.Description != nil)
+	tst.AssertEqual(t, *v0.Description, "The active status")
+	tst.AssertTrue(t, v0.Data == nil)
+}
+
+func TestProcessEnumFileWithDataComment(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package m
+
+type Severity string // @enum:type
+
+const (
+	SeverityLow  Severity = "low"  // {"description": "Low severity", "weight": 1}
+	SeverityHigh Severity = "high" // {"description": "High severity", "weight": 9}
+)
+`
+	fp := writeTestFile(t, dir, "sev.go", src)
+
+	enums, _, err := processEnumFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, len(enums), 1)
+	tst.AssertEqual(t, len(enums[0].Values), 2)
+
+	v0 := enums[0].Values[0]
+	tst.AssertTrue(t, v0.Data != nil)
+	tst.AssertTrue(t, v0.Description != nil)
+	tst.AssertEqual(t, *v0.Description, "Low severity")
+}
+
+func TestProcessEnumFileNonMatchingValuesNotAttached(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package m
+
+type Color string // @enum:type
+
+const (
+	ColorRed Color = "red"
+)
+
+const (
+	OtherX OtherType = "x"
+)
+`
+	fp := writeTestFile(t, dir, "c.go", src)
+
+	enums, _, err := processEnumFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, len(enums), 1)
+	tst.AssertEqual(t, len(enums[0].Values), 1)
+	tst.AssertEqual(t, enums[0].Values[0].VarName, "ColorRed")
+}
+
+func TestProcessEnumFileGeneratedHeaderSkipped(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `// Code generated by enum-generate.go DO NOT EDIT.
+
+package x
+
+type Foo string // @enum:type
+`
+	fp := writeTestFile(t, dir, "skip.go", src)
+
+	enums, pkg, err := processEnumFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "")
+	tst.AssertEqual(t, len(enums), 0)
+}
+
+func TestTryParseDataCommentValid(t *testing.T) {
+	m, ok := tryParseDataComment(`{"description": "hello", "weight": 5}`)
+	tst.AssertTrue(t, ok)
+	descr, _ := m["description"].(string)
+	tst.AssertEqual(t, descr, "hello")
+	weight, _ := m["weight"].(float64)
+	tst.AssertEqual(t, weight, float64(5))
+}
+
+func TestTryParseDataCommentBool(t *testing.T) {
+	m, ok := tryParseDataComment(`{"a": true, "b": false}`)
+	tst.AssertTrue(t, ok)
+	a, _ := m["a"].(bool)
+	tst.AssertTrue(t, a)
+	b, _ := m["b"].(bool)
+	tst.AssertFalse(t, b)
+}
+
+func TestTryParseDataCommentRejectsNull(t *testing.T) {
+	// null becomes a nil interface — its reflect.Kind is Invalid, not Pointer,
+	// so it does not match any of the allowed kinds and is rejected.
+	_, ok := tryParseDataComment(`{"x": null}`)
+	tst.AssertFalse(t, ok)
+}
+
+func TestTryParseDataCommentInvalidJSON(t *testing.T) {
+	_, ok := tryParseDataComment(`{not valid json}`)
+	tst.AssertFalse(t, ok)
+}
+
+func TestTryParseDataCommentRejectsArrays(t *testing.T) {
+	// arrays as values are not in the supported kinds list
+	_, ok := tryParseDataComment(`{"x": [1, 2, 3]}`)
+	tst.AssertFalse(t, ok)
+}
+
+func TestTryParseDataCommentRejectsObjects(t *testing.T) {
+	_, ok := tryParseDataComment(`{"x": {"nested": 1}}`)
+	tst.AssertFalse(t, ok)
+}
+
+func TestGenerateEnumSpecsEndToEnd(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type Color string // @enum:type
+
+const (
+	ColorRed   Color = "red"
+	ColorGreen Color = "green"
+	ColorBlue  Color = "blue"
+)
+`
+	writeTestFile(t, dir, "color.go", src)
+	dest := filepath.Join(dir, "enum_gen.go")
+
+	err := GenerateEnumSpecs(dir, dest, EnumGenOptions{
+		DebugOutput: langext.PFalse,
+		GoFormat:    langext.PTrue,
+	})
+	tst.AssertNoErr(t, err)
+
+	out, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+	outStr := string(out)
+
+	tst.AssertTrue(t, strings.Contains(outStr, "package models"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ChecksumEnumGenerator"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ParseColor"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ColorValues"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ColorRed"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ColorBlue"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ColorGreen"))
+}
+
+func TestGenerateEnumSpecsDeterministic(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type Status string // @enum:type
+
+const (
+	StatusActive Status = "active" // The active one
+	StatusOff    Status = "off"    // The off one
+)
+`
+	writeTestFile(t, dir, "s.go", src)
+
+	s1, cs1, changed1, err := _generateEnumSpecs(dir, "", "N/A", true, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, changed1)
+
+	s2, cs2, changed2, err := _generateEnumSpecs(dir, "", "N/A", true, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, changed2)
+
+	tst.AssertEqual(t, cs1, cs2)
+	tst.AssertEqual(t, s1, s2)
+}
+
+func TestGenerateEnumSpecsNoChangeWhenChecksumMatches(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type Status string // @enum:type
+
+const (
+	StatusActive Status = "active"
+)
+`
+	writeTestFile(t, dir, "s.go", src)
+
+	_, cs, changed, err := _generateEnumSpecs(dir, "", "N/A", true, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, changed)
+
+	s2, cs2, changed2, err := _generateEnumSpecs(dir, "", cs, true, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertFalse(t, changed2)
+	tst.AssertEqual(t, cs2, cs)
+	tst.AssertEqual(t, s2, "")
+}
+
+func TestGenerateEnumSpecsWithoutGoFormat(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type Color string // @enum:type
+
+const (
+	ColorRed Color = "red"
+)
+`
+	writeTestFile(t, dir, "c.go", src)
+
+	out, _, _, err := _generateEnumSpecs(dir, "", "N/A", false, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.Contains(out, "ColorRed"))
+	tst.AssertTrue(t, strings.Contains(out, "package models"))
+}
+
+func TestGenerateEnumSpecsErrorsWithoutPackage(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `// Code generated by enum-generate.go DO NOT EDIT.
+
+package x
+
+type Foo string // @enum:type
+`
+	writeTestFile(t, dir, "z.go", src)
+
+	_, _, _, err := _generateEnumSpecs(dir, "", "N/A", false, false)
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestGenerateEnumSpecsMissingDir(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "definitely-missing")
+	_, _, _, err := _generateEnumSpecs(dir, "", "N/A", false, false)
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestFmtEnumOutputContainsTypes(t *testing.T) {
+	descr := "the red one"
+	enums := []EnumDef{
+		{
+			File:         "color.go",
+			FileRelative: "color.go",
+			EnumTypeName: "Color",
+			Type:         "string",
+			Values: []EnumDefVal{
+				{VarName: "ColorRed", Value: `"red"`, Description: &descr},
+				{VarName: "ColorBlue", Value: `"blue"`, Description: &descr},
+			},
+		},
+	}
+	out := fmtEnumOutput("CHK1", enums, "models")
+	tst.AssertTrue(t, strings.Contains(out, "package models"))
+	tst.AssertTrue(t, strings.Contains(out, "CHK1"))
+	tst.AssertTrue(t, strings.Contains(out, "ColorRed"))
+	tst.AssertTrue(t, strings.Contains(out, "ColorBlue"))
+	tst.AssertTrue(t, strings.Contains(out, "ParseColor"))
+}
+
+func TestGenerateEnumSpecsSkipsGenFile(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type Color string // @enum:type
+
+const (
+	ColorRed Color = "red"
+)
+`
+	writeTestFile(t, dir, "c.go", src)
+
+	// generated file in same dir - should be filtered out
+	gensrc := `package models
+
+type ShouldBeIgnored string // @enum:type
+`
+	writeTestFile(t, dir, "ignored_gen.go", gensrc)
+
+	out, _, _, err := _generateEnumSpecs(dir, filepath.Join(dir, "enum_gen.go"), "N/A", false, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.Contains(out, "ColorRed"))
+	tst.AssertFalse(t, strings.Contains(out, "ShouldBeIgnored"))
+}

bfcodegen/unit_id_test.go

@@ -0,0 +1,209 @@
+package bfcodegen
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func writeTestFile(t *testing.T, dir string, name string, content string) string {
+	t.Helper()
+	p := filepath.Join(dir, name)
+	err := os.WriteFile(p, []byte(content), 0o644)
+	tst.AssertNoErr(t, err)
+	return p
+}
+
+func TestProcessIDFileSimple(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package mymodels
+
+type UserID string // @id:type
+type OrderID string // @id:type
+`
+	fp := writeTestFile(t, dir, "models.go", src)
+
+	ids, pkg, err := processIDFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "mymodels")
+	tst.AssertEqual(t, len(ids), 2)
+	tst.AssertEqual(t, ids[0].Name, "UserID")
+	tst.AssertEqual(t, ids[1].Name, "OrderID")
+	tst.AssertEqual(t, ids[0].FileRelative, "models.go")
+}
+
+func TestProcessIDFileNoMatches(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package x
+
+type Foo string
+type Bar int
+type Baz string // not the right marker
+`
+	fp := writeTestFile(t, dir, "x.go", src)
+
+	ids, pkg, err := processIDFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "x")
+	tst.AssertEqual(t, len(ids), 0)
+}
+
+func TestProcessIDFileGeneratedHeaderSkipped(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `// Code generated by id-generate.go DO NOT EDIT.
+
+package x
+
+type SkipMeID string // @id:type
+`
+	fp := writeTestFile(t, dir, "skip.go", src)
+
+	ids, pkg, err := processIDFile(dir, fp, false)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, pkg, "")
+	tst.AssertEqual(t, len(ids), 0)
+}
+
+func TestProcessIDFileMissingFile(t *testing.T) {
+	_, _, err := processIDFile(t.TempDir(), filepath.Join(t.TempDir(), "does_not_exist.go"), false)
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestGenerateIDSpecsEndToEnd(t *testing.T) {
+	dir := t.TempDir()
+
+	src1 := `package models
+
+type UserID string // @id:type
+type AnyID string // @id:type
+`
+	writeTestFile(t, dir, "a_models.go", src1)
+
+	src2 := `package models
+
+type OrderID string // @id:type
+`
+	writeTestFile(t, dir, "b_models.go", src2)
+
+	dest := filepath.Join(dir, "id_gen.go")
+
+	err := GenerateIDSpecs(dir, dest, IDGenOptions{DebugOutput: langext.PFalse})
+	tst.AssertNoErr(t, err)
+
+	out, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+	outStr := string(out)
+
+	tst.AssertTrue(t, strings.Contains(outStr, "package models"))
+	tst.AssertTrue(t, strings.Contains(outStr, "ChecksumIDGenerator"))
+	tst.AssertTrue(t, strings.Contains(outStr, "func NewUserID()"))
+	tst.AssertTrue(t, strings.Contains(outStr, "func NewOrderID()"))
+	tst.AssertTrue(t, strings.Contains(outStr, "func NewAnyID()"))
+	tst.AssertTrue(t, strings.Contains(outStr, "AsAny()"))
+}
+
+func TestGenerateIDSpecsIdempotentWhenUnchanged(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type SomeID string // @id:type
+`
+	writeTestFile(t, dir, "models.go", src)
+	dest := filepath.Join(dir, "id_gen.go")
+
+	err := GenerateIDSpecs(dir, dest, IDGenOptions{})
+	tst.AssertNoErr(t, err)
+
+	stat1, err := os.Stat(dest)
+	tst.AssertNoErr(t, err)
+
+	content1, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(dir, dest, IDGenOptions{})
+	tst.AssertNoErr(t, err)
+
+	stat2, err := os.Stat(dest)
+	tst.AssertNoErr(t, err)
+
+	content2, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, stat1.ModTime().Equal(stat2.ModTime()), true)
+	tst.AssertEqual(t, string(content1), string(content2))
+}
+
+func TestGenerateIDSpecsRegeneratesAfterChange(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `package models
+
+type FirstID string // @id:type
+`
+	fp := writeTestFile(t, dir, "models.go", src)
+	dest := filepath.Join(dir, "id_gen.go")
+
+	err := GenerateIDSpecs(dir, dest, IDGenOptions{})
+	tst.AssertNoErr(t, err)
+
+	content1, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.Contains(string(content1), "FirstID"))
+	tst.AssertFalse(t, strings.Contains(string(content1), "SecondID"))
+
+	src2 := `package models
+
+type FirstID string // @id:type
+type SecondID string // @id:type
+`
+	err = os.WriteFile(fp, []byte(src2), 0o644)
+	tst.AssertNoErr(t, err)
+
+	err = GenerateIDSpecs(dir, dest, IDGenOptions{})
+	tst.AssertNoErr(t, err)
+
+	content2, err := os.ReadFile(dest)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.Contains(string(content2), "SecondID"))
+}
+
+func TestGenerateIDSpecsErrorsWithoutPackage(t *testing.T) {
+	dir := t.TempDir()
+
+	src := `// Code generated by id-generate.go DO NOT EDIT.
+
+package x
+
+type SkippedID string // @id:type
+`
+	writeTestFile(t, dir, "z.go", src)
+	dest := filepath.Join(dir, "id_gen.go")
+
+	err := GenerateIDSpecs(dir, dest, IDGenOptions{})
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestGenerateIDSpecsMissingDir(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "definitely-missing")
+	err := GenerateIDSpecs(dir, filepath.Join(dir, "id_gen.go"), IDGenOptions{})
+	tst.AssertTrue(t, err != nil)
+}
+
+func TestFmtIDOutputContainsAllNames(t *testing.T) {
+	ids := []IDDef{
+		{File: "a.go", FileRelative: "a.go", Name: "AlphaID"},
+		{File: "b.go", FileRelative: "b.go", Name: "BetaID"},
+	}
+	out := fmtIDOutput("CHK_ABC", ids, "models")
+	tst.AssertTrue(t, strings.Contains(out, "package models"))
+	tst.AssertTrue(t, strings.Contains(out, "CHK_ABC"))
+	tst.AssertTrue(t, strings.Contains(out, "AlphaID"))
+	tst.AssertTrue(t, strings.Contains(out, "BetaID"))
+}

cmdext/builder.go

@@ -0,0 +1,100 @@
+package cmdext
+
+import (
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"time"
+)
+
+type CommandRunner struct {
+	program          string
+	args             []string
+	timeout          *time.Duration
+	env              []string
+	listener         []CommandListener
+	enforceExitCodes *[]int
+	enforceNoTimeout bool
+	enforceNoStderr  bool
+}
+
+func Runner(program string) *CommandRunner {
+	return &CommandRunner{
+		program:          program,
+		args:             make([]string, 0),
+		timeout:          nil,
+		env:              make([]string, 0),
+		listener:         make([]CommandListener, 0),
+		enforceExitCodes: nil,
+		enforceNoTimeout: false,
+		enforceNoStderr:  false,
+	}
+}
+
+func (r *CommandRunner) Arg(arg string) *CommandRunner {
+	r.args = append(r.args, arg)
+	return r
+}
+
+func (r *CommandRunner) Args(arg []string) *CommandRunner {
+	r.args = append(r.args, arg...)
+	return r
+}
+
+func (r *CommandRunner) Timeout(timeout time.Duration) *CommandRunner {
+	r.timeout = &timeout
+	return r
+}
+
+func (r *CommandRunner) Env(key, value string) *CommandRunner {
+	r.env = append(r.env, fmt.Sprintf("%s=%s", key, value))
+	return r
+}
+
+func (r *CommandRunner) RawEnv(env string) *CommandRunner {
+	r.env = append(r.env, env)
+	return r
+}
+
+func (r *CommandRunner) Envs(env []string) *CommandRunner {
+	r.env = append(r.env, env...)
+	return r
+}
+
+func (r *CommandRunner) EnsureExitcode(arg ...int) *CommandRunner {
+	r.enforceExitCodes = new(langext.ForceArray(arg))
+	return r
+}
+
+func (r *CommandRunner) FailOnExitCode() *CommandRunner {
+	r.enforceExitCodes = new([]int{0})
+	return r
+}
+
+func (r *CommandRunner) FailOnTimeout() *CommandRunner {
+	r.enforceNoTimeout = true
+	return r
+}
+
+func (r *CommandRunner) FailOnStderr() *CommandRunner {
+	r.enforceNoStderr = true
+	return r
+}
+
+func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
+	r.listener = append(r.listener, lstr)
+	return r
+}
+
+func (r *CommandRunner) ListenStdout(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStdoutLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) ListenStderr(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStderrLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) Run() (CommandResult, error) {
+	return run(*r)
+}

cmdext/builder_test.go

@@ -0,0 +1,324 @@
+package cmdext
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestRunnerInit(t *testing.T) {
+	r := Runner("myprog")
+
+	if r == nil {
+		t.Fatalf("Runner returned nil")
+	}
+	if r.program != "myprog" {
+		t.Errorf("program == %v, want myprog", r.program)
+	}
+	if r.args == nil {
+		t.Errorf("args is nil, want empty slice")
+	}
+	if len(r.args) != 0 {
+		t.Errorf("len(args) == %v, want 0", len(r.args))
+	}
+	if r.env == nil {
+		t.Errorf("env is nil, want empty slice")
+	}
+	if len(r.env) != 0 {
+		t.Errorf("len(env) == %v, want 0", len(r.env))
+	}
+	if r.listener == nil {
+		t.Errorf("listener is nil, want empty slice")
+	}
+	if len(r.listener) != 0 {
+		t.Errorf("len(listener) == %v, want 0", len(r.listener))
+	}
+	if r.timeout != nil {
+		t.Errorf("timeout == %v, want nil", r.timeout)
+	}
+	if r.enforceExitCodes != nil {
+		t.Errorf("enforceExitCodes == %v, want nil", r.enforceExitCodes)
+	}
+	if r.enforceNoTimeout {
+		t.Errorf("enforceNoTimeout == true, want false")
+	}
+	if r.enforceNoStderr {
+		t.Errorf("enforceNoStderr == true, want false")
+	}
+}
+
+func TestArgSingle(t *testing.T) {
+	r := Runner("p").Arg("a")
+	if !reflect.DeepEqual(r.args, []string{"a"}) {
+		t.Errorf("args == %v, want [a]", r.args)
+	}
+}
+
+func TestArgMultiple(t *testing.T) {
+	r := Runner("p").Arg("a").Arg("b").Arg("c")
+	if !reflect.DeepEqual(r.args, []string{"a", "b", "c"}) {
+		t.Errorf("args == %v, want [a b c]", r.args)
+	}
+}
+
+func TestArgsAppendsAll(t *testing.T) {
+	r := Runner("p").Args([]string{"x", "y"}).Args([]string{"z"})
+	if !reflect.DeepEqual(r.args, []string{"x", "y", "z"}) {
+		t.Errorf("args == %v, want [x y z]", r.args)
+	}
+}
+
+func TestArgAndArgsMixed(t *testing.T) {
+	r := Runner("p").Arg("a").Args([]string{"b", "c"}).Arg("d")
+	if !reflect.DeepEqual(r.args, []string{"a", "b", "c", "d"}) {
+		t.Errorf("args == %v, want [a b c d]", r.args)
+	}
+}
+
+func TestArgsEmptySlice(t *testing.T) {
+	r := Runner("p").Args([]string{})
+	if len(r.args) != 0 {
+		t.Errorf("len(args) == %v, want 0", len(r.args))
+	}
+}
+
+func TestTimeoutSet(t *testing.T) {
+	d := 500 * time.Millisecond
+	r := Runner("p").Timeout(d)
+	if r.timeout == nil {
+		t.Fatalf("timeout is nil")
+	}
+	if *r.timeout != d {
+		t.Errorf("timeout == %v, want %v", *r.timeout, d)
+	}
+}
+
+func TestTimeoutOverride(t *testing.T) {
+	r := Runner("p").Timeout(1 * time.Second).Timeout(2 * time.Second)
+	if *r.timeout != 2*time.Second {
+		t.Errorf("timeout == %v, want 2s", *r.timeout)
+	}
+}
+
+func TestEnv(t *testing.T) {
+	r := Runner("p").Env("KEY", "VALUE")
+	if !reflect.DeepEqual(r.env, []string{"KEY=VALUE"}) {
+		t.Errorf("env == %v, want [KEY=VALUE]", r.env)
+	}
+}
+
+func TestEnvMultiple(t *testing.T) {
+	r := Runner("p").Env("A", "1").Env("B", "2")
+	if !reflect.DeepEqual(r.env, []string{"A=1", "B=2"}) {
+		t.Errorf("env == %v, want [A=1 B=2]", r.env)
+	}
+}
+
+func TestEnvWithEmptyValue(t *testing.T) {
+	r := Runner("p").Env("KEY", "")
+	if !reflect.DeepEqual(r.env, []string{"KEY="}) {
+		t.Errorf("env == %v, want [KEY=]", r.env)
+	}
+}
+
+func TestRawEnv(t *testing.T) {
+	r := Runner("p").RawEnv("FOO=BAR=BAZ")
+	if !reflect.DeepEqual(r.env, []string{"FOO=BAR=BAZ"}) {
+		t.Errorf("env == %v, want [FOO=BAR=BAZ]", r.env)
+	}
+}
+
+func TestEnvs(t *testing.T) {
+	r := Runner("p").Envs([]string{"A=1", "B=2"})
+	if !reflect.DeepEqual(r.env, []string{"A=1", "B=2"}) {
+		t.Errorf("env == %v, want [A=1 B=2]", r.env)
+	}
+}
+
+func TestEnvMixed(t *testing.T) {
+	r := Runner("p").Env("A", "1").RawEnv("B=2").Envs([]string{"C=3", "D=4"})
+	if !reflect.DeepEqual(r.env, []string{"A=1", "B=2", "C=3", "D=4"}) {
+		t.Errorf("env == %v, want [A=1 B=2 C=3 D=4]", r.env)
+	}
+}
+
+func TestEnsureExitcodeSingle(t *testing.T) {
+	r := Runner("p").EnsureExitcode(2)
+	if r.enforceExitCodes == nil {
+		t.Fatalf("enforceExitCodes is nil")
+	}
+	if !reflect.DeepEqual(*r.enforceExitCodes, []int{2}) {
+		t.Errorf("enforceExitCodes == %v, want [2]", *r.enforceExitCodes)
+	}
+}
+
+func TestEnsureExitcodeMultiple(t *testing.T) {
+	r := Runner("p").EnsureExitcode(0, 1, 2)
+	if r.enforceExitCodes == nil {
+		t.Fatalf("enforceExitCodes is nil")
+	}
+	if !reflect.DeepEqual(*r.enforceExitCodes, []int{0, 1, 2}) {
+		t.Errorf("enforceExitCodes == %v, want [0 1 2]", *r.enforceExitCodes)
+	}
+}
+
+func TestFailOnExitCode(t *testing.T) {
+	r := Runner("p").FailOnExitCode()
+	if r.enforceExitCodes == nil {
+		t.Fatalf("enforceExitCodes is nil")
+	}
+	if !reflect.DeepEqual(*r.enforceExitCodes, []int{0}) {
+		t.Errorf("enforceExitCodes == %v, want [0]", *r.enforceExitCodes)
+	}
+}
+
+func TestFailOnTimeoutFlag(t *testing.T) {
+	r := Runner("p")
+	if r.enforceNoTimeout {
+		t.Errorf("enforceNoTimeout was true before set")
+	}
+	r = r.FailOnTimeout()
+	if !r.enforceNoTimeout {
+		t.Errorf("enforceNoTimeout == false after FailOnTimeout()")
+	}
+}
+
+func TestFailOnStderrFlag(t *testing.T) {
+	r := Runner("p")
+	if r.enforceNoStderr {
+		t.Errorf("enforceNoStderr was true before set")
+	}
+	r = r.FailOnStderr()
+	if !r.enforceNoStderr {
+		t.Errorf("enforceNoStderr == false after FailOnStderr()")
+	}
+}
+
+func TestListen(t *testing.T) {
+	r := Runner("p").Listen(genericCommandListener{})
+	if len(r.listener) != 1 {
+		t.Errorf("len(listener) == %v, want 1", len(r.listener))
+	}
+}
+
+func TestListenMultiple(t *testing.T) {
+	r := Runner("p").
+		Listen(genericCommandListener{}).
+		Listen(genericCommandListener{}).
+		Listen(genericCommandListener{})
+	if len(r.listener) != 3 {
+		t.Errorf("len(listener) == %v, want 3", len(r.listener))
+	}
+}
+
+func TestListenStdoutAddsListener(t *testing.T) {
+	r := Runner("p").ListenStdout(func(string) {})
+	if len(r.listener) != 1 {
+		t.Errorf("len(listener) == %v, want 1", len(r.listener))
+	}
+}
+
+func TestListenStdoutForwardsCalls(t *testing.T) {
+	got := ""
+	r := Runner("p").ListenStdout(func(s string) { got = s })
+	if len(r.listener) != 1 {
+		t.Fatalf("len(listener) == %v, want 1", len(r.listener))
+	}
+	r.listener[0].ReadStdoutLine("hello")
+	if got != "hello" {
+		t.Errorf("listener got %q, want hello", got)
+	}
+	// non-stdout methods should not panic and should not affect state
+	r.listener[0].ReadStderrLine("nope")
+	r.listener[0].ReadRawStdout([]byte("raw"))
+	r.listener[0].ReadRawStderr([]byte("raw"))
+	r.listener[0].Finished(0)
+	r.listener[0].Timeout()
+	if got != "hello" {
+		t.Errorf("listener got mutated to %q, want hello", got)
+	}
+}
+
+func TestListenStderrAddsListener(t *testing.T) {
+	r := Runner("p").ListenStderr(func(string) {})
+	if len(r.listener) != 1 {
+		t.Errorf("len(listener) == %v, want 1", len(r.listener))
+	}
+}
+
+func TestListenStderrForwardsCalls(t *testing.T) {
+	got := ""
+	r := Runner("p").ListenStderr(func(s string) { got = s })
+	if len(r.listener) != 1 {
+		t.Fatalf("len(listener) == %v, want 1", len(r.listener))
+	}
+	r.listener[0].ReadStderrLine("oops")
+	if got != "oops" {
+		t.Errorf("listener got %q, want oops", got)
+	}
+	r.listener[0].ReadStdoutLine("nope")
+	if got != "oops" {
+		t.Errorf("listener got mutated to %q, want oops", got)
+	}
+}
+
+func TestChainReturnsSameInstance(t *testing.T) {
+	r := Runner("p")
+	if r.Arg("a") != r {
+		t.Errorf("Arg returned different instance")
+	}
+	if r.Args([]string{"b"}) != r {
+		t.Errorf("Args returned different instance")
+	}
+	if r.Timeout(time.Second) != r {
+		t.Errorf("Timeout returned different instance")
+	}
+	if r.Env("K", "V") != r {
+		t.Errorf("Env returned different instance")
+	}
+	if r.RawEnv("K=V") != r {
+		t.Errorf("RawEnv returned different instance")
+	}
+	if r.Envs([]string{"K=V"}) != r {
+		t.Errorf("Envs returned different instance")
+	}
+	if r.EnsureExitcode(0) != r {
+		t.Errorf("EnsureExitcode returned different instance")
+	}
+	if r.FailOnExitCode() != r {
+		t.Errorf("FailOnExitCode returned different instance")
+	}
+	if r.FailOnTimeout() != r {
+		t.Errorf("FailOnTimeout returned different instance")
+	}
+	if r.FailOnStderr() != r {
+		t.Errorf("FailOnStderr returned different instance")
+	}
+	if r.Listen(genericCommandListener{}) != r {
+		t.Errorf("Listen returned different instance")
+	}
+	if r.ListenStdout(func(string) {}) != r {
+		t.Errorf("ListenStdout returned different instance")
+	}
+	if r.ListenStderr(func(string) {}) != r {
+		t.Errorf("ListenStderr returned different instance")
+	}
+}
+
+func TestSeparateInstancesIndependent(t *testing.T) {
+	r1 := Runner("p1").Arg("a")
+	r2 := Runner("p2").Arg("b")
+
+	if r1.program != "p1" {
+		t.Errorf("r1.program == %v, want p1", r1.program)
+	}
+	if r2.program != "p2" {
+		t.Errorf("r2.program == %v, want p2", r2.program)
+	}
+	if !reflect.DeepEqual(r1.args, []string{"a"}) {
+		t.Errorf("r1.args == %v, want [a]", r1.args)
+	}
+	if !reflect.DeepEqual(r2.args, []string{"b"}) {
+		t.Errorf("r2.args == %v, want [b]", r2.args)
+	}
+}

cmdext/cmdrunner.go

@@ -0,0 +1,196 @@
+package cmdext
+
+import (
+	"errors"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/mathext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/syncext"
+	"os/exec"
+	"time"
+)
+
+var ErrExitCode = errors.New("process exited with an unexpected exitcode")
+var ErrTimeout = errors.New("process did not exit after the specified timeout")
+var ErrStderrPrint = errors.New("process did print to stderr stream")
+
+type CommandResult struct {
+	StdOut          string
+	StdErr          string
+	StdCombined     string
+	ExitCode        int
+	CommandTimedOut bool
+}
+
+func run(opt CommandRunner) (CommandResult, error) {
+	cmd := exec.Command(opt.program, opt.args...)
+
+	cmd.Env = append(cmd.Env, opt.env...)
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	preader := pipeReader{
+		lineBufferSize: new(128 * 1024 * 1024), // 128MB max size of a single line, is hopefully enough....
+		stdout:         stdoutPipe,
+		stderr:         stderrPipe,
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	type resultObj struct {
+		stdout      string
+		stderr      string
+		stdcombined string
+		err         error
+	}
+
+	stderrFailChan := make(chan bool)
+
+	outputChan := make(chan resultObj)
+	go func() {
+		// we need to first fully read the pipes and then call Wait
+		// see https://pkg.go.dev/os/exec#Cmd.StdoutPipe
+
+		listener := make([]CommandListener, 0)
+		listener = append(listener, opt.listener...)
+
+		if opt.enforceNoStderr {
+			listener = append(listener, genericCommandListener{
+				_readRawStderr: new(func(v []byte) {
+					if len(v) > 0 {
+						stderrFailChan <- true
+					}
+				}),
+			})
+		}
+
+		stdout, stderr, stdcombined, err := preader.Read(listener)
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+			_ = cmd.Process.Kill()
+			return
+		}
+
+		err = cmd.Wait()
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+		} else {
+			outputChan <- resultObj{stdout, stderr, stdcombined, nil}
+		}
+
+	}()
+
+	var timeoutChan <-chan time.Time = make(chan time.Time, 1)
+	if opt.timeout != nil {
+		timeoutChan = time.After(*opt.timeout)
+	}
+
+	select {
+
+	case <-timeoutChan:
+		_ = cmd.Process.Kill()
+		for _, lstr := range opt.listener {
+			lstr.Timeout()
+		}
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
+			// most of the time the cmd.Process.Kill() should also ahve finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		}
+
+	case <-stderrFailChan:
+		_ = cmd.Process.Kill()
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, 32*time.Millisecond); ok {
+			// most of the time the cmd.Process.Kill() should also have finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
+				ExitCode:        -1,
+				CommandTimedOut: false,
+			}
+			return res, ErrStderrPrint
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: false,
+			}
+			return res, ErrStderrPrint
+		}
+
+	case outobj := <-outputChan:
+		var exiterr *exec.ExitError
+		if errors.As(outobj.err, &exiterr) {
+			excode := exiterr.ExitCode()
+			for _, lstr := range opt.listener {
+				lstr.Finished(excode)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        excode,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(excode, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		} else if err != nil {
+			return CommandResult{}, err
+		} else {
+			for _, lstr := range opt.listener {
+				lstr.Finished(0)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        0,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(0, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		}
+	}
+}

cmdext/cmdrunner_test.go

@@ -0,0 +1,348 @@
+package cmdext
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestStdout(t *testing.T) {
+
+	res1, err := Runner("printf").Arg("hello").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "hello" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "hello\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStdcombined(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"1\", file=sys.stderr, flush=True); time.sleep(0.1); print(\"2\", file=sys.stdout, flush=True); time.sleep(0.1); print(\"3\", file=sys.stderr, flush=True)").
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "1\n3\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "2\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "1\n2\n3\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialRead(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", flush=True); time.sleep(5); print(\"cant see me\", flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadStderr(t *testing.T) {
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, flush=True); time.sleep(5); print(\"cant see me\", file=sys.stderr, flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStdout(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stdout, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "message101" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "message101" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushed(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", end=''); time.sleep(5); print(\"cant see me\", end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushedStderr(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, end=''); time.sleep(5); print(\"cant see me\", file=sys.stderr, end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestListener(t *testing.T) {
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys;" +
+			"import time;" +
+			"print(\"message 1\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 2\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 3\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 4\", file=sys.stderr, flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 5\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"final\");").
+		ListenStdout(func(s string) { fmt.Printf("@@STDOUT <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		ListenStderr(func(s string) { fmt.Printf("@@STDERR <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		Timeout(10 * time.Second).
+		Run()
+
+	if err != nil {
+		panic(err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+}
+
+func TestLongStdout(t *testing.T) {
+
+	res1, err := Runner("python3").
+		Arg("-c").
+		Arg("import sys; import time; print(\"X\" * 125001 + \"\\n\"); print(\"Y\" * 125001 + \"\\n\"); print(\"Z\" * 125001 + \"\\n\");").
+		Timeout(5000 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if len(res1.StdOut) != 375009 {
+		t.Errorf("len(res1.StdOut) == '%v'", len(res1.StdOut))
+	}
+
+}
+
+func TestFailOnTimeout(t *testing.T) {
+
+	_, err := Runner("sleep").Arg("2").Timeout(200 * time.Millisecond).FailOnTimeout().Run()
+	if !errors.Is(err, ErrTimeout) {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestFailOnStderr(t *testing.T) {
+
+	res1, err := Runner("python3").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").FailOnStderr().Run()
+	if err == nil {
+		t.Errorf("no err")
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != -1 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestFailOnExitcode(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).FailOnExitCode().Run()
+	if !errors.Is(err, ErrExitCode) {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode1(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).EnsureExitcode(1).Run()
+	if err != nil {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode2(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200*time.Millisecond).EnsureExitcode(0, 2, 3).Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}

cmdext/helper.go

@@ -0,0 +1,12 @@
+package cmdext
+
+import "time"
+
+func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+	b := Runner(program)
+	b = b.Args(args)
+	if timeout != nil {
+		b = b.Timeout(*timeout)
+	}
+	return b.Run()
+}

cmdext/listener.go

@@ -0,0 +1,57 @@
+package cmdext
+
+type CommandListener interface {
+	ReadRawStdout([]byte)
+	ReadRawStderr([]byte)
+
+	ReadStdoutLine(string)
+	ReadStderrLine(string)
+
+	Finished(int)
+	Timeout()
+}
+
+type genericCommandListener struct {
+	_readRawStdout  *func([]byte)
+	_readRawStderr  *func([]byte)
+	_readStdoutLine *func(string)
+	_readStderrLine *func(string)
+	_finished       *func(int)
+	_timeout        *func()
+}
+
+func (g genericCommandListener) ReadRawStdout(v []byte) {
+	if g._readRawStdout != nil {
+		(*g._readRawStdout)(v)
+	}
+}
+
+func (g genericCommandListener) ReadRawStderr(v []byte) {
+	if g._readRawStderr != nil {
+		(*g._readRawStderr)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStdoutLine(v string) {
+	if g._readStdoutLine != nil {
+		(*g._readStdoutLine)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStderrLine(v string) {
+	if g._readStderrLine != nil {
+		(*g._readStderrLine)(v)
+	}
+}
+
+func (g genericCommandListener) Finished(v int) {
+	if g._finished != nil {
+		(*g._finished)(v)
+	}
+}
+
+func (g genericCommandListener) Timeout() {
+	if g._timeout != nil {
+		(*g._timeout)()
+	}
+}

cmdext/listener_test.go

@@ -0,0 +1,143 @@
+package cmdext
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGenericListenerEmptyDoesNotPanic(t *testing.T) {
+	l := genericCommandListener{}
+	l.ReadRawStdout([]byte("x"))
+	l.ReadRawStderr([]byte("x"))
+	l.ReadStdoutLine("x")
+	l.ReadStderrLine("x")
+	l.Finished(0)
+	l.Timeout()
+}
+
+func TestGenericListenerReadRawStdout(t *testing.T) {
+	var got []byte
+	fn := func(b []byte) { got = append(got, b...) }
+	l := genericCommandListener{_readRawStdout: &fn}
+
+	l.ReadRawStdout([]byte("hello"))
+	l.ReadRawStdout([]byte(" world"))
+
+	if string(got) != "hello world" {
+		t.Errorf("got %q, want %q", string(got), "hello world")
+	}
+}
+
+func TestGenericListenerReadRawStderr(t *testing.T) {
+	var got []byte
+	fn := func(b []byte) { got = append(got, b...) }
+	l := genericCommandListener{_readRawStderr: &fn}
+
+	l.ReadRawStderr([]byte("err"))
+
+	if string(got) != "err" {
+		t.Errorf("got %q, want %q", string(got), "err")
+	}
+}
+
+func TestGenericListenerReadStdoutLine(t *testing.T) {
+	var got []string
+	fn := func(s string) { got = append(got, s) }
+	l := genericCommandListener{_readStdoutLine: &fn}
+
+	l.ReadStdoutLine("line1")
+	l.ReadStdoutLine("line2")
+
+	if !reflect.DeepEqual(got, []string{"line1", "line2"}) {
+		t.Errorf("got %v, want [line1 line2]", got)
+	}
+}
+
+func TestGenericListenerReadStderrLine(t *testing.T) {
+	var got []string
+	fn := func(s string) { got = append(got, s) }
+	l := genericCommandListener{_readStderrLine: &fn}
+
+	l.ReadStderrLine("line1")
+	l.ReadStderrLine("line2")
+
+	if !reflect.DeepEqual(got, []string{"line1", "line2"}) {
+		t.Errorf("got %v, want [line1 line2]", got)
+	}
+}
+
+func TestGenericListenerFinished(t *testing.T) {
+	var got int
+	called := false
+	fn := func(v int) { got = v; called = true }
+	l := genericCommandListener{_finished: &fn}
+
+	l.Finished(42)
+
+	if !called {
+		t.Errorf("Finished callback was not called")
+	}
+	if got != 42 {
+		t.Errorf("got %v, want 42", got)
+	}
+}
+
+func TestGenericListenerTimeout(t *testing.T) {
+	called := false
+	fn := func() { called = true }
+	l := genericCommandListener{_timeout: &fn}
+
+	l.Timeout()
+
+	if !called {
+		t.Errorf("Timeout callback was not called")
+	}
+}
+
+func TestGenericListenerOnlySpecifiedCalled(t *testing.T) {
+	stdoutCalled := false
+	stderrCalled := false
+	stdoutFn := func(string) { stdoutCalled = true }
+	stderrFn := func(string) { stderrCalled = true }
+	l := genericCommandListener{_readStdoutLine: &stdoutFn, _readStderrLine: &stderrFn}
+
+	l.ReadStdoutLine("x")
+	if !stdoutCalled {
+		t.Errorf("stdout callback not called")
+	}
+	if stderrCalled {
+		t.Errorf("stderr callback called when it shouldn't be")
+	}
+
+	stdoutCalled = false
+	l.ReadStderrLine("x")
+	if stdoutCalled {
+		t.Errorf("stdout callback called when it shouldn't be")
+	}
+	if !stderrCalled {
+		t.Errorf("stderr callback not called")
+	}
+
+	// these have no callbacks set; should be no-ops
+	l.ReadRawStdout([]byte("x"))
+	l.ReadRawStderr([]byte("x"))
+	l.Finished(0)
+	l.Timeout()
+}
+
+func TestGenericListenerImplementsCommandListener(t *testing.T) {
+	var _ CommandListener = genericCommandListener{}
+}
+
+func TestGenericListenerEmptyByteSlice(t *testing.T) {
+	calls := 0
+	fn := func(b []byte) { calls++ }
+	l := genericCommandListener{_readRawStdout: &fn}
+
+	l.ReadRawStdout([]byte{})
+	l.ReadRawStdout(nil)
+
+	if calls != 2 {
+		t.Errorf("calls == %v, want 2", calls)
+	}
+}

cmdext/pipereader.go

@@ -0,0 +1,155 @@
+package cmdext
+
+import (
+	"bufio"
+	"git.blackforestbytes.com/BlackForestBytes/goext/syncext"
+	"io"
+	"strings"
+	"sync"
+)
+
+type pipeReader struct {
+	lineBufferSize *int
+	stdout         io.ReadCloser
+	stderr         io.ReadCloser
+}
+
+// Read ready stdout and stdin until finished
+// also splits both pipes into lines and calld the listener
+func (pr *pipeReader) Read(listener []CommandListener) (string, string, string, error) {
+	type combevt struct {
+		line string
+		stop bool
+	}
+
+	errch := make(chan error, 8)
+
+	wg := sync.WaitGroup{}
+
+	// [1] read raw stdout
+
+	wg.Add(1)
+	stdoutBufferReader, stdoutBufferWriter := io.Pipe()
+	var stdout strings.Builder
+	go func() {
+		buf := make([]byte, 128)
+		for {
+			n, err := pr.stdout.Read(buf)
+			if n > 0 {
+				txt := string(buf[:n])
+				stdout.WriteString(txt)
+				_, _ = stdoutBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStdout(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stdoutBufferWriter.Close()
+		wg.Done()
+	}()
+
+	// [2] read raw stderr
+
+	wg.Add(1)
+	stderrBufferReader, stderrBufferWriter := io.Pipe()
+	var stderr strings.Builder
+	go func() {
+		buf := make([]byte, 128)
+		for {
+			n, err := pr.stderr.Read(buf)
+
+			if n > 0 {
+				txt := string(buf[:n])
+				stderr.WriteString(txt)
+				_, _ = stderrBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStderr(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stderrBufferWriter.Close()
+		wg.Done()
+	}()
+
+	combch := make(chan combevt, 32)
+
+	// [3] collect stdout line-by-line
+
+	wg.Go(func() {
+		scanner := bufio.NewScanner(stdoutBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStdoutLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+	})
+
+	// [4] collect stderr line-by-line
+
+	wg.Go(func() {
+		scanner := bufio.NewScanner(stderrBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStderrLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+	})
+
+	// [5] combine stdcombined
+
+	wg.Add(1)
+	var stdcombined strings.Builder
+	go func() {
+		stopctr := 0
+		for stopctr < 2 {
+			vvv := <-combch
+			if vvv.stop {
+				stopctr++
+			} else {
+				stdcombined.WriteString(vvv.line + "\n") // this comes from bufio.Scanner and has no newlines...
+			}
+		}
+		wg.Done()
+	}()
+
+	// wait for all (5) goroutines to finish
+	wg.Wait()
+
+	if err, ok := syncext.ReadNonBlocking(errch); ok {
+		return "", "", "", err
+	}
+
+	return stdout.String(), stderr.String(), stdcombined.String(), nil
+}

confext/confParser.go

@@ -3,20 +3,33 @@ package confext
 import (
 	"errors"
 	"fmt"
-	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/timeext"
 	"math/bits"
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 )
 
 // ApplyEnvOverrides overrides field values from environment variables
 //
 // fields must be tagged with `env:"env_key"`
-func ApplyEnvOverrides[T any](c *T) error {
-
+//
+// only works on exported fields
+//
+// fields without an env tag are ignored
+// fields with an `env:"-"` tag are ignore
+//
+// sub-structs are recursively parsed (if they have an env tag) and the env-variable keys are delimited by the delim parameter
+// sub-structs with `env:""` are also parsed, but the delimited is skipped (they are handled as if they were one level higher)
+func ApplyEnvOverrides[T any](prefix string, c *T, delim string) error {
 	rval := reflect.ValueOf(c).Elem()
+
+	return processEnvOverrides(rval, delim, prefix)
+}
+
+func processEnvOverrides(rval reflect.Value, delim string, prefix string) error {
 	rtyp := rval.Type()
 
 	for i := 0; i < rtyp.NumField(); i++ {
@@ -24,113 +37,159 @@ func ApplyEnvOverrides[T any](c *T) error {
 		rsfield := rtyp.Field(i)
 		rvfield := rval.Field(i)
 
-		envkey := rsfield.Tag.Get("env")
-		if envkey == "" {
+		if !rsfield.IsExported() {
 			continue
 		}
 
-		envval, efound := os.LookupEnv(envkey)
+		envkey, found := rsfield.Tag.Lookup("env")
+		if !found || envkey == "-" {
+			continue
+		}
+
+		if rvfield.Kind() == reflect.Struct && rvfield.Type() != reflect.TypeOf(time.UnixMilli(0)) {
+
+			subPrefix := prefix
+			if envkey != "" {
+				subPrefix = subPrefix + envkey + delim
+			}
+
+			err := processEnvOverrides(rvfield, delim, subPrefix)
+			if err != nil {
+				return err
+			}
+
+			continue
+		}
+
+		fullEnvKey := prefix + envkey
+
+		envval, efound := os.LookupEnv(fullEnvKey)
 		if !efound {
 			continue
 		}
 
-		if rvfield.Type() == reflect.TypeOf("") {
+		if rvfield.Type().Kind() == reflect.Pointer {
 
-			rvfield.Set(reflect.ValueOf(envval))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type().Elem())
 			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", envkey, envval))
+				return err
 			}
 
-			rvfield.Set(reflect.ValueOf(int(envint)))
+			// converts reflect.Value to pointer
+			ptrval := reflect.New(rvfield.Type().Elem())
+			ptrval.Elem().Set(newval)
 
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
+			rvfield.Set(ptrval)
 
-		} else if rvfield.Type() == reflect.TypeOf(int64(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 64)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int64(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int32(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 32)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int32(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int8(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 8)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int8(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(time.Duration(0)) {
-
-			dur, err := timeext.ParseDurationShortString(envval)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(dur))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, dur.String())
-
-		} else if rvfield.Type() == reflect.TypeOf(time.UnixMilli(0)) {
-
-			tim, err := time.Parse(time.RFC3339Nano, envval)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(tim))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, tim.String())
-
-		} else if rvfield.Type().ConvertibleTo(reflect.TypeOf(int(0))) {
-
-			envint, err := strconv.ParseInt(envval, 10, 8)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvfield.Type().Name(), envkey, envval))
-			}
-
-			envcvl := reflect.ValueOf(envint).Convert(rvfield.Type())
-
-			rvfield.Set(envcvl)
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%v'\n", envkey, envcvl.Interface())
-
-		} else if rvfield.Type().ConvertibleTo(reflect.TypeOf("")) {
-
-			envcvl := reflect.ValueOf(envval).Convert(rvfield.Type())
-
-			rvfield.Set(envcvl)
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%v'\n", envkey, envcvl.Interface())
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
 
 		} else {
-			return errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvfield.Kind().String(), rvfield.Type().String()))
+
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type())
+			if err != nil {
+				return err
+			}
+
+			rvfield.Set(newval)
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
+
 		}
+
 	}
 
 	return nil
 }
+
+func parseEnvToValue(envval string, fullEnvKey string, rvtype reflect.Type) (reflect.Value, error) {
+	if rvtype == reflect.TypeFor[string]() {
+
+		return reflect.ValueOf(envval), nil
+
+	} else if rvtype == reflect.TypeFor[int]() {
+
+		envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int(envint)), nil
+
+	} else if rvtype == reflect.TypeFor[int64]() {
+
+		envint, err := strconv.ParseInt(envval, 10, 64)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int64(envint)), nil
+
+	} else if rvtype == reflect.TypeFor[int32]() {
+
+		envint, err := strconv.ParseInt(envval, 10, 32)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int32(envint)), nil
+
+	} else if rvtype == reflect.TypeFor[int8]() {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int8(envint)), nil
+
+	} else if rvtype == reflect.TypeFor[time.Duration]() {
+
+		dur, err := timeext.ParseDurationShortString(envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(dur), nil
+
+	} else if rvtype == reflect.TypeOf(time.UnixMilli(0)) {
+
+		tim, err := time.Parse(time.RFC3339Nano, envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(tim), nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeFor[int]()) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+		envcvl := reflect.ValueOf(envint).Convert(rvtype)
+
+		return envcvl, nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeFor[bool]()) {
+
+		if strings.TrimSpace(strings.ToLower(envval)) == "true" {
+			return reflect.ValueOf(true).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "false" {
+			return reflect.ValueOf(false).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "1" {
+			return reflect.ValueOf(true).Convert(rvtype), nil
+		} else if strings.TrimSpace(strings.ToLower(envval)) == "0" {
+			return reflect.ValueOf(false).Convert(rvtype), nil
+		} else {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,bool> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+	} else if rvtype.ConvertibleTo(reflect.TypeFor[string]()) {
+
+		envcvl := reflect.ValueOf(envval).Convert(rvtype)
+		return envcvl, nil
+
+	} else {
+		return reflect.Value{}, errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvtype.Kind().String(), rvtype.String()))
+	}
+}

confext/confParser_extra_test.go

@@ -0,0 +1,390 @@
+package confext
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+	"time"
+)
+
+func TestApplyEnvOverridesPrefix(t *testing.T) {
+	type testdata struct {
+		V1 int    `env:"V1"`
+		V2 string `env:"V2"`
+	}
+
+	data := testdata{V1: 1, V2: "x"}
+
+	t.Setenv("MYAPP_V1", "42")
+	t.Setenv("MYAPP_V2", "hello")
+	t.Setenv("V1", "111")
+	t.Setenv("V2", "noprefix")
+
+	err := ApplyEnvOverrides("MYAPP_", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.V1, 42)
+	tst.AssertEqual(t, data.V2, "hello")
+}
+
+func TestApplyEnvOverridesUnexportedFieldsIgnored(t *testing.T) {
+	type testdata struct {
+		V1 int `env:"TEST_V1"`
+		v2 int `env:"TEST_V2"`
+	}
+
+	data := testdata{V1: 1, v2: 2}
+
+	t.Setenv("TEST_V1", "11")
+	t.Setenv("TEST_V2", "22")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.V1, 11)
+	tst.AssertEqual(t, data.v2, 2)
+}
+
+func TestApplyEnvOverridesNoEnvTagIgnored(t *testing.T) {
+	type testdata struct {
+		V1 int `env:"TEST_V1"`
+		V2 int ``
+	}
+
+	data := testdata{V1: 1, V2: 2}
+
+	t.Setenv("TEST_V1", "11")
+	t.Setenv("V2", "22")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.V1, 11)
+	tst.AssertEqual(t, data.V2, 2)
+}
+
+func TestApplyEnvOverridesDashTagIgnored(t *testing.T) {
+	type testdata struct {
+		V1 int    `env:"TEST_V1"`
+		V2 string `env:"-"`
+	}
+
+	data := testdata{V1: 1, V2: "no"}
+
+	t.Setenv("TEST_V1", "11")
+	t.Setenv("-", "yes")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.V1, 11)
+	tst.AssertEqual(t, data.V2, "no")
+}
+
+func TestApplyEnvOverridesEnvNotSetKeepsValue(t *testing.T) {
+	type testdata struct {
+		V1 int    `env:"NOT_SET_INT_KEY_XYZ"`
+		V2 string `env:"NOT_SET_STR_KEY_XYZ"`
+	}
+
+	data := testdata{V1: 7, V2: "keep"}
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.V1, 7)
+	tst.AssertEqual(t, data.V2, "keep")
+}
+
+func TestApplyEnvOverridesBoolVariants(t *testing.T) {
+	type testdata struct {
+		B1 bool `env:"B1"`
+		B2 bool `env:"B2"`
+		B3 bool `env:"B3"`
+		B4 bool `env:"B4"`
+		B5 bool `env:"B5"`
+		B6 bool `env:"B6"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("B1", "true")
+	t.Setenv("B2", "false")
+	t.Setenv("B3", "1")
+	t.Setenv("B4", "0")
+	t.Setenv("B5", "  TRUE  ")
+	t.Setenv("B6", "FaLsE")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.B1, true)
+	tst.AssertEqual(t, data.B2, false)
+	tst.AssertEqual(t, data.B3, true)
+	tst.AssertEqual(t, data.B4, false)
+	tst.AssertEqual(t, data.B5, true)
+	tst.AssertEqual(t, data.B6, false)
+}
+
+func TestApplyEnvOverridesInvalidIntReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 int `env:"BAD_INT"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_INT", "not_a_number")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid int, got nil")
+	}
+}
+
+func TestApplyEnvOverridesInvalidInt8ReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 int8 `env:"BAD_INT8"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_INT8", "9999")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid int8, got nil")
+	}
+}
+
+func TestApplyEnvOverridesInvalidInt32ReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 int32 `env:"BAD_INT32"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_INT32", "not_an_int32")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid int32, got nil")
+	}
+}
+
+func TestApplyEnvOverridesInvalidInt64ReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 int64 `env:"BAD_INT64"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_INT64", "not_an_int64")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid int64, got nil")
+	}
+}
+
+func TestApplyEnvOverridesInvalidDurationReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 time.Duration `env:"BAD_DUR"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_DUR", "not_a_duration")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid duration, got nil")
+	}
+}
+
+func TestApplyEnvOverridesInvalidTimeReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 time.Time `env:"BAD_TIME"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_TIME", "not_a_time")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid time, got nil")
+	}
+}
+
+func TestApplyEnvOverridesInvalidBoolReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 bool `env:"BAD_BOOL"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("BAD_BOOL", "yesno")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid bool, got nil")
+	}
+}
+
+func TestApplyEnvOverridesUnsupportedTypeReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 []int `env:"UNSUPPORTED"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("UNSUPPORTED", "1,2,3")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for unsupported type, got nil")
+	}
+}
+
+func TestApplyEnvOverridesFloatUnsupportedReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 float64 `env:"UNSUPPORTED_FLOAT"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("UNSUPPORTED_FLOAT", "1.5")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for float64, got nil")
+	}
+}
+
+func TestApplyEnvOverridesPointerInvalidReturnsError(t *testing.T) {
+	type testdata struct {
+		V1 *int `env:"PTR_BAD_INT"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("PTR_BAD_INT", "not_a_number")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error for invalid pointer int, got nil")
+	}
+}
+
+func TestApplyEnvOverridesPointerNotSetStaysNil(t *testing.T) {
+	type testdata struct {
+		V1 *int    `env:"PTR_NOT_SET_KEY_ABC"`
+		V2 *string `env:"PTR_NOT_SET_KEY_DEF"`
+	}
+
+	data := testdata{}
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	if data.V1 != nil {
+		t.Errorf("expected V1 to remain nil, got %v", *data.V1)
+	}
+	if data.V2 != nil {
+		t.Errorf("expected V2 to remain nil, got %v", *data.V2)
+	}
+}
+
+func TestApplyEnvOverridesAliasBool(t *testing.T) {
+	type aliasbool bool
+
+	type testdata struct {
+		V1 aliasbool `env:"ALIAS_BOOL"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("ALIAS_BOOL", "true")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.V1, aliasbool(true))
+}
+
+func TestApplyEnvOverridesNestedRecursiveError(t *testing.T) {
+	type subdata struct {
+		V1 int `env:"V1"`
+	}
+	type testdata struct {
+		Sub subdata `env:"SUB"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("SUB.V1", "not_a_number")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err == nil {
+		t.Errorf("expected error from nested struct invalid value, got nil")
+	}
+}
+
+func TestApplyEnvOverridesTimeFieldInsideStructIsParsed(t *testing.T) {
+	type testdata struct {
+		T time.Time `env:"MYTIME"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("MYTIME", "2023-01-02T03:04:05Z")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.T.Equal(time.Date(2023, 1, 2, 3, 4, 5, 0, time.UTC)), true)
+}
+
+func TestApplyEnvOverridesPointerStringAlias(t *testing.T) {
+	type aliasstr string
+
+	type testdata struct {
+		V1 *aliasstr `env:"PTR_ALIAS_STR"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("PTR_ALIAS_STR", "hello")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	tst.AssertNoErr(t, err)
+
+	if data.V1 == nil {
+		t.Fatalf("expected V1 to be set")
+	}
+	tst.AssertEqual(t, *data.V1, aliasstr("hello"))
+}
+
+func TestApplyEnvOverridesEmptyEnvTagOnSubstruct(t *testing.T) {
+	type subdata struct {
+		V1 int `env:"INNER"`
+	}
+	type testdata struct {
+		Sub subdata `env:""`
+	}
+
+	data := testdata{}
+
+	t.Setenv("INNER", "55")
+
+	err := ApplyEnvOverrides("PRE_", &data, "_")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.Sub.V1, 0)
+
+	t.Setenv("PRE_INNER", "77")
+
+	err = ApplyEnvOverrides("PRE_", &data, "_")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, data.Sub.V1, 77)
+}

confext/confParser_test.go

@@ -1,6 +1,8 @@
 package confext
 
 import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/timeext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
 	"testing"
 	"time"
 )
@@ -40,13 +42,13 @@ func TestApplyEnvOverridesNoop(t *testing.T) {
 
 	output := input
 
-	err := ApplyEnvOverrides(&output)
+	err := ApplyEnvOverrides("", &output, ".")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
 	}
 
-	assertEqual(t, input, output)
+	tst.AssertEqual(t, input, output)
 }
 
 func TestApplyEnvOverridesSimple(t *testing.T) {
@@ -66,6 +68,7 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 		V7 aliasstring   `env:"TEST_V7"`
 		V8 time.Duration `env:"TEST_V8"`
 		V9 time.Time     `env:"TEST_V9"`
+		VA bool          `env:"TEST_VA"`
 	}
 
 	data := testdata{
@@ -80,6 +83,7 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 		V7: "7",
 		V8: 9,
 		V9: time.Unix(1671102873, 0),
+		VA: false,
 	}
 
 	t.Setenv("TEST_V1", "846")
@@ -91,22 +95,175 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 	t.Setenv("TEST_V7", "AAAAAA")
 	t.Setenv("TEST_V8", "1min4s")
 	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+	t.Setenv("TEST_VA", "true")
 
-	err := ApplyEnvOverrides(&data)
+	err := ApplyEnvOverrides("", &data, ".")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
 	}
 
-	assertEqual(t, data.V1, 846)
-	assertEqual(t, data.V2, "hello_world")
-	assertEqual(t, data.V3, 6)
-	assertEqual(t, data.V4, 333)
-	assertEqual(t, data.V5, -937)
-	assertEqual(t, data.V6, 70)
-	assertEqual(t, data.V7, "AAAAAA")
-	assertEqual(t, data.V8, time.Second*64)
-	assertEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.V1, 846)
+	tst.AssertEqual(t, data.V2, "hello_world")
+	tst.AssertEqual(t, data.V3, 6)
+	tst.AssertEqual(t, data.V4, 333)
+	tst.AssertEqual(t, data.V5, -937)
+	tst.AssertEqual(t, data.V6, 70)
+	tst.AssertEqual(t, data.V7, "AAAAAA")
+	tst.AssertEqual(t, data.V8, time.Second*64)
+	tst.AssertEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.VA, true)
+}
+
+func TestApplyEnvOverridesRecursive(t *testing.T) {
+
+	type subdata struct {
+		V1 int           `env:"SUB_V1"`
+		VX string        ``
+		V2 string        `env:"SUB_V2"`
+		V8 time.Duration `env:"SUB_V3"`
+		V9 time.Time     `env:"SUB_V4"`
+	}
+
+	type testdata struct {
+		V1   int     `env:"TEST_V1"`
+		VX   string  ``
+		Sub1 subdata ``
+		Sub2 subdata `env:"TEST_V2"`
+		Sub3 subdata `env:"TEST_V3"`
+		Sub4 subdata `env:""`
+		V5   string  `env:"-"`
+	}
+
+	data := testdata{
+		V1: 1,
+		VX: "2",
+		V5: "no",
+		Sub1: subdata{
+			V1: 3,
+			VX: "4",
+			V2: "5",
+			V8: 6 * time.Second,
+			V9: time.Date(2000, 1, 7, 1, 1, 1, 0, time.UTC),
+		},
+		Sub2: subdata{
+			V1: 8,
+			VX: "9",
+			V2: "10",
+			V8: 11 * time.Second,
+			V9: time.Date(2000, 1, 12, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+		Sub3: subdata{
+			V1: 13,
+			VX: "14",
+			V2: "15",
+			V8: 16 * time.Second,
+			V9: time.Date(2000, 1, 17, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+		Sub4: subdata{
+			V1: 18,
+			VX: "19",
+			V2: "20",
+			V8: 21 * time.Second,
+			V9: time.Date(2000, 1, 22, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+	}
+
+	t.Setenv("TEST_V1", "999")
+	t.Setenv("-", "yes")
+
+	t.Setenv("TEST_V2_SUB_V1", "846")
+	t.Setenv("TEST_V2_SUB_V2", "222_hello_world")
+	t.Setenv("TEST_V2_SUB_V3", "1min4s")
+	t.Setenv("TEST_V2_SUB_V4", "2009-11-10T23:00:00Z")
+
+	t.Setenv("TEST_V3_SUB_V1", "33846")
+	t.Setenv("TEST_V3_SUB_V2", "33_hello_world")
+	t.Setenv("TEST_V3_SUB_V3", "33min4s")
+	t.Setenv("TEST_V3_SUB_V4", "2033-11-10T23:00:00Z")
+
+	t.Setenv("SUB_V1", "11")
+	t.Setenv("SUB_V2", "22")
+	t.Setenv("SUB_V3", "33min")
+	t.Setenv("SUB_V4", "2044-01-01T00:00:00Z")
+
+	err := ApplyEnvOverrides("", &data, "_")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertEqual(t, data.V1, 999)
+	tst.AssertEqual(t, data.VX, "2")
+	tst.AssertEqual(t, data.V5, "no")
+	tst.AssertEqual(t, data.Sub1.V1, 3)
+	tst.AssertEqual(t, data.Sub1.VX, "4")
+	tst.AssertEqual(t, data.Sub1.V2, "5")
+	tst.AssertEqual(t, data.Sub1.V8, time.Second*6)
+	tst.AssertEqual(t, data.Sub1.V9, time.Unix(947206861, 0).UTC())
+	tst.AssertEqual(t, data.Sub2.V1, 846)
+	tst.AssertEqual(t, data.Sub2.VX, "9")
+	tst.AssertEqual(t, data.Sub2.V2, "222_hello_world")
+	tst.AssertEqual(t, data.Sub2.V8, time.Second*64)
+	tst.AssertEqual(t, data.Sub2.V9, time.Unix(1257894000, 0).UTC())
+	tst.AssertEqual(t, data.Sub3.V1, 33846)
+	tst.AssertEqual(t, data.Sub3.VX, "14")
+	tst.AssertEqual(t, data.Sub3.V2, "33_hello_world")
+	tst.AssertEqual(t, data.Sub3.V8, time.Second*1984)
+	tst.AssertEqual(t, data.Sub3.V9, time.Unix(2015276400, 0).UTC())
+	tst.AssertEqual(t, data.Sub4.V1, 11)
+	tst.AssertEqual(t, data.Sub4.VX, "19")
+	tst.AssertEqual(t, data.Sub4.V2, "22")
+	tst.AssertEqual(t, data.Sub4.V8, time.Second*1980)
+	tst.AssertEqual(t, data.Sub4.V9, time.Unix(2335219200, 0).UTC())
+}
+
+func TestApplyEnvOverridesPointer(t *testing.T) {
+
+	type aliasint int
+	type aliasstring string
+
+	type testdata struct {
+		V1 *int           `env:"TEST_V1"`
+		VX *string        ``
+		V2 *string        `env:"TEST_V2"`
+		V3 *int8          `env:"TEST_V3"`
+		V4 *int32         `env:"TEST_V4"`
+		V5 *int64         `env:"TEST_V5"`
+		V6 *aliasint      `env:"TEST_V6"`
+		VY *aliasint      ``
+		V7 *aliasstring   `env:"TEST_V7"`
+		V8 *time.Duration `env:"TEST_V8"`
+		V9 *time.Time     `env:"TEST_V9"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("TEST_V1", "846")
+	t.Setenv("TEST_V2", "hello_world")
+	t.Setenv("TEST_V3", "6")
+	t.Setenv("TEST_V4", "333")
+	t.Setenv("TEST_V5", "-937")
+	t.Setenv("TEST_V6", "070")
+	t.Setenv("TEST_V7", "AAAAAA")
+	t.Setenv("TEST_V8", "1min4s")
+	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+
+	err := ApplyEnvOverrides("", &data, ".")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	tst.AssertDeRefEqual(t, data.V1, 846)
+	tst.AssertDeRefEqual(t, data.V2, "hello_world")
+	tst.AssertDeRefEqual(t, data.V3, 6)
+	tst.AssertDeRefEqual(t, data.V4, 333)
+	tst.AssertDeRefEqual(t, data.V5, -937)
+	tst.AssertDeRefEqual(t, data.V6, 70)
+	tst.AssertDeRefEqual(t, data.V7, "AAAAAA")
+	tst.AssertDeRefEqual(t, data.V8, time.Second*64)
+	tst.AssertDeRefEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
 }
 
 func assertEqual[T comparable](t *testing.T, actual T, expected T) {
@@ -114,3 +271,12 @@ func assertEqual[T comparable](t *testing.T, actual T, expected T) {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
+
+func assertPtrEqual[T comparable](t *testing.T, actual *T, expected T) {
+	if actual == nil {
+		t.Errorf("values differ: Actual: NIL, Expected: '%v'", expected)
+	}
+	if *actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

cryptext/aes.go

@@ -0,0 +1,132 @@
+package cryptext
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"golang.org/x/crypto/scrypt"
+	"io"
+)
+
+// https://stackoverflow.com/a/18819040/1761622
+
+type aesPayload struct {
+	Salt    []byte `json:"s"`
+	IV      []byte `json:"i"`
+	Data    []byte `json:"d"`
+	Rounds  int    `json:"r"`
+	Version uint   `json:"v"`
+}
+
+func EncryptAESSimple(password []byte, data []byte, rounds int) (string, error) {
+
+	salt := make([]byte, 8)
+	_, err := io.ReadFull(rand.Reader, salt)
+	if err != nil {
+		return "", err
+	}
+
+	key, err := scrypt.Key(password, salt, rounds, 8, 1, 32)
+	if err != nil {
+		return "", err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	h := sha256.New()
+	h.Write(data)
+	checksum := h.Sum(nil)
+	if len(checksum) != 32 {
+		return "", errors.New("wrong cs size")
+	}
+
+	ciphertext := make([]byte, 32+len(data))
+
+	iv := make([]byte, aes.BlockSize)
+	_, err = io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return "", err
+	}
+
+	combinedData := make([]byte, 0, 32+len(data))
+	combinedData = append(combinedData, checksum...)
+	combinedData = append(combinedData, data...)
+
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext, combinedData)
+
+	pl := aesPayload{
+		Salt:    salt,
+		IV:      iv,
+		Data:    ciphertext,
+		Version: 1,
+		Rounds:  rounds,
+	}
+
+	jbin, err := json.Marshal(pl)
+	if err != nil {
+		return "", err
+	}
+
+	res := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(jbin)
+
+	return res, nil
+}
+
+func DecryptAESSimple(password []byte, encText string) ([]byte, error) {
+
+	jbin, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(encText)
+	if err != nil {
+		return nil, err
+	}
+
+	var pl aesPayload
+	err = json.Unmarshal(jbin, &pl)
+	if err != nil {
+		return nil, err
+	}
+
+	if pl.Version != 1 {
+		return nil, errors.New("unsupported version")
+	}
+
+	key, err := scrypt.Key(password, pl.Salt, pl.Rounds, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	dest := make([]byte, len(pl.Data))
+
+	cfb := cipher.NewCFBDecrypter(block, pl.IV)
+	cfb.XORKeyStream(dest, pl.Data)
+
+	if len(dest) < 32 {
+		return nil, errors.New("payload too small")
+	}
+
+	chck := dest[:32]
+	data := dest[32:]
+
+	h := sha256.New()
+	h.Write(data)
+	chck2 := h.Sum(nil)
+
+	if !bytes.Equal(chck, chck2) {
+		return nil, errors.New("checksum mismatch")
+	}
+
+	return data, nil
+}

cryptext/aes_additional_test.go

@@ -0,0 +1,139 @@
+package cryptext
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"strings"
+	"testing"
+)
+
+func TestAESSimpleEmptyData(t *testing.T) {
+	pw := []byte("password")
+	enc, err := EncryptAESSimple(pw, []byte{}, 256)
+	tst.AssertNoErr(t, err)
+	tst.AssertNotEqual(t, enc, "")
+
+	dec, err := DecryptAESSimple(pw, enc)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, len(dec), 0)
+}
+
+func TestAESSimpleEmptyPassword(t *testing.T) {
+	pw := []byte{}
+	plain := []byte("some content")
+
+	enc, err := EncryptAESSimple(pw, plain, 256)
+	tst.AssertNoErr(t, err)
+
+	dec, err := DecryptAESSimple(pw, enc)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(dec), string(plain))
+}
+
+func TestAESSimpleWrongPassword(t *testing.T) {
+	plain := []byte("Hello World")
+	enc, err := EncryptAESSimple([]byte("right"), plain, 256)
+	tst.AssertNoErr(t, err)
+
+	_, err = DecryptAESSimple([]byte("wrong"), enc)
+	if err == nil {
+		t.Errorf("expected error when decrypting with wrong password")
+	}
+}
+
+func TestAESSimpleInvalidBase32(t *testing.T) {
+	_, err := DecryptAESSimple([]byte("pw"), "!!!not-base32!!!")
+	if err == nil {
+		t.Errorf("expected error on invalid base32 input")
+	}
+}
+
+func TestAESSimpleInvalidJSON(t *testing.T) {
+	// "AAAAAAAA" decodes to valid base32 but not valid JSON
+	_, err := DecryptAESSimple([]byte("pw"), "AAAAAAAA")
+	if err == nil {
+		t.Errorf("expected error on invalid JSON payload")
+	}
+}
+
+func TestAESSimpleEmptyEncText(t *testing.T) {
+	_, err := DecryptAESSimple([]byte("pw"), "")
+	if err == nil {
+		t.Errorf("expected error on empty text")
+	}
+}
+
+func TestAESSimpleLargeData(t *testing.T) {
+	pw := []byte("hunter12")
+	plain := []byte(strings.Repeat("ABCDEFGHIJ", 1024))
+
+	enc, err := EncryptAESSimple(pw, plain, 256)
+	tst.AssertNoErr(t, err)
+
+	dec, err := DecryptAESSimple(pw, enc)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(dec), string(plain))
+}
+
+func TestAESSimpleBinaryData(t *testing.T) {
+	pw := []byte("hunter12")
+	plain := []byte{0x00, 0x01, 0x02, 0x7F, 0x80, 0xFE, 0xFF, 0x00, 0xAA, 0x55}
+
+	enc, err := EncryptAESSimple(pw, plain, 256)
+	tst.AssertNoErr(t, err)
+
+	dec, err := DecryptAESSimple(pw, enc)
+	tst.AssertNoErr(t, err)
+	tst.AssertArrayEqual(t, dec, plain)
+}
+
+func TestAESSimpleDifferentRoundsForEachCall(t *testing.T) {
+	pw := []byte("hunter12")
+	plain := []byte("Hello")
+
+	enc1, err := EncryptAESSimple(pw, plain, 256)
+	tst.AssertNoErr(t, err)
+
+	enc2, err := EncryptAESSimple(pw, plain, 256)
+	tst.AssertNoErr(t, err)
+
+	// Two separate encrypt calls on same plaintext should differ (random salt + IV)
+	tst.AssertNotEqual(t, enc1, enc2)
+
+	// Both should decrypt back to the same plaintext
+	d1, err := DecryptAESSimple(pw, enc1)
+	tst.AssertNoErr(t, err)
+	d2, err := DecryptAESSimple(pw, enc2)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(d1), string(plain))
+	tst.AssertEqual(t, string(d2), string(plain))
+}
+
+func TestAESSimpleVariableRounds(t *testing.T) {
+	pw := []byte("hunter12")
+	plain := []byte("rounds-test")
+
+	for _, r := range []int{16, 32, 64, 128, 256, 512, 1024} {
+		enc, err := EncryptAESSimple(pw, plain, r)
+		tst.AssertNoErr(t, err)
+		dec, err := DecryptAESSimple(pw, enc)
+		tst.AssertNoErr(t, err)
+		tst.AssertEqual(t, string(dec), string(plain))
+	}
+}
+
+func TestAESSimpleResultIsBase32(t *testing.T) {
+	pw := []byte("hunter12")
+	plain := []byte("Hello World")
+
+	enc, err := EncryptAESSimple(pw, plain, 64)
+	tst.AssertNoErr(t, err)
+
+	for _, c := range enc {
+		isUpper := c >= 'A' && c <= 'Z'
+		isDigit := c >= '2' && c <= '7'
+		if !(isUpper || isDigit) {
+			t.Errorf("non-base32 character %q in output", c)
+			break
+		}
+	}
+}

cryptext/aes_test.go

@@ -0,0 +1,36 @@
+package cryptext
+
+import (
+	"fmt"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestEncryptAESSimple(t *testing.T) {
+
+	pw := []byte("hunter12")
+
+	str1 := []byte("Hello World")
+
+	str2, err := EncryptAESSimple(pw, str1, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%s\n", str2)
+
+	str3, err := DecryptAESSimple(pw, str2)
+	if err != nil {
+		panic(err)
+	}
+
+	tst.AssertEqual(t, string(str1), string(str3))
+
+	str4, err := EncryptAESSimple(pw, str3, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	tst.AssertNotEqual(t, string(str2), string(str4))
+
+}

cryptext/hash_additional_test.go

@@ -0,0 +1,54 @@
+package cryptext
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"strings"
+	"testing"
+)
+
+func TestStrSha256SameAsBytesSha256(t *testing.T) {
+	inputs := []string{"", "a", "Hello World", "lorem ipsum dolor sit amet", "🎉 unicode"}
+	for _, in := range inputs {
+		tst.AssertEqual(t, StrSha256(in), BytesSha256([]byte(in)))
+	}
+}
+
+func TestStrSha256Length(t *testing.T) {
+	// SHA-256 hex output must always be 64 characters
+	tst.AssertEqual(t, len(StrSha256("")), 64)
+	tst.AssertEqual(t, len(StrSha256("x")), 64)
+	tst.AssertEqual(t, len(StrSha256(strings.Repeat("x", 10000))), 64)
+}
+
+func TestStrSha256Deterministic(t *testing.T) {
+	v := "deterministic input"
+	a := StrSha256(v)
+	b := StrSha256(v)
+	tst.AssertEqual(t, a, b)
+}
+
+func TestStrSha256DifferentInputs(t *testing.T) {
+	tst.AssertNotEqual(t, StrSha256("a"), StrSha256("b"))
+	tst.AssertNotEqual(t, StrSha256("Hello"), StrSha256("hello"))
+	tst.AssertNotEqual(t, StrSha256("Hello World"), StrSha256("Hello World "))
+}
+
+func TestStrSha256IsHex(t *testing.T) {
+	out := StrSha256("anything")
+	for _, c := range out {
+		isLowerHex := (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')
+		if !isLowerHex {
+			t.Errorf("non-hex char %q in StrSha256 output", c)
+			return
+		}
+	}
+}
+
+func TestBytesSha256NilSameAsEmpty(t *testing.T) {
+	tst.AssertEqual(t, BytesSha256(nil), BytesSha256([]byte{}))
+}
+
+func TestBytesSha256KnownVectors(t *testing.T) {
+	// "abc" => sha-256 standard vector
+	tst.AssertEqual(t, StrSha256("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+}

cryptext/hash_test.go

@@ -1,25 +1,20 @@
 package cryptext
 
 import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
 func TestStrSha256(t *testing.T) {
-	assertEqual(t, StrSha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	assertEqual(t, StrSha256("0"), "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9")
-	assertEqual(t, StrSha256("80085"), "b3786e141d65638ad8a98173e26b5f6a53c927737b23ff31fb1843937250f44b")
-	assertEqual(t, StrSha256("Hello World"), "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e")
+	tst.AssertEqual(t, StrSha256(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	tst.AssertEqual(t, StrSha256("0"), "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9")
+	tst.AssertEqual(t, StrSha256("80085"), "b3786e141d65638ad8a98173e26b5f6a53c927737b23ff31fb1843937250f44b")
+	tst.AssertEqual(t, StrSha256("Hello World"), "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e")
 }
 
 func TestBytesSha256(t *testing.T) {
-	assertEqual(t, BytesSha256([]byte{}), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	assertEqual(t, BytesSha256([]byte{0}), "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d")
-	assertEqual(t, BytesSha256([]byte{128}), "76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71")
-	assertEqual(t, BytesSha256([]byte{0, 1, 2, 4, 8, 16, 32, 64, 128, 255}), "55016a318ba538e00123c736b2a8b6db368d00e7e25727547655b653e5853603")
-}
-
-func assertEqual(t *testing.T, actual string, expected string) {
-	if actual != expected {
-		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
-	}
+	tst.AssertEqual(t, BytesSha256([]byte{}), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+	tst.AssertEqual(t, BytesSha256([]byte{0}), "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d")
+	tst.AssertEqual(t, BytesSha256([]byte{128}), "76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71")
+	tst.AssertEqual(t, BytesSha256([]byte{0, 1, 2, 4, 8, 16, 32, 64, 128, 255}), "55016a318ba538e00123c736b2a8b6db368d00e7e25727547655b653e5853603")
 }

cryptext/passHash.go

@@ -0,0 +1,437 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/totpext"
+	"golang.org/x/crypto/bcrypt"
+)
+
+const LatestPassHashVersion = 5
+
+// PassHash
+// - [v0]: plaintext password ( `0|...` )                  // simple, used to write PW's directly in DB
+// - [v1]: sha256(plaintext)                               // simple hashing
+// - [v2]: seed | sha256<seed>(plaintext)                  // add seed
+// - [v3]: seed | sha256<seed>(plaintext) | [hex(totp)]    // add TOTP support
+// - [v4]: bcrypt(plaintext) | [hex(totp)]                 // use proper bcrypt
+// - [v5]: bcrypt(sha512(plaintext)) | [hex(totp)]         // hash pw before bcrypt (otherwise max pw-len = 72)
+type PassHash string
+
+func (ph PassHash) Valid() bool {
+	_, _, _, _, _, valid := ph.Data()
+	return valid
+}
+
+func (ph PassHash) HasTOTP() bool {
+	_, _, _, otp, _, _ := ph.Data()
+	return otp
+}
+
+func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bool, _totpsecret []byte, _valid bool) {
+
+	split := strings.Split(string(ph), "|")
+	if len(split) == 0 {
+		return -1, nil, nil, false, nil, false
+	}
+
+	version, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return -1, nil, nil, false, nil, false
+	}
+
+	if version == 0 {
+		if len(split) != 2 {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), nil, []byte(split[1]), false, nil, true
+	}
+
+	if version == 1 {
+		if len(split) != 2 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), nil, payload, false, nil, true
+	}
+
+	if version == 2 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		seed, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[2])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), seed, payload, false, nil, true
+	}
+
+	if version == 3 {
+		if len(split) != 4 {
+			return -1, nil, nil, false, nil, false
+		}
+		seed, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[2])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[3] != "0" {
+			totpsecret, err = hex.DecodeString(split[3])
+			totp = true
+		}
+		return int(version), seed, payload, totp, totpsecret, true
+	}
+
+	if version == 4 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload := []byte(split[1])
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[2] != "0" {
+			totpsecret, err = hex.DecodeString(split[2])
+			totp = true
+		}
+		return int(version), nil, payload, totp, totpsecret, true
+	}
+
+	if version == 5 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload := []byte(split[1])
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[2] != "0" {
+			totpsecret, err = hex.DecodeString(split[2])
+			totp = true
+		}
+		return int(version), nil, payload, totp, totpsecret, true
+	}
+
+	return -1, nil, nil, false, nil, false
+}
+
+func (ph PassHash) Verify(plainpass string, totp *string) bool {
+	version, seed, payload, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return false
+	}
+
+	if hastotp && totp == nil {
+		return false
+	}
+
+	if version == 0 {
+		return langext.ArrEqualsExact([]byte(plainpass), payload)
+	}
+
+	if version == 1 {
+		return langext.ArrEqualsExact(hash256(plainpass), payload)
+	}
+
+	if version == 2 {
+		return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload)
+	}
+
+	if version == 3 {
+		if !hastotp {
+			return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload)
+		} else {
+			return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload) && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	if version == 4 {
+		if !hastotp {
+			return bcrypt.CompareHashAndPassword(payload, []byte(plainpass)) == nil
+		} else {
+			return bcrypt.CompareHashAndPassword(payload, []byte(plainpass)) == nil && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	if version == 5 {
+		if !hastotp {
+			return bcrypt.CompareHashAndPassword(payload, hash512(plainpass)) == nil
+		} else {
+			return bcrypt.CompareHashAndPassword(payload, hash512(plainpass)) == nil && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	return false
+}
+
+func (ph PassHash) NeedsPasswordUpgrade() bool {
+	version, _, _, _, _, valid := ph.Data()
+	return valid && version < LatestPassHashVersion
+}
+
+func (ph PassHash) Upgrade(plainpass string) (PassHash, error) {
+	version, _, _, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid password")
+	}
+	if version == LatestPassHashVersion {
+		return ph, nil
+	}
+	if hastotp {
+		return HashPassword(plainpass, totpsecret)
+	} else {
+		return HashPassword(plainpass, nil)
+	}
+}
+
+func (ph PassHash) ClearTOTP() (PassHash, error) {
+	version, _, _, _, _, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return ph, nil
+	}
+
+	if version == 1 {
+		return ph, nil
+	}
+
+	if version == 2 {
+		return ph, nil
+	}
+
+	if version == 3 {
+		split := strings.Split(string(ph), "|")
+		split[3] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 4 {
+		split := strings.Split(string(ph), "|")
+		split[2] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 5 {
+		split := strings.Split(string(ph), "|")
+		split[2] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) WithTOTP(totpSecret []byte) (PassHash, error) {
+	version, _, _, _, _, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 1 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 2 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 3 {
+		split := strings.Split(string(ph), "|")
+		split[3] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 4 {
+		split := strings.Split(string(ph), "|")
+		split[2] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 5 {
+		split := strings.Split(string(ph), "|")
+		split[2] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) Change(newPlainPass string) (PassHash, error) {
+	version, _, _, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return HashPasswordV0(newPlainPass)
+	}
+
+	if version == 1 {
+		return HashPasswordV1(newPlainPass)
+	}
+
+	if version == 2 {
+		return HashPasswordV2(newPlainPass)
+	}
+
+	if version == 3 {
+		return HashPasswordV3(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	if version == 4 {
+		return HashPasswordV4(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	if version == 5 {
+		return HashPasswordV5(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) String() string {
+	return string(ph)
+}
+
+func (ph PassHash) MarshalJSON() ([]byte, error) {
+	if ph == "" {
+		return json.Marshal("")
+	}
+	return json.Marshal("*****")
+}
+
+func HashPassword(plainpass string, totpSecret []byte) (PassHash, error) {
+	return HashPasswordV5(plainpass, totpSecret)
+}
+
+func HashPasswordV5(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	payload, err := bcrypt.GenerateFromPassword(hash512(plainpass), bcrypt.MinCost)
+	if err != nil {
+		return "", err
+	}
+
+	return PassHash(fmt.Sprintf("5|%s|%s", string(payload), strtotp)), nil
+}
+
+func HashPasswordV4(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	payload, err := bcrypt.GenerateFromPassword([]byte(plainpass), bcrypt.MinCost)
+	if err != nil {
+		return "", err
+	}
+
+	return PassHash(fmt.Sprintf("4|%s|%s", string(payload), strtotp)), nil
+}
+
+func HashPasswordV3(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	seed, err := newSeed()
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash256Seeded(plainpass, seed)
+
+	return PassHash(fmt.Sprintf("3|%s|%s|%s",
+		base64.RawStdEncoding.EncodeToString(seed),
+		base64.RawStdEncoding.EncodeToString(checksum),
+		strtotp)), nil
+}
+
+func HashPasswordV2(plainpass string) (PassHash, error) {
+	seed, err := newSeed()
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash256Seeded(plainpass, seed)
+
+	return PassHash(fmt.Sprintf("2|%s|%s", base64.RawStdEncoding.EncodeToString(seed), base64.RawStdEncoding.EncodeToString(checksum))), nil
+}
+
+func HashPasswordV1(plainpass string) (PassHash, error) {
+	return PassHash(fmt.Sprintf("1|%s", base64.RawStdEncoding.EncodeToString(hash256(plainpass)))), nil
+}
+
+func HashPasswordV0(plainpass string) (PassHash, error) {
+	return PassHash(fmt.Sprintf("0|%s", plainpass)), nil
+}
+
+func hash512(s string) []byte {
+	h := sha512.New()
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func hash256(s string) []byte {
+	h := sha256.New()
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func hash256Seeded(s string, seed []byte) []byte {
+	h := sha256.New()
+	h.Write(seed)
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func newSeed() ([]byte, error) {
+	secret := make([]byte, 32)
+	_, err := rand.Read(secret)
+	if err != nil {
+		return nil, err
+	}
+	return secret, nil
+}

cryptext/passHash_additional_test.go

@@ -0,0 +1,379 @@
+package cryptext
+
+import (
+	"encoding/json"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"strings"
+	"testing"
+)
+
+func TestPassHashInvalidEmpty(t *testing.T) {
+	ph := PassHash("")
+	tst.AssertFalse(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+}
+
+func TestPassHashInvalidGarbage(t *testing.T) {
+	for _, raw := range []string{
+		"garbage",
+		"99|nope",
+		"abc|payload",
+		"3|onlytwo",
+		"4|onlytwo",
+		"5|onlytwo",
+		"2|notbase64!|notbase64!",
+		"1|!!!notbase64!!!",
+		"3|!!notb64|!!notb64|0",
+		"3|abc|!!notb64|0",
+	} {
+		ph := PassHash(raw)
+		if ph.Valid() {
+			t.Errorf("expected %q to be invalid", raw)
+		}
+	}
+}
+
+func TestPassHashVerifyInvalid(t *testing.T) {
+	ph := PassHash("garbage-value")
+	tst.AssertFalse(t, ph.Verify("anything", nil))
+}
+
+func TestPassHashUpgradeInvalid(t *testing.T) {
+	ph := PassHash("garbage-value")
+	_, err := ph.Upgrade("anything")
+	if err == nil {
+		t.Errorf("expected error for invalid PassHash upgrade")
+	}
+}
+
+func TestPassHashStringRoundtrip(t *testing.T) {
+	ph, err := HashPassword("hunter2", nil)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, ph.String(), string(ph))
+}
+
+func TestPassHashMarshalJSONEmpty(t *testing.T) {
+	ph := PassHash("")
+	data, err := json.Marshal(ph)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(data), `""`)
+}
+
+func TestPassHashMarshalJSONMasked(t *testing.T) {
+	ph, err := HashPassword("hunter2", nil)
+	tst.AssertNoErr(t, err)
+	data, err := json.Marshal(ph)
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(data), `"*****"`)
+}
+
+func TestPassHashDataV0(t *testing.T) {
+	ph, err := HashPasswordV0("test123")
+	tst.AssertNoErr(t, err)
+
+	v, seed, payload, hastotp, totpsecret, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, 0)
+	tst.AssertEqual(t, len(seed), 0)
+	tst.AssertEqual(t, string(payload), "test123")
+	tst.AssertFalse(t, hastotp)
+	tst.AssertEqual(t, len(totpsecret), 0)
+}
+
+func TestPassHashDataV1(t *testing.T) {
+	ph, err := HashPasswordV1("test123")
+	tst.AssertNoErr(t, err)
+
+	v, seed, payload, hastotp, _, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, 1)
+	tst.AssertEqual(t, len(seed), 0)
+	tst.AssertEqual(t, len(payload), 32) // sha-256 is 32 bytes
+	tst.AssertFalse(t, hastotp)
+}
+
+func TestPassHashDataV2(t *testing.T) {
+	ph, err := HashPasswordV2("test123")
+	tst.AssertNoErr(t, err)
+
+	v, seed, payload, hastotp, _, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, 2)
+	tst.AssertEqual(t, len(seed), 32)
+	tst.AssertEqual(t, len(payload), 32)
+	tst.AssertFalse(t, hastotp)
+}
+
+func TestPassHashDataV3(t *testing.T) {
+	ph, err := HashPasswordV3("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	v, seed, payload, hastotp, _, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, 3)
+	tst.AssertEqual(t, len(seed), 32)
+	tst.AssertEqual(t, len(payload), 32)
+	tst.AssertFalse(t, hastotp)
+}
+
+func TestPassHashDataV4(t *testing.T) {
+	ph, err := HashPasswordV4("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	v, _, _, hastotp, _, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, 4)
+	tst.AssertFalse(t, hastotp)
+}
+
+func TestPassHashDataV5(t *testing.T) {
+	ph, err := HashPasswordV5("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	v, _, _, hastotp, _, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, 5)
+	tst.AssertFalse(t, hastotp)
+}
+
+func TestPassHashLatestIsV5(t *testing.T) {
+	ph, err := HashPassword("test", nil)
+	tst.AssertNoErr(t, err)
+	v, _, _, _, _, valid := ph.Data()
+	tst.AssertTrue(t, valid)
+	tst.AssertEqual(t, v, LatestPassHashVersion)
+	tst.AssertEqual(t, v, 5)
+}
+
+func TestPassHashUpgradeLatestIsNoop(t *testing.T) {
+	ph, err := HashPassword("test", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	ph2, err := ph.Upgrade("test")
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(ph), string(ph2))
+}
+
+func TestPassHashClearTOTPInvalid(t *testing.T) {
+	_, err := PassHash("garbage").ClearTOTP()
+	if err == nil {
+		t.Errorf("expected error from ClearTOTP on invalid")
+	}
+}
+
+func TestPassHashClearTOTPV0V1V2Noop(t *testing.T) {
+	ph0, _ := HashPasswordV0("x")
+	r0, err := ph0.ClearTOTP()
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(r0), string(ph0))
+
+	ph1, _ := HashPasswordV1("x")
+	r1, err := ph1.ClearTOTP()
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(r1), string(ph1))
+
+	ph2, _ := HashPasswordV2("x")
+	r2, err := ph2.ClearTOTP()
+	tst.AssertNoErr(t, err)
+	tst.AssertEqual(t, string(r2), string(ph2))
+}
+
+func TestPassHashClearTOTPV3(t *testing.T) {
+	secret := []byte{0x01, 0x02, 0x03}
+	ph, err := HashPasswordV3("test123", secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, ph.HasTOTP())
+
+	cleared, err := ph.ClearTOTP()
+	tst.AssertNoErr(t, err)
+	tst.AssertFalse(t, cleared.HasTOTP())
+	tst.AssertTrue(t, cleared.Valid())
+	tst.AssertTrue(t, cleared.Verify("test123", nil))
+}
+
+func TestPassHashClearTOTPV4(t *testing.T) {
+	secret := []byte{0x01, 0x02, 0x03}
+	ph, err := HashPasswordV4("test123", secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, ph.HasTOTP())
+
+	cleared, err := ph.ClearTOTP()
+	tst.AssertNoErr(t, err)
+	tst.AssertFalse(t, cleared.HasTOTP())
+	tst.AssertTrue(t, cleared.Verify("test123", nil))
+}
+
+func TestPassHashClearTOTPV5(t *testing.T) {
+	secret := []byte{0x01, 0x02, 0x03}
+	ph, err := HashPasswordV5("test123", secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, ph.HasTOTP())
+
+	cleared, err := ph.ClearTOTP()
+	tst.AssertNoErr(t, err)
+	tst.AssertFalse(t, cleared.HasTOTP())
+	tst.AssertTrue(t, cleared.Verify("test123", nil))
+}
+
+func TestPassHashWithTOTPInvalid(t *testing.T) {
+	_, err := PassHash("garbage").WithTOTP([]byte{0x01})
+	if err == nil {
+		t.Errorf("expected error for WithTOTP on invalid")
+	}
+}
+
+func TestPassHashWithTOTPV0V1V2Errors(t *testing.T) {
+	ph0, _ := HashPasswordV0("x")
+	if _, err := ph0.WithTOTP([]byte{0x01}); err == nil {
+		t.Errorf("expected v0 not to support TOTP")
+	}
+	ph1, _ := HashPasswordV1("x")
+	if _, err := ph1.WithTOTP([]byte{0x01}); err == nil {
+		t.Errorf("expected v1 not to support TOTP")
+	}
+	ph2, _ := HashPasswordV2("x")
+	if _, err := ph2.WithTOTP([]byte{0x01}); err == nil {
+		t.Errorf("expected v2 not to support TOTP")
+	}
+}
+
+func TestPassHashWithTOTPV3V4V5(t *testing.T) {
+	secret := []byte{0xDE, 0xAD, 0xBE, 0xEF}
+
+	ph3, _ := HashPasswordV3("pw", nil)
+	tst.AssertFalse(t, ph3.HasTOTP())
+	r3, err := ph3.WithTOTP(secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, r3.HasTOTP())
+
+	ph4, _ := HashPasswordV4("pw", nil)
+	tst.AssertFalse(t, ph4.HasTOTP())
+	r4, err := ph4.WithTOTP(secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, r4.HasTOTP())
+
+	ph5, _ := HashPasswordV5("pw", nil)
+	tst.AssertFalse(t, ph5.HasTOTP())
+	r5, err := ph5.WithTOTP(secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, r5.HasTOTP())
+}
+
+func TestPassHashChangeInvalid(t *testing.T) {
+	_, err := PassHash("garbage").Change("new-pw")
+	if err == nil {
+		t.Errorf("expected error from Change on invalid")
+	}
+}
+
+func TestPassHashChangeKeepsVersion(t *testing.T) {
+	cases := []struct {
+		name    string
+		hashed  func() (PassHash, error)
+		version int
+	}{
+		{"V0", func() (PassHash, error) { return HashPasswordV0("old") }, 0},
+		{"V1", func() (PassHash, error) { return HashPasswordV1("old") }, 1},
+		{"V2", func() (PassHash, error) { return HashPasswordV2("old") }, 2},
+		{"V3", func() (PassHash, error) { return HashPasswordV3("old", nil) }, 3},
+		{"V4", func() (PassHash, error) { return HashPasswordV4("old", nil) }, 4},
+		{"V5", func() (PassHash, error) { return HashPasswordV5("old", nil) }, 5},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			ph, err := c.hashed()
+			tst.AssertNoErr(t, err)
+
+			changed, err := ph.Change("new-pw")
+			tst.AssertNoErr(t, err)
+
+			v, _, _, _, _, valid := changed.Data()
+			tst.AssertTrue(t, valid)
+			tst.AssertEqual(t, v, c.version)
+
+			tst.AssertTrue(t, changed.Verify("new-pw", nil))
+			tst.AssertFalse(t, changed.Verify("old", nil))
+		})
+	}
+}
+
+func TestPassHashChangeKeepsTOTPV3(t *testing.T) {
+	secret := []byte{0xAB, 0xCD}
+	ph, err := HashPasswordV3("old", secret)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, ph.HasTOTP())
+
+	changed, err := ph.Change("new")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, changed.HasTOTP())
+}
+
+func TestPassHashV0Format(t *testing.T) {
+	ph, err := HashPasswordV0("plaintext-pw")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.HasPrefix(string(ph), "0|"))
+	tst.AssertEqual(t, string(ph), "0|plaintext-pw")
+}
+
+func TestPassHashV1Format(t *testing.T) {
+	ph, err := HashPasswordV1("test")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.HasPrefix(string(ph), "1|"))
+}
+
+func TestPassHashV2Format(t *testing.T) {
+	ph, err := HashPasswordV2("test")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.HasPrefix(string(ph), "2|"))
+	tst.AssertEqual(t, strings.Count(string(ph), "|"), 2)
+}
+
+func TestPassHashV3Format(t *testing.T) {
+	ph, err := HashPasswordV3("test", nil)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.HasPrefix(string(ph), "3|"))
+	tst.AssertEqual(t, strings.Count(string(ph), "|"), 3)
+	tst.AssertTrue(t, strings.HasSuffix(string(ph), "|0"))
+}
+
+func TestPassHashV4Format(t *testing.T) {
+	ph, err := HashPasswordV4("test", nil)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.HasPrefix(string(ph), "4|"))
+	tst.AssertTrue(t, strings.HasSuffix(string(ph), "|0"))
+}
+
+func TestPassHashV5Format(t *testing.T) {
+	ph, err := HashPasswordV5("test", nil)
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, strings.HasPrefix(string(ph), "5|"))
+	tst.AssertTrue(t, strings.HasSuffix(string(ph), "|0"))
+}
+
+func TestPassHashV5VerifyLongPassword(t *testing.T) {
+	// V5 hashes via sha512 first → bcrypt's 72-byte limit shouldn't apply
+	longPw := strings.Repeat("a", 200)
+	ph, err := HashPasswordV5(longPw, nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Verify(longPw, nil))
+	tst.AssertFalse(t, ph.Verify(longPw+"x", nil))
+}
+
+func TestPassHashV5DifferentEachCall(t *testing.T) {
+	ph1, err := HashPasswordV5("samepw", nil)
+	tst.AssertNoErr(t, err)
+	ph2, err := HashPasswordV5("samepw", nil)
+	tst.AssertNoErr(t, err)
+
+	// Bcrypt salts internally — same password should produce different hashes
+	tst.AssertNotEqual(t, string(ph1), string(ph2))
+
+	// Both must verify
+	tst.AssertTrue(t, ph1.Verify("samepw", nil))
+	tst.AssertTrue(t, ph2.Verify("samepw", nil))
+}

cryptext/passHash_test.go

@@ -0,0 +1,209 @@
+package cryptext
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/totpext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestPassHash1(t *testing.T) {
+	ph, err := HashPassword("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashTOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPassword("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", new(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashUpgrade_V0(t *testing.T) {
+	ph, err := HashPasswordV0("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V1(t *testing.T) {
+	ph, err := HashPasswordV1("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V2(t *testing.T) {
+	ph, err := HashPasswordV2("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V3(t *testing.T) {
+	ph, err := HashPasswordV3("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V3_TOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPasswordV3("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", new(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", new(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}
+
+func TestPassHashUpgrade_V4(t *testing.T) {
+	ph, err := HashPasswordV4("test123", nil)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertFalse(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertTrue(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+}
+
+func TestPassHashUpgrade_V4_TOTP(t *testing.T) {
+	sec, err := totpext.GenerateSecret()
+	tst.AssertNoErr(t, err)
+
+	ph, err := HashPasswordV4("test123", sec)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertTrue(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", new(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+
+	ph, err = ph.Upgrade("test123")
+	tst.AssertNoErr(t, err)
+
+	tst.AssertTrue(t, ph.Valid())
+	tst.AssertTrue(t, ph.HasTOTP())
+	tst.AssertFalse(t, ph.NeedsPasswordUpgrade())
+
+	tst.AssertFalse(t, ph.Verify("test123", nil))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+	tst.AssertTrue(t, ph.Verify("test123", new(totpext.TOTP(sec))))
+	tst.AssertFalse(t, ph.Verify("test124", nil))
+}

cryptext/pronouncablePassword.go

@@ -0,0 +1,263 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"io"
+	"math/big"
+	mathrand "math/rand"
+	"strings"
+)
+
+const (
+	ppStartChar            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppEndChar              = "ABDEFIKMNORSTUXYZ"
+	ppVowel                = "AEIOUY"
+	ppConsonant            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppSegmentLenMin        = 3
+	ppSegmentLenMax        = 7
+	ppMaxRepeatedVowel     = 2
+	ppMaxRepeatedConsonant = 2
+)
+
+var ppContinuation = map[uint8]string{
+	'A': "BCDFGHJKLMNPRSTVWXYZ",
+	'B': "ADFIKLMNORSTUY",
+	'C': "AEIKOUY",
+	'D': "AEILORSUYZ",
+	'E': "BCDFGHJKLMNPRSTVWXYZ",
+	'F': "ADEGIKLOPRTUY",
+	'G': "ABDEFHILMNORSTUY",
+	'H': "AEIOUY",
+	'I': "BCDFGHJKLMNPRSTVWXZ",
+	'J': "AEIOUY",
+	'K': "ADEFHILMNORSTUY",
+	'L': "ADEFGIJKMNOPSTUVWYZ",
+	'M': "ABEFIKOPSTUY",
+	'N': "ABEFIKOPSTUY",
+	'O': "BCDFGHJKLMNPRSTVWXYZ",
+	'P': "AEFIJLORSTUY",
+	'Q': "AEIOUY",
+	'R': "ADEFGHIJKLMNOPSTUVYZ",
+	'S': "ACDEIKLOPTUYZ",
+	'T': "AEHIJOPRSUWY",
+	'U': "BCDFGHJKLMNPRSTVWXZ",
+	'V': "AEIOUY",
+	'W': "AEIOUY",
+	'X': "AEIOUY",
+	'Y': "ABCDFGHKLMNPRSTVXZ",
+	'Z': "AEILOTUY",
+}
+
+var ppLog2Map = map[int]float64{
+	1:  0.00000000,
+	2:  1.00000000,
+	3:  1.58496250,
+	4:  2.00000000,
+	5:  2.32192809,
+	6:  2.58496250,
+	7:  2.80735492,
+	8:  3.00000000,
+	9:  3.16992500,
+	10: 3.32192809,
+	11: 3.45943162,
+	12: 3.58496250,
+	13: 3.70043972,
+	14: 3.80735492,
+	15: 3.90689060,
+	16: 4.00000000,
+	17: 4.08746284,
+	18: 4.16992500,
+	19: 4.24792751,
+	20: 4.32192809,
+	21: 4.39231742,
+	22: 4.45943162,
+	23: 4.52356196,
+	24: 4.58496250,
+	25: 4.64385619,
+	26: 4.70043972,
+	27: 4.75488750,
+	28: 4.80735492,
+	29: 4.85798100,
+	30: 4.90689060,
+	31: 4.95419631,
+	32: 5.00000000,
+}
+
+var (
+	ppVowelMap     = ppMakeSet(ppVowel)
+	ppConsonantMap = ppMakeSet(ppConsonant)
+	ppEndCharMap   = ppMakeSet(ppEndChar)
+)
+
+func ppMakeSet(v string) map[uint8]bool {
+	mp := make(map[uint8]bool, len(v))
+	for _, chr := range v {
+		mp[uint8(chr)] = true
+	}
+	return mp
+}
+
+func ppRandInt(rng io.Reader, max int) int {
+	v, err := rand.Int(rng, big.NewInt(int64(max)))
+	if err != nil {
+		panic(err)
+	}
+	return int(v.Int64())
+}
+
+func ppRand(rng io.Reader, chars string, entropy *float64) uint8 {
+	chr := chars[ppRandInt(rng, len(chars))]
+
+	*entropy = *entropy + ppLog2Map[len(chars)]
+
+	return chr
+}
+
+func ppCharType(chr uint8) (bool, bool) {
+	_, ok1 := ppVowelMap[chr]
+	_, ok2 := ppConsonantMap[chr]
+
+	return ok1, ok2
+}
+
+func ppCharsetRemove(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; !ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func ppCharsetFilter(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func PronouncablePasswordExt(rng io.Reader, pwlen int) (string, float64) {
+
+	// kinda pseudo markov-chain - with a few extra rules and no weights...
+
+	if pwlen <= 0 {
+		return "", 0
+	}
+
+	vowelCount := 0
+	consoCount := 0
+	entropy := float64(0)
+
+	startChar := ppRand(rng, ppStartChar, &entropy)
+
+	result := string(startChar)
+	currentChar := startChar
+
+	isVowel, isConsonant := ppCharType(currentChar)
+	if isVowel {
+		vowelCount = 1
+	}
+	if isConsonant {
+		consoCount = ppMaxRepeatedConsonant
+	}
+
+	segmentLen := 1
+
+	segmentLenTarget := ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+
+	for len(result) < pwlen {
+
+		charset := ppContinuation[currentChar]
+		if vowelCount >= ppMaxRepeatedVowel {
+			charset = ppCharsetRemove(charset, ppVowelMap, false)
+		}
+		if consoCount >= ppMaxRepeatedConsonant {
+			charset = ppCharsetRemove(charset, ppConsonantMap, false)
+		}
+
+		lastOfSegment := false
+		newSegment := false
+
+		if len(result)+1 == pwlen {
+			// last of result
+			charset = ppCharsetFilter(charset, ppEndCharMap, false)
+		} else if segmentLen+1 == segmentLenTarget {
+			// last of segment
+			charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+			if charsetNew != "" {
+				charset = charsetNew
+				lastOfSegment = true
+			}
+		} else if segmentLen >= segmentLenTarget {
+			// (perhaps) start of new segment
+			if _, ok := ppEndCharMap[currentChar]; ok {
+				charset = ppStartChar
+				newSegment = true
+			} else {
+				// continue segment for one more char to (hopefully) find an end-char
+				charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+				if charsetNew != "" {
+					charset = charsetNew
+					lastOfSegment = true
+				}
+			}
+		} else {
+			// normal continuation
+		}
+
+		newChar := ppRand(rng, charset, &entropy)
+		if lastOfSegment {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		} else if newSegment {
+			currentChar = newChar
+			segmentLen = 1
+			result += strings.ToUpper(string(newChar))
+			segmentLenTarget = ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+			vowelCount = 0
+			consoCount = 0
+		} else {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		}
+
+		isVowel, isConsonant := ppCharType(currentChar)
+		if isVowel {
+			vowelCount++
+			consoCount = 0
+		}
+		if isConsonant {
+			vowelCount = 0
+			if newSegment {
+				consoCount = ppMaxRepeatedConsonant
+			} else {
+				consoCount++
+			}
+		}
+	}
+
+	return result, entropy
+}
+
+func PronouncablePassword(len int) string {
+	v, _ := PronouncablePasswordExt(rand.Reader, len)
+	return v
+}
+
+func PronouncablePasswordSeeded(seed int64, len int) string {
+
+	v, _ := PronouncablePasswordExt(mathrand.New(mathrand.NewSource(seed)), len)
+	return v
+}

cryptext/pronouncablePassword_additional_test.go

@@ -0,0 +1,180 @@
+package cryptext
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	mathrand "math/rand"
+	"strings"
+	"testing"
+	"unicode"
+)
+
+func TestPronouncablePasswordLength(t *testing.T) {
+	for _, n := range []int{1, 2, 3, 5, 8, 13, 21, 50, 128} {
+		pw := PronouncablePassword(n)
+		tst.AssertEqual(t, len(pw), n)
+	}
+}
+
+func TestPronouncablePasswordZeroOrNegative(t *testing.T) {
+	tst.AssertEqual(t, PronouncablePassword(0), "")
+	tst.AssertEqual(t, PronouncablePassword(-1), "")
+	tst.AssertEqual(t, PronouncablePassword(-1000), "")
+}
+
+func TestPronouncablePasswordSeededDeterministic(t *testing.T) {
+	pw1 := PronouncablePasswordSeeded(42, 16)
+	pw2 := PronouncablePasswordSeeded(42, 16)
+	tst.AssertEqual(t, pw1, pw2)
+	tst.AssertEqual(t, len(pw1), 16)
+}
+
+func TestPronouncablePasswordSeededDifferentSeeds(t *testing.T) {
+	pw1 := PronouncablePasswordSeeded(1, 16)
+	pw2 := PronouncablePasswordSeeded(2, 16)
+	tst.AssertNotEqual(t, pw1, pw2)
+}
+
+func TestPronouncablePasswordExtEntropy(t *testing.T) {
+	rng := mathrand.New(mathrand.NewSource(1))
+	pw, entropy := PronouncablePasswordExt(rng, 32)
+	tst.AssertEqual(t, len(pw), 32)
+	if entropy <= 0 {
+		t.Errorf("expected positive entropy, got %f", entropy)
+	}
+}
+
+func TestPronouncablePasswordExtZeroLen(t *testing.T) {
+	rng := mathrand.New(mathrand.NewSource(1))
+	pw, entropy := PronouncablePasswordExt(rng, 0)
+	tst.AssertEqual(t, pw, "")
+	tst.AssertEqual(t, entropy, float64(0))
+}
+
+func TestPronouncablePasswordCharacters(t *testing.T) {
+	// Output should be only ASCII letters
+	for i := range 50 {
+		pw := PronouncablePasswordSeeded(int64(i), 32)
+		for _, c := range pw {
+			if !unicode.IsLetter(c) || c > unicode.MaxASCII {
+				t.Errorf("non-letter or non-ASCII rune %q in password %q", c, pw)
+				break
+			}
+		}
+	}
+}
+
+func TestPronouncablePasswordStartsUpper(t *testing.T) {
+	for i := range 50 {
+		pw := PronouncablePasswordSeeded(int64(i), 16)
+		if pw == "" {
+			continue
+		}
+		first := rune(pw[0])
+		if !unicode.IsUpper(first) {
+			t.Errorf("expected first letter uppercase in %q (seed %d)", pw, i)
+		}
+		if !strings.ContainsRune(ppStartChar, first) {
+			t.Errorf("expected first letter from start-set in %q (seed %d)", pw, i)
+		}
+	}
+}
+
+func TestPpMakeSet(t *testing.T) {
+	set := ppMakeSet("ABC")
+	tst.AssertTrue(t, set['A'])
+	tst.AssertTrue(t, set['B'])
+	tst.AssertTrue(t, set['C'])
+	tst.AssertFalse(t, set['D'])
+	tst.AssertEqual(t, len(set), 3)
+}
+
+func TestPpMakeSetEmpty(t *testing.T) {
+	set := ppMakeSet("")
+	tst.AssertEqual(t, len(set), 0)
+}
+
+func TestPpCharType(t *testing.T) {
+	v, c := ppCharType('A')
+	tst.AssertTrue(t, v)
+	tst.AssertFalse(t, c)
+
+	v, c = ppCharType('B')
+	tst.AssertFalse(t, v)
+	tst.AssertTrue(t, c)
+
+	v, c = ppCharType('Y')
+	tst.AssertTrue(t, v)
+	tst.AssertFalse(t, c)
+
+	v, c = ppCharType('1')
+	tst.AssertFalse(t, v)
+	tst.AssertFalse(t, c)
+}
+
+func TestPpCharsetRemove(t *testing.T) {
+	set := ppMakeSet("AEIOU")
+	out := ppCharsetRemove("ABCDEFG", set, false)
+	tst.AssertEqual(t, out, "BCDFG")
+}
+
+func TestPpCharsetRemoveEmptyDisallowed(t *testing.T) {
+	set := ppMakeSet("AB")
+	out := ppCharsetRemove("AB", set, false)
+	// when result would be empty and allowEmpty=false, it returns the original
+	tst.AssertEqual(t, out, "AB")
+}
+
+func TestPpCharsetRemoveEmptyAllowed(t *testing.T) {
+	set := ppMakeSet("AB")
+	out := ppCharsetRemove("AB", set, true)
+	tst.AssertEqual(t, out, "")
+}
+
+func TestPpCharsetFilter(t *testing.T) {
+	set := ppMakeSet("AEIOU")
+	out := ppCharsetFilter("ABCDEFG", set, false)
+	tst.AssertEqual(t, out, "AE")
+}
+
+func TestPpCharsetFilterEmptyDisallowed(t *testing.T) {
+	set := ppMakeSet("XYZ")
+	out := ppCharsetFilter("ABC", set, false)
+	tst.AssertEqual(t, out, "ABC") // returns original when result empty & not allowed
+}
+
+func TestPpCharsetFilterEmptyAllowed(t *testing.T) {
+	set := ppMakeSet("XYZ")
+	out := ppCharsetFilter("ABC", set, true)
+	tst.AssertEqual(t, out, "")
+}
+
+func TestPronouncablePasswordContinuationFollowsRules(t *testing.T) {
+	// Make sure each continuation pair (lowercased) appears in ppContinuation
+	// Note: when a new segment starts (uppercase letter mid-string), the continuation
+	// check does not apply across the segment boundary.
+	for s := range 30 {
+		seed := int64(s)
+		pw := PronouncablePasswordSeeded(seed, 32)
+		if len(pw) < 2 {
+			continue
+		}
+		runes := []byte(strings.ToUpper(pw))
+		for i := 1; i < len(runes); i++ {
+			// Detect new segment (original char was uppercase and it's not the first char)
+			origUpper := pw[i] >= 'A' && pw[i] <= 'Z'
+			if origUpper && i > 0 {
+				continue
+			}
+			prev := runes[i-1]
+			cur := runes[i]
+			cont, ok := ppContinuation[prev]
+			if !ok {
+				t.Errorf("no continuation map for %q (pw=%q)", prev, pw)
+				continue
+			}
+			if !strings.ContainsRune(cont, rune(cur)) {
+				t.Errorf("invalid continuation %q -> %q in %q (seed %d)", prev, cur, pw, seed)
+			}
+		}
+	}
+}

cryptext/pronouncablePassword_test.go

@@ -0,0 +1,35 @@
+package cryptext
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestPronouncablePasswordExt(t *testing.T) {
+	for i := range 20 {
+		pw, entropy := PronouncablePasswordExt(rand.New(rand.NewSource(int64(i))), 16)
+		fmt.Printf("[%.2f] => %s\n", entropy, pw)
+	}
+}
+
+func TestPronouncablePasswordSeeded(t *testing.T) {
+	for i := range 20 {
+		pw := PronouncablePasswordSeeded(int64(i), 8)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePassword(t *testing.T) {
+	for i := range 20 {
+		pw := PronouncablePassword(i + 1)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePasswordWrongLen(t *testing.T) {
+	PronouncablePassword(0)
+	PronouncablePassword(-1)
+	PronouncablePassword(-2)
+	PronouncablePassword(-3)
+}

ctxext/getter.go

@@ -0,0 +1,27 @@
+package ctxext
+
+import "context"
+
+func Value[T any](ctx context.Context, key any) (T, bool) {
+	v := ctx.Value(key)
+	if v == nil {
+		return *new(T), false
+	}
+	if tv, ok := v.(T); !ok {
+		return *new(T), false
+	} else {
+		return tv, true
+	}
+}
+
+func ValueOrDefault[T any](ctx context.Context, key any, def T) T {
+	v := ctx.Value(key)
+	if v == nil {
+		return def
+	}
+	if tv, ok := v.(T); !ok {
+		return def
+	} else {
+		return tv
+	}
+}

ctxext/getter_test.go

@@ -0,0 +1,237 @@
+package ctxext
+
+import (
+	"context"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+type ctxKey string
+
+const (
+	keyString ctxKey = "string-key"
+	keyInt    ctxKey = "int-key"
+	keyStruct ctxKey = "struct-key"
+	keyPtr    ctxKey = "ptr-key"
+	keyMissing ctxKey = "missing-key"
+)
+
+type sampleStruct struct {
+	Name string
+	N    int
+}
+
+func TestValueStringPresent(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "hello")
+	v, ok := Value[string](ctx, keyString)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v, "hello")
+}
+
+func TestValueIntPresent(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyInt, 42)
+	v, ok := Value[int](ctx, keyInt)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v, 42)
+}
+
+func TestValueStructPresent(t *testing.T) {
+	want := sampleStruct{Name: "abc", N: 7}
+	ctx := context.WithValue(context.Background(), keyStruct, want)
+	v, ok := Value[sampleStruct](ctx, keyStruct)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v.Name, "abc")
+	tst.AssertEqual(t, v.N, 7)
+}
+
+func TestValuePointerPresent(t *testing.T) {
+	want := &sampleStruct{Name: "ptr", N: 99}
+	ctx := context.WithValue(context.Background(), keyPtr, want)
+	v, ok := Value[*sampleStruct](ctx, keyPtr)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v == want, true)
+	tst.AssertEqual(t, v.Name, "ptr")
+}
+
+func TestValueMissing(t *testing.T) {
+	ctx := context.Background()
+	v, ok := Value[string](ctx, keyMissing)
+	tst.AssertEqual(t, ok, false)
+	tst.AssertEqual(t, v, "")
+}
+
+func TestValueMissingInt(t *testing.T) {
+	ctx := context.Background()
+	v, ok := Value[int](ctx, keyMissing)
+	tst.AssertEqual(t, ok, false)
+	tst.AssertEqual(t, v, 0)
+}
+
+func TestValueMissingStruct(t *testing.T) {
+	ctx := context.Background()
+	v, ok := Value[sampleStruct](ctx, keyMissing)
+	tst.AssertEqual(t, ok, false)
+	tst.AssertEqual(t, v.Name, "")
+	tst.AssertEqual(t, v.N, 0)
+}
+
+func TestValueMissingPointer(t *testing.T) {
+	ctx := context.Background()
+	v, ok := Value[*sampleStruct](ctx, keyMissing)
+	tst.AssertEqual(t, ok, false)
+	tst.AssertEqual(t, v == nil, true)
+}
+
+func TestValueWrongType(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "hello")
+	v, ok := Value[int](ctx, keyString)
+	tst.AssertEqual(t, ok, false)
+	tst.AssertEqual(t, v, 0)
+}
+
+func TestValueWrongTypeStructToString(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyStruct, sampleStruct{Name: "x"})
+	v, ok := Value[string](ctx, keyStruct)
+	tst.AssertEqual(t, ok, false)
+	tst.AssertEqual(t, v, "")
+}
+
+func TestValueNilStoredAsInterface(t *testing.T) {
+	var stored *sampleStruct = nil
+	ctx := context.WithValue(context.Background(), keyPtr, stored)
+	v, ok := Value[*sampleStruct](ctx, keyPtr)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v == nil, true)
+}
+
+func TestValueEmptyString(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "")
+	v, ok := Value[string](ctx, keyString)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v, "")
+}
+
+func TestValueZeroInt(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyInt, 0)
+	v, ok := Value[int](ctx, keyInt)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v, 0)
+}
+
+func TestValueWithStringKey(t *testing.T) {
+	type stringKey string
+	k := stringKey("my-key")
+	ctx := context.WithValue(context.Background(), k, "value")
+	v, ok := Value[string](ctx, k)
+	tst.AssertEqual(t, ok, true)
+	tst.AssertEqual(t, v, "value")
+}
+
+func TestValueOrDefaultPresent(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "hello")
+	v := ValueOrDefault(ctx, keyString, "default")
+	tst.AssertEqual(t, v, "hello")
+}
+
+func TestValueOrDefaultIntPresent(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyInt, 42)
+	v := ValueOrDefault(ctx, keyInt, -1)
+	tst.AssertEqual(t, v, 42)
+}
+
+func TestValueOrDefaultMissing(t *testing.T) {
+	ctx := context.Background()
+	v := ValueOrDefault(ctx, keyMissing, "default")
+	tst.AssertEqual(t, v, "default")
+}
+
+func TestValueOrDefaultMissingInt(t *testing.T) {
+	ctx := context.Background()
+	v := ValueOrDefault(ctx, keyMissing, 99)
+	tst.AssertEqual(t, v, 99)
+}
+
+func TestValueOrDefaultMissingStruct(t *testing.T) {
+	ctx := context.Background()
+	def := sampleStruct{Name: "default", N: 1}
+	v := ValueOrDefault(ctx, keyMissing, def)
+	tst.AssertEqual(t, v.Name, "default")
+	tst.AssertEqual(t, v.N, 1)
+}
+
+func TestValueOrDefaultWrongType(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "hello")
+	v := ValueOrDefault(ctx, keyString, 7)
+	tst.AssertEqual(t, v, 7)
+}
+
+func TestValueOrDefaultWrongTypeStruct(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyStruct, sampleStruct{Name: "x"})
+	def := "fallback"
+	v := ValueOrDefault(ctx, keyStruct, def)
+	tst.AssertEqual(t, v, "fallback")
+}
+
+func TestValueOrDefaultEmptyStringStored(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "")
+	v := ValueOrDefault(ctx, keyString, "default")
+	tst.AssertEqual(t, v, "")
+}
+
+func TestValueOrDefaultZeroIntStored(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyInt, 0)
+	v := ValueOrDefault(ctx, keyInt, 99)
+	tst.AssertEqual(t, v, 0)
+}
+
+func TestValueOrDefaultPointerPresent(t *testing.T) {
+	want := &sampleStruct{Name: "p", N: 5}
+	ctx := context.WithValue(context.Background(), keyPtr, want)
+	def := &sampleStruct{Name: "def", N: 0}
+	v := ValueOrDefault(ctx, keyPtr, def)
+	tst.AssertEqual(t, v == want, true)
+}
+
+func TestValueOrDefaultPointerMissing(t *testing.T) {
+	ctx := context.Background()
+	def := &sampleStruct{Name: "def", N: 0}
+	v := ValueOrDefault(ctx, keyMissing, def)
+	tst.AssertEqual(t, v == def, true)
+}
+
+func TestValueOrDefaultNilPointerStored(t *testing.T) {
+	var stored *sampleStruct = nil
+	ctx := context.WithValue(context.Background(), keyPtr, stored)
+	def := &sampleStruct{Name: "def"}
+	v := ValueOrDefault(ctx, keyPtr, def)
+	tst.AssertEqual(t, v == nil, true)
+}
+
+func TestValueNestedContext(t *testing.T) {
+	ctx := context.WithValue(context.Background(), keyString, "outer")
+	ctx = context.WithValue(ctx, keyInt, 123)
+	ctx = context.WithValue(ctx, keyString, "inner")
+
+	vs, oks := Value[string](ctx, keyString)
+	tst.AssertEqual(t, oks, true)
+	tst.AssertEqual(t, vs, "inner")
+
+	vi, oki := Value[int](ctx, keyInt)
+	tst.AssertEqual(t, oki, true)
+	tst.AssertEqual(t, vi, 123)
+}
+
+func TestValueDifferentKeyTypesDoNotCollide(t *testing.T) {
+	type keyA string
+	type keyB string
+	ctx := context.WithValue(context.Background(), keyA("k"), "a-val")
+	ctx = context.WithValue(ctx, keyB("k"), "b-val")
+
+	va, oka := Value[string](ctx, keyA("k"))
+	tst.AssertEqual(t, oka, true)
+	tst.AssertEqual(t, va, "a-val")
+
+	vb, okb := Value[string](ctx, keyB("k"))
+	tst.AssertEqual(t, okb, true)
+	tst.AssertEqual(t, vb, "b-val")
+}

cursortoken/direction.go

@@ -0,0 +1,18 @@
+package cursortoken
+
+type SortDirection string //@enum:type
+
+const (
+	SortASC  SortDirection = "ASC"
+	SortDESC SortDirection = "DESC"
+)
+
+func (sd SortDirection) ToMongo() int {
+	if sd == SortASC {
+		return 1
+	} else if sd == SortDESC {
+		return -1
+	} else {
+		return 0
+	}
+}

cursortoken/direction_test.go

@@ -0,0 +1,29 @@
+package cursortoken
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestSortDirectionToMongoASC(t *testing.T) {
+	tst.AssertEqual(t, SortASC.ToMongo(), 1)
+}
+
+func TestSortDirectionToMongoDESC(t *testing.T) {
+	tst.AssertEqual(t, SortDESC.ToMongo(), -1)
+}
+
+func TestSortDirectionToMongoEmpty(t *testing.T) {
+	var sd SortDirection
+	tst.AssertEqual(t, sd.ToMongo(), 0)
+}
+
+func TestSortDirectionToMongoUnknown(t *testing.T) {
+	sd := SortDirection("xyz")
+	tst.AssertEqual(t, sd.ToMongo(), 0)
+}
+
+func TestSortDirectionConstants(t *testing.T) {
+	tst.AssertEqual(t, string(SortASC), "ASC")
+	tst.AssertEqual(t, string(SortDESC), "DESC")
+}

cursortoken/filter.go

@@ -0,0 +1,16 @@
+package cursortoken
+
+import (
+	"context"
+
+	"go.mongodb.org/mongo-driver/v2/mongo"
+)
+
+type RawFilter interface {
+	FilterQuery(ctx context.Context) mongo.Pipeline
+}
+
+type Filter interface {
+	FilterQuery(ctx context.Context) mongo.Pipeline
+	Pagination(ctx context.Context) (string, SortDirection, string, SortDirection)
+}

cursortoken/token.go

@@ -0,0 +1,96 @@
+package cursortoken
+
+import (
+	"encoding/base32"
+	"encoding/json"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/exerr"
+)
+
+type CursorToken interface {
+	Token() string
+	IsStart() bool
+	IsEnd() bool
+}
+
+type Mode string
+
+const (
+	CTMStart  Mode = "START"
+	CTMNormal Mode = "NORMAL"
+	CTMEnd    Mode = "END"
+)
+
+type Extra struct {
+	Timestamp *time.Time
+	Id        *string
+	Page      *int
+	PageSize  *int
+}
+
+func Decode(tok string) (CursorToken, error) {
+	if tok == "" {
+		return Start(), nil
+	}
+	if strings.ToLower(tok) == "@start" {
+		return Start(), nil
+	}
+	if strings.ToLower(tok) == "@end" {
+		return End(), nil
+	}
+	if strings.ToLower(tok) == "$end" {
+		return PageEnd(), nil
+	}
+	if strings.HasPrefix(tok, "$") && len(tok) > 1 {
+		n, err := strconv.ParseInt(tok[1:], 10, 64)
+		if err != nil {
+			return nil, exerr.Wrap(err, "failed to deserialize token").Str("token", tok).WithType(exerr.TypeCursorTokenDecode).Build()
+		}
+		return Page(int(n)), nil
+	}
+
+	if strings.HasPrefix(tok, "tok_") {
+
+		body, err := base32.StdEncoding.DecodeString(tok[len("tok_"):])
+		if err != nil {
+			return nil, err
+		}
+
+		var tokenDeserialize cursorTokenKeySortSerialize
+		err = json.Unmarshal(body, &tokenDeserialize)
+		if err != nil {
+			return nil, exerr.Wrap(err, "failed to deserialize token").Str("token", tok).WithType(exerr.TypeCursorTokenDecode).Build()
+		}
+
+		token := CTKeySort{Mode: CTMNormal}
+
+		if tokenDeserialize.ValuePrimary != nil {
+			token.ValuePrimary = *tokenDeserialize.ValuePrimary
+		}
+		if tokenDeserialize.ValueSecondary != nil {
+			token.ValueSecondary = *tokenDeserialize.ValueSecondary
+		}
+		if tokenDeserialize.Direction != nil {
+			token.Direction = *tokenDeserialize.Direction
+		}
+		if tokenDeserialize.DirectionSecondary != nil {
+			token.DirectionSecondary = *tokenDeserialize.DirectionSecondary
+		}
+		if tokenDeserialize.PageSize != nil {
+			token.PageSize = *tokenDeserialize.PageSize
+		}
+
+		token.Extra.Timestamp = tokenDeserialize.ExtraTimestamp
+		token.Extra.Id = tokenDeserialize.ExtraId
+		token.Extra.Page = tokenDeserialize.ExtraPage
+		token.Extra.PageSize = tokenDeserialize.ExtraPageSize
+
+		return token, nil
+
+	} else {
+		return nil, exerr.New(exerr.TypeCursorTokenDecode, "could not decode token, missing/unknown prefix").Str("token", tok).Build()
+	}
+}

cursortoken/tokenKeySort.go

@@ -0,0 +1,137 @@
+package cursortoken
+
+import (
+	"encoding/base32"
+	"encoding/json"
+	"time"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type CTKeySort struct {
+	Mode               Mode
+	ValuePrimary       string
+	ValueSecondary     string
+	Direction          SortDirection
+	DirectionSecondary SortDirection
+	PageSize           int
+	Extra              Extra
+}
+
+type cursorTokenKeySortSerialize struct {
+	ValuePrimary       *string        `json:"v1,omitempty"`
+	ValueSecondary     *string        `json:"v2,omitempty"`
+	Direction          *SortDirection `json:"dir,omitempty"`
+	DirectionSecondary *SortDirection `json:"dir2,omitempty"`
+	PageSize           *int           `json:"size,omitempty"`
+
+	ExtraTimestamp *time.Time `json:"ts,omitempty"`
+	ExtraId        *string    `json:"id,omitempty"`
+	ExtraPage      *int       `json:"pg,omitempty"`
+	ExtraPageSize  *int       `json:"sz,omitempty"`
+}
+
+func NewKeySortToken(valuePrimary string, valueSecondary string, direction SortDirection, directionSecondary SortDirection, pageSize int, extra Extra) CursorToken {
+	return CTKeySort{
+		Mode:               CTMNormal,
+		ValuePrimary:       valuePrimary,
+		ValueSecondary:     valueSecondary,
+		Direction:          direction,
+		DirectionSecondary: directionSecondary,
+		PageSize:           pageSize,
+		Extra:              extra,
+	}
+}
+
+func Start() CursorToken {
+	return CTKeySort{
+		Mode:               CTMStart,
+		ValuePrimary:       "",
+		ValueSecondary:     "",
+		Direction:          "",
+		DirectionSecondary: "",
+		PageSize:           0,
+		Extra:              Extra{},
+	}
+}
+
+func End() CursorToken {
+	return CTKeySort{
+		Mode:               CTMEnd,
+		ValuePrimary:       "",
+		ValueSecondary:     "",
+		Direction:          "",
+		DirectionSecondary: "",
+		PageSize:           0,
+		Extra:              Extra{},
+	}
+}
+
+func (c CTKeySort) Token() string {
+
+	if c.Mode == CTMStart {
+		return "@start"
+	}
+	if c.Mode == CTMEnd {
+		return "@end"
+	}
+
+	// We kinda manually implement omitempty for the CursorToken here
+	// because omitempty does not work for time.Time and otherwise we would always
+	// get weird time values when decoding a token that initially didn't have an Timestamp set
+	// For this usecase we treat Unix=0 as an empty timestamp
+
+	sertok := cursorTokenKeySortSerialize{}
+
+	if c.ValuePrimary != "" {
+		sertok.ValuePrimary = &c.ValuePrimary
+	}
+	if c.ValueSecondary != "" {
+		sertok.ValueSecondary = &c.ValueSecondary
+	}
+	if c.Direction != "" {
+		sertok.Direction = &c.Direction
+	}
+	if c.DirectionSecondary != "" {
+		sertok.DirectionSecondary = &c.DirectionSecondary
+	}
+	if c.PageSize != 0 {
+		sertok.PageSize = &c.PageSize
+	}
+
+	sertok.ExtraTimestamp = c.Extra.Timestamp
+	sertok.ExtraId = c.Extra.Id
+	sertok.ExtraPage = c.Extra.Page
+	sertok.ExtraPageSize = c.Extra.PageSize
+
+	body, err := json.Marshal(sertok)
+	if err != nil {
+		panic(err)
+	}
+
+	return "tok_" + base32.StdEncoding.EncodeToString(body)
+}
+
+func (c CTKeySort) IsEnd() bool {
+	return c.Mode == CTMEnd
+}
+
+func (c CTKeySort) IsStart() bool {
+	return c.Mode == CTMStart
+}
+
+func (c CTKeySort) valuePrimaryObjectId() (bson.ObjectID, bool) {
+	if oid, err := bson.ObjectIDFromHex(c.ValuePrimary); err == nil {
+		return oid, true
+	} else {
+		return bson.ObjectID{}, false
+	}
+}
+
+func (c CTKeySort) valueSecondaryObjectId() (bson.ObjectID, bool) {
+	if oid, err := bson.ObjectIDFromHex(c.ValueSecondary); err == nil {
+		return oid, true
+	} else {
+		return bson.ObjectID{}, false
+	}
+}

cursortoken/tokenKeySort_test.go

@@ -0,0 +1,136 @@
+package cursortoken
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestStartToken(t *testing.T) {
+	tok := Start()
+	tst.AssertEqual(t, tok.Token(), "@start")
+	tst.AssertTrue(t, tok.IsStart())
+	tst.AssertFalse(t, tok.IsEnd())
+}
+
+func TestEndToken(t *testing.T) {
+	tok := End()
+	tst.AssertEqual(t, tok.Token(), "@end")
+	tst.AssertTrue(t, tok.IsEnd())
+	tst.AssertFalse(t, tok.IsStart())
+}
+
+func TestNewKeySortTokenBasic(t *testing.T) {
+	tok := NewKeySortToken("alpha", "beta", SortASC, SortDESC, 50, Extra{})
+	tst.AssertFalse(t, tok.IsEnd())
+	tst.AssertFalse(t, tok.IsStart())
+	str := tok.Token()
+	tst.AssertTrue(t, strings.HasPrefix(str, "tok_"))
+}
+
+func TestNewKeySortTokenRoundTrip(t *testing.T) {
+	original := NewKeySortToken("primary-val", "secondary-val", SortASC, SortDESC, 25, Extra{})
+	encoded := original.Token()
+
+	decoded, err := Decode(encoded)
+	tst.AssertNoErr(t, err)
+
+	ks, ok := decoded.(CTKeySort)
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, ks.ValuePrimary, "primary-val")
+	tst.AssertEqual(t, ks.ValueSecondary, "secondary-val")
+	tst.AssertEqual(t, ks.Direction, SortASC)
+	tst.AssertEqual(t, ks.DirectionSecondary, SortDESC)
+	tst.AssertEqual(t, ks.PageSize, 25)
+	tst.AssertEqual(t, ks.Mode, CTMNormal)
+}
+
+func TestKeySortTokenWithExtra(t *testing.T) {
+	ts := time.Date(2024, 6, 15, 10, 30, 0, 0, time.UTC)
+	id := "object-id-123"
+	page := 7
+	pageSize := 42
+
+	original := NewKeySortToken("p", "s", SortDESC, SortASC, 10, Extra{
+		Timestamp: &ts,
+		Id:        &id,
+		Page:      &page,
+		PageSize:  &pageSize,
+	})
+	encoded := original.Token()
+
+	decoded, err := Decode(encoded)
+	tst.AssertNoErr(t, err)
+
+	ks, ok := decoded.(CTKeySort)
+	tst.AssertTrue(t, ok)
+	tst.AssertTrue(t, ks.Extra.Timestamp != nil)
+	tst.AssertTrue(t, ks.Extra.Timestamp.Equal(ts))
+	tst.AssertDeRefEqual(t, ks.Extra.Id, "object-id-123")
+	tst.AssertDeRefEqual(t, ks.Extra.Page, 7)
+	tst.AssertDeRefEqual(t, ks.Extra.PageSize, 42)
+}
+
+func TestKeySortTokenStartRoundTrip(t *testing.T) {
+	original := Start()
+	decoded, err := Decode(original.Token())
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, decoded.IsStart())
+	tst.AssertFalse(t, decoded.IsEnd())
+}
+
+func TestKeySortTokenEndRoundTrip(t *testing.T) {
+	original := End()
+	decoded, err := Decode(original.Token())
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, decoded.IsEnd())
+	tst.AssertFalse(t, decoded.IsStart())
+}
+
+func TestKeySortTokenEmptyValues(t *testing.T) {
+	tok := CTKeySort{Mode: CTMNormal}
+	encoded := tok.Token()
+	tst.AssertTrue(t, strings.HasPrefix(encoded, "tok_"))
+
+	decoded, err := Decode(encoded)
+	tst.AssertNoErr(t, err)
+
+	ks, ok := decoded.(CTKeySort)
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, ks.ValuePrimary, "")
+	tst.AssertEqual(t, ks.ValueSecondary, "")
+	tst.AssertEqual(t, ks.Direction, SortDirection(""))
+	tst.AssertEqual(t, ks.DirectionSecondary, SortDirection(""))
+	tst.AssertEqual(t, ks.PageSize, 0)
+}
+
+func TestKeySortTokenOnlyTimestamp(t *testing.T) {
+	ts := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
+	tok := CTKeySort{
+		Mode:  CTMNormal,
+		Extra: Extra{Timestamp: &ts},
+	}
+
+	decoded, err := Decode(tok.Token())
+	tst.AssertNoErr(t, err)
+
+	ks, ok := decoded.(CTKeySort)
+	tst.AssertTrue(t, ok)
+	tst.AssertTrue(t, ks.Extra.Timestamp != nil)
+	tst.AssertTrue(t, ks.Extra.Timestamp.Equal(ts))
+	tst.AssertTrue(t, ks.Extra.Id == nil)
+	tst.AssertTrue(t, ks.Extra.Page == nil)
+	tst.AssertTrue(t, ks.Extra.PageSize == nil)
+}
+
+func TestKeySortTokenSpecialChars(t *testing.T) {
+	original := NewKeySortToken("hello world / @!#$%", "äöü€", SortASC, SortASC, 1, Extra{})
+	decoded, err := Decode(original.Token())
+	tst.AssertNoErr(t, err)
+
+	ks, ok := decoded.(CTKeySort)
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, ks.ValuePrimary, "hello world / @!#$%")
+	tst.AssertEqual(t, ks.ValueSecondary, "äöü€")
+}

cursortoken/tokenPaginate.go

@@ -0,0 +1,41 @@
+package cursortoken
+
+import "strconv"
+
+type CTPaginated struct {
+	Mode Mode
+	Page int
+}
+
+func Page(p int) CursorToken {
+	return CTPaginated{
+		Mode: CTMNormal,
+		Page: p,
+	}
+}
+
+func PageEnd() CursorToken {
+	return CTPaginated{
+		Mode: CTMEnd,
+		Page: 0,
+	}
+}
+
+func (c CTPaginated) Token() string {
+	if c.Mode == CTMStart {
+		return "$1"
+	}
+	if c.Mode == CTMEnd {
+		return "$end"
+	}
+
+	return "$" + strconv.Itoa(c.Page)
+}
+
+func (c CTPaginated) IsEnd() bool {
+	return c.Mode == CTMEnd
+}
+
+func (c CTPaginated) IsStart() bool {
+	return c.Mode == CTMStart || c.Page == 1
+}

cursortoken/tokenPaginate_test.go

@@ -0,0 +1,61 @@
+package cursortoken
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestPageToken(t *testing.T) {
+	tok := Page(5)
+	tst.AssertEqual(t, tok.Token(), "$5")
+	tst.AssertFalse(t, tok.IsEnd())
+	tst.AssertFalse(t, tok.IsStart())
+}
+
+func TestPageTokenOne(t *testing.T) {
+	tok := Page(1)
+	tst.AssertEqual(t, tok.Token(), "$1")
+	tst.AssertFalse(t, tok.IsEnd())
+	tst.AssertTrue(t, tok.IsStart())
+}
+
+func TestPageTokenLarge(t *testing.T) {
+	tok := Page(123456)
+	tst.AssertEqual(t, tok.Token(), "$123456")
+}
+
+func TestPageTokenZero(t *testing.T) {
+	tok := Page(0)
+	tst.AssertEqual(t, tok.Token(), "$0")
+	tst.AssertFalse(t, tok.IsEnd())
+	tst.AssertFalse(t, tok.IsStart())
+}
+
+func TestPageEndToken(t *testing.T) {
+	tok := PageEnd()
+	tst.AssertEqual(t, tok.Token(), "$end")
+	tst.AssertTrue(t, tok.IsEnd())
+	tst.AssertFalse(t, tok.IsStart())
+}
+
+func TestPaginatedStartMode(t *testing.T) {
+	tok := CTPaginated{Mode: CTMStart, Page: 0}
+	tst.AssertEqual(t, tok.Token(), "$1")
+	tst.AssertTrue(t, tok.IsStart())
+	tst.AssertFalse(t, tok.IsEnd())
+}
+
+func TestPaginatedEndMode(t *testing.T) {
+	tok := CTPaginated{Mode: CTMEnd, Page: 99}
+	tst.AssertEqual(t, tok.Token(), "$end")
+	tst.AssertTrue(t, tok.IsEnd())
+}
+
+func TestPaginatedRoundTrip(t *testing.T) {
+	for _, page := range []int{2, 3, 7, 100, 9999} {
+		tok := Page(page)
+		decoded, err := Decode(tok.Token())
+		tst.AssertNoErr(t, err)
+		tst.AssertEqual(t, decoded.Token(), tok.Token())
+	}
+}

cursortoken/token_test.go

@@ -0,0 +1,125 @@
+package cursortoken
+
+import (
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestDecodeEmpty(t *testing.T) {
+	tok, err := Decode("")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsStart())
+	tst.AssertFalse(t, tok.IsEnd())
+	tst.AssertEqual(t, tok.Token(), "@start")
+}
+
+func TestDecodeAtStart(t *testing.T) {
+	tok, err := Decode("@start")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsStart())
+	tst.AssertFalse(t, tok.IsEnd())
+}
+
+func TestDecodeAtStartUppercase(t *testing.T) {
+	tok, err := Decode("@START")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsStart())
+}
+
+func TestDecodeAtStartMixedCase(t *testing.T) {
+	tok, err := Decode("@StArT")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsStart())
+}
+
+func TestDecodeAtEnd(t *testing.T) {
+	tok, err := Decode("@end")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsEnd())
+	tst.AssertFalse(t, tok.IsStart())
+}
+
+func TestDecodeAtEndUppercase(t *testing.T) {
+	tok, err := Decode("@END")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsEnd())
+}
+
+func TestDecodeDollarEnd(t *testing.T) {
+	tok, err := Decode("$end")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsEnd())
+	_, ok := tok.(CTPaginated)
+	tst.AssertTrue(t, ok)
+}
+
+func TestDecodeDollarEndUppercase(t *testing.T) {
+	tok, err := Decode("$END")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsEnd())
+}
+
+func TestDecodeDollarPage(t *testing.T) {
+	tok, err := Decode("$5")
+	tst.AssertNoErr(t, err)
+	pg, ok := tok.(CTPaginated)
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, pg.Page, 5)
+	tst.AssertEqual(t, pg.Mode, CTMNormal)
+}
+
+func TestDecodeDollarPageOne(t *testing.T) {
+	tok, err := Decode("$1")
+	tst.AssertNoErr(t, err)
+	tst.AssertTrue(t, tok.IsStart())
+	pg, ok := tok.(CTPaginated)
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, pg.Page, 1)
+}
+
+func TestDecodeDollarPageInvalid(t *testing.T) {
+	_, err := Decode("$abc")
+	if err == nil {
+		t.Fatalf("expected error for invalid page")
+	}
+}
+
+func TestDecodeUnknownPrefix(t *testing.T) {
+	_, err := Decode("foobar")
+	if err == nil {
+		t.Fatalf("expected error for unknown prefix")
+	}
+}
+
+func TestDecodeInvalidBase32(t *testing.T) {
+	_, err := Decode("tok_!!!")
+	if err == nil {
+		t.Fatalf("expected error for invalid base32 body")
+	}
+}
+
+func TestDecodeInvalidJSON(t *testing.T) {
+	// "tok_" prefix with valid base32 but invalid JSON content
+	_, err := Decode("tok_NBSWY3DP")
+	if err == nil {
+		t.Fatalf("expected error for invalid json body")
+	}
+}
+
+func TestDecodeJustDollar(t *testing.T) {
+	// "$" alone (length == 1) should fall through to the unknown-prefix branch
+	_, err := Decode("$")
+	if err == nil {
+		t.Fatalf("expected error for bare $")
+	}
+}
+
+func TestDecodeKnownTokenContent(t *testing.T) {
+	tok := NewKeySortToken("k1", "k2", SortASC, SortDESC, 33, Extra{})
+	encoded := tok.Token()
+
+	decoded, err := Decode(encoded)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, decoded.Token(), encoded)
+}

dataext/broadcaster.go

@@ -0,0 +1,230 @@
+package dataext
+
+import (
+	"context"
+	"iter"
+	"sync"
+	"time"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/syncext"
+	"github.com/rs/xid"
+)
+
+// Broadcaster is a simple Broadcaster channel
+// This is a simpler interface over Broadcaster - which does not have distinct namespaces
+type Broadcaster[TData any] struct {
+	masterLock *sync.Mutex
+
+	subscriptions []*broadcastSubscription[TData]
+}
+
+type BroadcastSubscription interface {
+	Unsubscribe()
+}
+
+type broadcastSubscription[TData any] struct {
+	ID string
+
+	parent *Broadcaster[TData]
+
+	subLock *sync.Mutex
+
+	Func func(TData)
+	Chan chan TData
+
+	UnsubChan chan bool
+}
+
+func (p *broadcastSubscription[TData]) Unsubscribe() {
+	p.parent.unsubscribe(p)
+}
+
+func NewBroadcaster[TData any](capacity int) *Broadcaster[TData] {
+	return &Broadcaster[TData]{
+		masterLock:    &sync.Mutex{},
+		subscriptions: make([]*broadcastSubscription[TData], 0, capacity),
+	}
+}
+
+func (bb *Broadcaster[TData]) SubscriberCount() int {
+	bb.masterLock.Lock()
+	defer bb.masterLock.Unlock()
+
+	return len(bb.subscriptions)
+}
+
+// Publish sends `data` to all subscriber
+// But unbuffered - if one is currently not listening, we skip (the actualReceiver < subscriber)
+func (bb *Broadcaster[TData]) Publish(data TData) (subscriber int, actualReceiver int) {
+	bb.masterLock.Lock()
+	subs := langext.ArrCopy(bb.subscriptions)
+	bb.masterLock.Unlock()
+
+	subscriber = len(subs)
+	actualReceiver = 0
+
+	for _, sub := range subs {
+		func() {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+				actualReceiver++
+			} else if sub.Chan != nil {
+				msgSent := syncext.WriteNonBlocking(sub.Chan, data)
+				if msgSent {
+					actualReceiver++
+				}
+			}
+		}()
+	}
+
+	return subscriber, actualReceiver
+}
+
+// PublishWithContext sends `data` to all subscriber
+// buffered - if one is currently not listening, we wait (but error out when the context runs out)
+func (bb *Broadcaster[TData]) PublishWithContext(ctx context.Context, data TData) (subscriber int, actualReceiver int, err error) {
+	bb.masterLock.Lock()
+	subs := langext.ArrCopy(bb.subscriptions)
+	bb.masterLock.Unlock()
+
+	subscriber = len(subs)
+	actualReceiver = 0
+
+	for _, sub := range subs {
+		err := func() error {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+				actualReceiver++
+			} else if sub.Chan != nil {
+				err := syncext.WriteChannelWithContext(ctx, sub.Chan, data)
+				if err != nil {
+					return err
+				}
+				actualReceiver++
+			}
+
+			return nil
+		}()
+		if err != nil {
+			return subscriber, actualReceiver, err
+		}
+	}
+
+	return subscriber, actualReceiver, nil
+}
+
+// PublishWithTimeout sends `data` to all subscriber
+// buffered - if one is currently not listening, we wait (but wait at most `timeout` - if the timeout is exceeded then actualReceiver < subscriber)
+func (bb *Broadcaster[TData]) PublishWithTimeout(data TData, timeout time.Duration) (subscriber int, actualReceiver int) {
+	bb.masterLock.Lock()
+	subs := langext.ArrCopy(bb.subscriptions)
+	bb.masterLock.Unlock()
+
+	subscriber = len(subs)
+	actualReceiver = 0
+
+	for _, sub := range subs {
+		func() {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+				actualReceiver++
+			} else if sub.Chan != nil {
+				ok := syncext.WriteChannelWithTimeout(sub.Chan, data, timeout)
+				if ok {
+					actualReceiver++
+				}
+			}
+		}()
+	}
+
+	return subscriber, actualReceiver
+}
+
+func (bb *Broadcaster[TData]) SubscribeByCallback(fn func(TData)) BroadcastSubscription {
+	bb.masterLock.Lock()
+	defer bb.masterLock.Unlock()
+
+	sub := &broadcastSubscription[TData]{ID: xid.New().String(), parent: bb, subLock: &sync.Mutex{}, Func: fn, UnsubChan: nil}
+
+	bb.subscriptions = append(bb.subscriptions, sub)
+
+	return sub
+}
+
+func (bb *Broadcaster[TData]) SubscribeByChan(chanBufferSize int) (chan TData, BroadcastSubscription) {
+	bb.masterLock.Lock()
+	defer bb.masterLock.Unlock()
+
+	msgCh := make(chan TData, chanBufferSize)
+
+	sub := &broadcastSubscription[TData]{ID: xid.New().String(), parent: bb, subLock: &sync.Mutex{}, Chan: msgCh, UnsubChan: nil}
+
+	bb.subscriptions = append(bb.subscriptions, sub)
+
+	return msgCh, sub
+}
+
+func (bb *Broadcaster[TData]) SubscribeByIter(chanBufferSize int) (iter.Seq[TData], BroadcastSubscription) {
+	bb.masterLock.Lock()
+	defer bb.masterLock.Unlock()
+
+	msgCh := make(chan TData, chanBufferSize)
+	unsubChan := make(chan bool, 8)
+
+	sub := &broadcastSubscription[TData]{ID: xid.New().String(), parent: bb, subLock: &sync.Mutex{}, Chan: msgCh, UnsubChan: unsubChan}
+
+	bb.subscriptions = append(bb.subscriptions, sub)
+
+	iterFun := func(yield func(TData) bool) {
+		for {
+			select {
+			case msg := <-msgCh:
+				if !yield(msg) {
+					sub.Unsubscribe()
+					return
+				}
+			case <-sub.UnsubChan:
+				sub.Unsubscribe()
+				return
+			}
+		}
+	}
+
+	return iterFun, sub
+}
+
+func (bb *Broadcaster[TData]) unsubscribe(p *broadcastSubscription[TData]) {
+	bb.masterLock.Lock()
+	defer bb.masterLock.Unlock()
+
+	p.subLock.Lock()
+	defer p.subLock.Unlock()
+
+	if p.Chan != nil {
+		close(p.Chan)
+		p.Chan = nil
+	}
+	if p.UnsubChan != nil {
+		syncext.WriteNonBlocking(p.UnsubChan, true)
+		close(p.UnsubChan)
+		p.UnsubChan = nil
+	}
+
+	bb.subscriptions = langext.ArrFilter(bb.subscriptions, func(v *broadcastSubscription[TData]) bool {
+		return v.ID != p.ID
+	})
+}

dataext/broadcaster_test.go

@@ -0,0 +1,342 @@
+package dataext
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestNewBroadcast(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+	if bb == nil {
+		t.Fatal("NewBroadcaster returned nil")
+	}
+	if bb.masterLock == nil {
+		t.Fatal("masterLock is nil")
+	}
+	if bb.subscriptions == nil {
+		t.Fatal("subscriptions is nil")
+	}
+}
+
+func TestBroadcast_SubscribeByCallback(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	var received string
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	callback := func(msg string) {
+		received = msg
+		wg.Done()
+	}
+
+	sub := bb.SubscribeByCallback(callback)
+	defer sub.Unsubscribe()
+
+	// Publish a message
+	subs, receivers := bb.Publish("hello")
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Wait for the callback to be executed
+	wg.Wait()
+
+	if received != "hello" {
+		t.Fatalf("Expected to receive 'hello', got '%s'", received)
+	}
+}
+
+func TestBroadcast_SubscribeByChan(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	ch, sub := bb.SubscribeByChan(1)
+	defer sub.Unsubscribe()
+
+	// Publish a message
+	subs, receivers := bb.Publish("hello")
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Read from the channel with a timeout to avoid blocking
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+}
+
+func TestBroadcast_SubscribeByIter(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	iterSeq, sub := bb.SubscribeByIter(1)
+	defer sub.Unsubscribe()
+
+	// Channel to communicate when message is received
+	done := make(chan bool)
+	goroutineDone := make(chan struct{})
+	received := false
+
+	// Start a goroutine to use the iterator
+	go func() {
+		defer close(goroutineDone)
+		for msg := range iterSeq {
+			if msg == "hello" {
+				received = true
+				done <- true
+				return // Stop iteration — triggers Unsubscribe via yield returning false
+			}
+		}
+	}()
+
+	// Give time for the iterator to start
+	time.Sleep(100 * time.Millisecond)
+
+	// Publish a message
+	bb.Publish("hello")
+
+	// Wait for the message to be received or timeout
+	select {
+	case <-done:
+		if !received {
+			t.Fatal("Message was received but not 'hello'")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Wait for the goroutine to fully exit so Unsubscribe (triggered by the
+	// iterator cleanup when yield returns false) has completed.
+	select {
+	case <-goroutineDone:
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for goroutine to finish")
+	}
+
+	subCount := bb.SubscriberCount()
+	if subCount != 0 {
+		t.Fatalf("Expected 0 receivers, got %d", subCount)
+	}
+}
+
+func TestBroadcast_Publish(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	// Test publishing with no subscribers
+	subs, receivers := bb.Publish("hello")
+	if subs != 0 {
+		t.Fatalf("Expected 0 subscribers, got %d", subs)
+	}
+	if receivers != 0 {
+		t.Fatalf("Expected 0 receivers, got %d", receivers)
+	}
+
+	// Add a subscriber
+	ch, sub := bb.SubscribeByChan(1)
+	defer sub.Unsubscribe()
+
+	// Publish a message
+	subs, receivers = bb.Publish("hello")
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Test non-blocking behavior with a full channel
+	// First fill the channel
+	bb.Publish("fill")
+
+	// Now publish again - this should not block but may skip the receiver
+	subs, receivers = bb.Publish("overflow")
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	_ = receivers // may be 0 if channel is full
+
+	// Drain the channel
+	<-ch
+}
+
+func TestBroadcast_PublishWithTimeout(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	// Add a subscriber with a channel
+	ch, sub := bb.SubscribeByChan(1)
+	defer sub.Unsubscribe()
+
+	// Publish with a timeout
+	subs, receivers := bb.PublishWithTimeout("hello", 100*time.Millisecond)
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Fill the channel
+	bb.Publish("fill")
+
+	// Test timeout behavior with a full channel
+	start := time.Now()
+	subs, receivers = bb.PublishWithTimeout("timeout-test", 50*time.Millisecond)
+	elapsed := time.Since(start)
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	// The receiver count should be 0 if the timeout occurred
+	if elapsed < 50*time.Millisecond {
+		t.Fatalf("Expected to wait at least 50ms, only waited %v", elapsed)
+	}
+
+	// Drain the channel
+	<-ch
+}
+
+func TestBroadcast_PublishWithContext(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	// Add a subscriber with a channel
+	ch, sub := bb.SubscribeByChan(1)
+	defer sub.Unsubscribe()
+
+	// Create a context
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	// Publish with context
+	subs, receivers, err := bb.PublishWithContext(ctx, "hello")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Fill the channel
+	bb.Publish("fill")
+
+	// Test context cancellation with a full channel
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// Cancel the context after a short delay
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		cancel()
+	}()
+
+	start := time.Now()
+	subs, receivers, err = bb.PublishWithContext(ctx, "context-test")
+	elapsed := time.Since(start)
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	// Should get a context canceled error
+	if err == nil {
+		t.Fatal("Expected context canceled error, got nil")
+	}
+
+	if elapsed < 50*time.Millisecond {
+		t.Fatalf("Expected to wait at least 50ms, only waited %v", elapsed)
+	}
+
+	// Drain the channel
+	<-ch
+}
+
+func TestBroadcast_Unsubscribe(t *testing.T) {
+	bb := NewBroadcaster[string](10)
+
+	// Add a subscriber
+	ch, sub := bb.SubscribeByChan(1)
+
+	// Publish a message
+	subs, receivers := bb.Publish("hello")
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Unsubscribe
+	sub.Unsubscribe()
+
+	// Publish again
+	subs, receivers = bb.Publish("after-unsub")
+	if subs != 0 {
+		t.Fatalf("Expected 0 subscribers after unsubscribe, got %d", subs)
+	}
+	if receivers != 0 {
+		t.Fatalf("Expected 0 receivers after unsubscribe, got %d", receivers)
+	}
+
+	// Check that the subscriber count is 0
+	if bb.SubscriberCount() != 0 {
+		t.Fatalf("Expected SubscriberCount() == 0, got %d", bb.SubscriberCount())
+	}
+}

dataext/bufferedReadCloser.go

@@ -8,10 +8,10 @@ import (
 type brcMode int
 
 const (
-	modeSourceReading  = 0
-	modeSourceFinished = 1
-	modeBufferReading  = 2
-	modeBufferFinished = 3
+	modeSourceReading  brcMode = 0
+	modeSourceFinished brcMode = 1
+	modeBufferReading  brcMode = 2
+	modeBufferFinished brcMode = 3
 )
 
 type BufferedReadCloser interface {
@@ -115,6 +115,9 @@ func (b *bufferedReadCloser) BufferedAll() ([]byte, error) {
 				return nil, err
 			}
 		}
+		if err := b.Reset(); err != nil {
+			return nil, err
+		}
 		return b.buffer, nil
 
 	case modeSourceFinished:
@@ -131,10 +134,22 @@ func (b *bufferedReadCloser) BufferedAll() ([]byte, error) {
 	}
 }
 
+// Reset resets the buffer to the beginning of the buffer.
+// If the original source is partially read, we will finish reading it and fill our buffer
 func (b *bufferedReadCloser) Reset() error {
 	switch b.mode {
 	case modeSourceReading:
-		fallthrough
+		if b.off == 0 {
+			return nil // nobody has read anything yet
+		}
+		err := b.Close()
+		if err != nil {
+			return err
+		}
+		b.mode = modeBufferReading
+		b.off = 0
+		return nil
+
 	case modeSourceFinished:
 		err := b.Close()
 		if err != nil {

dataext/bufferedReadCloser_test.go

@@ -0,0 +1,129 @@
+package dataext
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+type fakeReadCloser struct {
+	r      *bytes.Reader
+	closed bool
+}
+
+func newFakeReadCloser(data []byte) *fakeReadCloser {
+	return &fakeReadCloser{r: bytes.NewReader(data)}
+}
+
+func (f *fakeReadCloser) Read(p []byte) (int, error) {
+	return f.r.Read(p)
+}
+
+func (f *fakeReadCloser) Close() error {
+	f.closed = true
+	return nil
+}
+
+func TestBufferedReadCloser_ReadAll(t *testing.T) {
+	data := []byte("hello world")
+	brc := NewBufferedReadCloser(newFakeReadCloser(data))
+
+	buf := make([]byte, 64)
+	total := 0
+	for {
+		n, err := brc.Read(buf[total:])
+		total += n
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+	}
+
+	if !bytes.Equal(buf[:total], data) {
+		t.Fatalf("got %q want %q", buf[:total], data)
+	}
+}
+
+func TestBufferedReadCloser_BufferedAllThenRead(t *testing.T) {
+	data := []byte("foobar baz")
+	brc := NewBufferedReadCloser(newFakeReadCloser(data))
+
+	all, err := brc.BufferedAll()
+	if err != nil {
+		t.Fatalf("BufferedAll err: %v", err)
+	}
+	if !bytes.Equal(all, data) {
+		t.Fatalf("BufferedAll got %q want %q", all, data)
+	}
+
+	// after BufferedAll, Reset put us in BufferReading mode - we can read again
+	out, err := io.ReadAll(brc)
+	if err != nil {
+		t.Fatalf("ReadAll err: %v", err)
+	}
+	if !bytes.Equal(out, data) {
+		t.Fatalf("ReadAll got %q want %q", out, data)
+	}
+}
+
+func TestBufferedReadCloser_FullyReadResetReread(t *testing.T) {
+	data := []byte("abcdefghij")
+	brc := NewBufferedReadCloser(newFakeReadCloser(data))
+
+	out, err := io.ReadAll(brc)
+	if err != nil {
+		t.Fatalf("first ReadAll err: %v", err)
+	}
+	if !bytes.Equal(out, data) {
+		t.Fatalf("first read got %q want %q", out, data)
+	}
+
+	if err := brc.Reset(); err != nil {
+		t.Fatalf("reset err: %v", err)
+	}
+
+	out2, err := io.ReadAll(brc)
+	if err != nil {
+		t.Fatalf("second ReadAll err: %v", err)
+	}
+	if !bytes.Equal(out2, data) {
+		t.Fatalf("after reset got %q want %q", out2, data)
+	}
+}
+
+func TestBufferedReadCloser_Close(t *testing.T) {
+	data := []byte("xyz")
+	inner := newFakeReadCloser(data)
+	brc := NewBufferedReadCloser(inner)
+
+	if err := brc.Close(); err != nil {
+		t.Fatalf("close err: %v", err)
+	}
+	if !inner.closed {
+		t.Fatal("inner not closed")
+	}
+
+	// double close should be no-op
+	if err := brc.Close(); err != nil {
+		t.Fatalf("second close err: %v", err)
+	}
+}
+
+func TestBufferedReadCloser_ResetWithoutRead(t *testing.T) {
+	data := []byte("abc")
+	brc := NewBufferedReadCloser(newFakeReadCloser(data))
+
+	if err := brc.Reset(); err != nil {
+		t.Fatalf("reset err: %v", err)
+	}
+
+	out, err := io.ReadAll(brc)
+	if err != nil {
+		t.Fatalf("ReadAll err: %v", err)
+	}
+	if !bytes.Equal(out, data) {
+		t.Fatalf("got %q want %q", out, data)
+	}
+}

dataext/casMutex.go

@@ -0,0 +1,254 @@
+package dataext
+
+import (
+	"context"
+	"golang.org/x/sync/semaphore"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+)
+
+// from https://github.com/viney-shih/go-lock/blob/2f19fd8ce335e33e0ab9dccb1ff2ce820c3da332/cas.go
+
+// CASMutex is the struct implementing RWMutex with CAS mechanism.
+type CASMutex struct {
+	state     casState
+	turnstile *semaphore.Weighted
+
+	broadcastChan chan struct{}
+	broadcastMut  sync.RWMutex
+}
+
+func NewCASMutex() *CASMutex {
+	return &CASMutex{
+		state:         casStateNoLock,
+		turnstile:     semaphore.NewWeighted(1),
+		broadcastChan: make(chan struct{}),
+	}
+}
+
+type casState int32
+
+const (
+	casStateUndefined casState = iota - 2 // -2
+	casStateWriteLock                     // -1
+	casStateNoLock                        // 0
+	casStateReadLock                      // >= 1
+)
+
+func (m *CASMutex) getState(n int32) casState {
+	switch st := casState(n); {
+	case st == casStateWriteLock:
+		fallthrough
+	case st == casStateNoLock:
+		return st
+	case st >= casStateReadLock:
+		return casStateReadLock
+	default:
+		// actually, it should not happened.
+		return casStateUndefined
+	}
+}
+
+func (m *CASMutex) listen() <-chan struct{} {
+	m.broadcastMut.RLock()
+	defer m.broadcastMut.RUnlock()
+
+	return m.broadcastChan
+}
+
+func (m *CASMutex) broadcast() {
+	newCh := make(chan struct{})
+
+	m.broadcastMut.Lock()
+	ch := m.broadcastChan
+	m.broadcastChan = newCh
+	m.broadcastMut.Unlock()
+
+	close(ch)
+}
+
+func (m *CASMutex) tryLock(ctx context.Context) bool {
+	for {
+		broker := m.listen()
+		if atomic.CompareAndSwapInt32(
+			(*int32)(unsafe.Pointer(&m.state)),
+			int32(casStateNoLock),
+			int32(casStateWriteLock),
+		) {
+			return true
+		}
+
+		if ctx == nil {
+			return false
+		}
+
+		select {
+		case <-ctx.Done():
+			// timeout or cancellation
+			return false
+		case <-broker:
+			// waiting for signal triggered by m.broadcast() and trying again.
+		}
+	}
+}
+
+// TryLockWithContext attempts to acquire the lock, blocking until resources
+// are available or ctx is done (timeout or cancellation).
+func (m *CASMutex) TryLockWithContext(ctx context.Context) bool {
+	if err := m.turnstile.Acquire(ctx, 1); err != nil {
+		// Acquire failed due to timeout or cancellation
+		return false
+	}
+
+	defer m.turnstile.Release(1)
+
+	return m.tryLock(ctx)
+}
+
+// Lock acquires the lock.
+// If it is currently held by others, Lock will wait until it has a chance to acquire it.
+func (m *CASMutex) Lock() {
+	ctx := context.Background()
+
+	m.TryLockWithContext(ctx)
+}
+
+// TryLock attempts to acquire the lock without blocking.
+// Return false if someone is holding it now.
+func (m *CASMutex) TryLock() bool {
+	if !m.turnstile.TryAcquire(1) {
+		return false
+	}
+
+	defer m.turnstile.Release(1)
+
+	return m.tryLock(nil)
+}
+
+// TryLockWithTimeout attempts to acquire the lock within a period of time.
+// Return false if spending time is more than duration and no chance to acquire it.
+func (m *CASMutex) TryLockWithTimeout(duration time.Duration) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), duration)
+	defer cancel()
+
+	return m.TryLockWithContext(ctx)
+}
+
+// Unlock releases the lock.
+func (m *CASMutex) Unlock() {
+	if ok := atomic.CompareAndSwapInt32(
+		(*int32)(unsafe.Pointer(&m.state)),
+		int32(casStateWriteLock),
+		int32(casStateNoLock),
+	); !ok {
+		panic("Unlock failed")
+	}
+
+	m.broadcast()
+}
+
+func (m *CASMutex) rTryLock(ctx context.Context) bool {
+	for {
+		broker := m.listen()
+		n := atomic.LoadInt32((*int32)(unsafe.Pointer(&m.state)))
+		st := m.getState(n)
+		switch st {
+		case casStateNoLock, casStateReadLock:
+			if atomic.CompareAndSwapInt32((*int32)(unsafe.Pointer(&m.state)), n, n+1) {
+				return true
+			}
+		}
+
+		if ctx == nil {
+			return false
+		}
+
+		select {
+		case <-ctx.Done():
+			// timeout or cancellation
+			return false
+		default:
+			switch st {
+			// read-lock failed due to concurrence issue, try again immediately
+			case casStateNoLock, casStateReadLock:
+				runtime.Gosched() // allow other goroutines to do stuff.
+				continue
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			// timeout or cancellation
+			return false
+		case <-broker:
+			// waiting for signal triggered by m.broadcast() and trying again.
+		}
+	}
+}
+
+// RTryLockWithContext attempts to acquire the read lock, blocking until resources
+// are available or ctx is done (timeout or cancellation).
+func (m *CASMutex) RTryLockWithContext(ctx context.Context) bool {
+	if err := m.turnstile.Acquire(ctx, 1); err != nil {
+		// Acquire failed due to timeout or cancellation
+		return false
+	}
+
+	m.turnstile.Release(1)
+
+	return m.rTryLock(ctx)
+}
+
+// RLock acquires the read lock.
+// If it is currently held by others writing, RLock will wait until it has a chance to acquire it.
+func (m *CASMutex) RLock() {
+	ctx := context.Background()
+
+	m.RTryLockWithContext(ctx)
+}
+
+// RTryLock attempts to acquire the read lock without blocking.
+// Return false if someone is writing it now.
+func (m *CASMutex) RTryLock() bool {
+	if !m.turnstile.TryAcquire(1) {
+		return false
+	}
+
+	m.turnstile.Release(1)
+
+	return m.rTryLock(nil)
+}
+
+// RTryLockWithTimeout attempts to acquire the read lock within a period of time.
+// Return false if spending time is more than duration and no chance to acquire it.
+func (m *CASMutex) RTryLockWithTimeout(duration time.Duration) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), duration)
+	defer cancel()
+
+	return m.RTryLockWithContext(ctx)
+}
+
+// RUnlock releases the read lock.
+func (m *CASMutex) RUnlock() {
+	n := atomic.AddInt32((*int32)(unsafe.Pointer(&m.state)), -1)
+	switch m.getState(n) {
+	case casStateUndefined, casStateWriteLock:
+		panic("RUnlock failed")
+	case casStateNoLock:
+		m.broadcast()
+	}
+}
+
+// RLocker returns a Locker interface that implements the Lock and Unlock methods
+// by calling CASMutex.RLock and CASMutex.RUnlock.
+func (m *CASMutex) RLocker() sync.Locker {
+	return (*rlocker)(m)
+}
+
+type rlocker CASMutex
+
+func (r *rlocker) Lock()   { (*CASMutex)(r).RLock() }
+func (r *rlocker) Unlock() { (*CASMutex)(r).RUnlock() }

dataext/casMutex_test.go

@@ -0,0 +1,122 @@
+package dataext
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestCASMutex_LockUnlock(t *testing.T) {
+	m := NewCASMutex()
+	m.Lock()
+	m.Unlock()
+}
+
+func TestCASMutex_TryLock(t *testing.T) {
+	m := NewCASMutex()
+	if !m.TryLock() {
+		t.Fatal("TryLock should succeed on fresh mutex")
+	}
+	if m.TryLock() {
+		t.Fatal("TryLock should fail when already locked")
+	}
+	m.Unlock()
+	if !m.TryLock() {
+		t.Fatal("TryLock should succeed after Unlock")
+	}
+	m.Unlock()
+}
+
+func TestCASMutex_TryLockWithTimeout(t *testing.T) {
+	m := NewCASMutex()
+	m.Lock()
+	start := time.Now()
+	if m.TryLockWithTimeout(20 * time.Millisecond) {
+		t.Fatal("TryLockWithTimeout should fail when locked")
+	}
+	if time.Since(start) < 15*time.Millisecond {
+		t.Fatal("TryLockWithTimeout returned too quickly")
+	}
+	m.Unlock()
+
+	if !m.TryLockWithTimeout(50 * time.Millisecond) {
+		t.Fatal("TryLockWithTimeout should succeed when unlocked")
+	}
+	m.Unlock()
+}
+
+func TestCASMutex_TryLockWithContext_Cancel(t *testing.T) {
+	m := NewCASMutex()
+	m.Lock()
+	defer m.Unlock()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(10 * time.Millisecond)
+		cancel()
+	}()
+	if m.TryLockWithContext(ctx) {
+		t.Fatal("expected lock to fail after cancel")
+	}
+}
+
+func TestCASMutex_RLockMultiple(t *testing.T) {
+	m := NewCASMutex()
+	if !m.RTryLock() {
+		t.Fatal("RTryLock should succeed")
+	}
+	if !m.RTryLock() {
+		t.Fatal("Second RTryLock should succeed")
+	}
+	if m.TryLock() {
+		t.Fatal("Write TryLock should fail with read locks held")
+	}
+	m.RUnlock()
+	m.RUnlock()
+	if !m.TryLock() {
+		t.Fatal("Write TryLock should succeed after read unlocks")
+	}
+	m.Unlock()
+}
+
+func TestCASMutex_RLocker(t *testing.T) {
+	m := NewCASMutex()
+	rl := m.RLocker()
+	rl.Lock()
+	rl.Unlock()
+}
+
+func TestCASMutex_Concurrent(t *testing.T) {
+	m := NewCASMutex()
+	var counter int64
+	const n = 50
+	var wg sync.WaitGroup
+	wg.Add(n)
+	for i := 0; i < n; i++ {
+		go func() {
+			defer wg.Done()
+			m.Lock()
+			atomic.AddInt64(&counter, 1)
+			m.Unlock()
+		}()
+	}
+	wg.Wait()
+	if atomic.LoadInt64(&counter) != n {
+		t.Fatalf("counter=%d want %d", counter, n)
+	}
+}
+
+func TestCASMutex_RTryLockWithTimeout(t *testing.T) {
+	m := NewCASMutex()
+	m.Lock()
+	if m.RTryLockWithTimeout(20 * time.Millisecond) {
+		t.Fatal("RTryLockWithTimeout should fail when write-locked")
+	}
+	m.Unlock()
+	if !m.RTryLockWithTimeout(20 * time.Millisecond) {
+		t.Fatal("RTryLockWithTimeout should succeed when free")
+	}
+	m.RUnlock()
+}

dataext/delayedCombiningInvoker.go

@@ -0,0 +1,199 @@
+package dataext
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/syncext"
+)
+
+// DelayedCombiningInvoker is a utility to combine multiple consecutive requests into a single execution
+//
+// Requests are made with Request(), and consecutive requests are combined during the `delay` period.
+//
+// Can be used, e.g., for search-controls, where we want to init the search when teh user stops typing
+// Or generally to queue an execution once a burst of requests is over.
+type DelayedCombiningInvoker struct {
+	syncLock           sync.Mutex
+	triggerChan        chan bool
+	cancelChan         chan bool
+	execNowChan        chan bool
+	action             func()
+	delay              time.Duration
+	maxDelay           time.Duration
+	executorRunning    *syncext.AtomicBool
+	pendingRequests    *syncext.Atomic[int]
+	lastRequestTime    time.Time
+	initialRequestTime time.Time
+
+	onExecutionStart []func(immediately bool)          // listener ( actual execution of action starts )
+	onExecutionDone  []func()                          // listener ( actual execution of action finished )
+	onRequest        []func(pending int, initial bool) // listener ( a request came in, waiting for execution )
+}
+
+func NewDelayedCombiningInvoker(action func(), delay time.Duration, maxDelay time.Duration) *DelayedCombiningInvoker {
+	return &DelayedCombiningInvoker{
+		action:             action,
+		delay:              delay,
+		maxDelay:           maxDelay,
+		executorRunning:    syncext.NewAtomicBool(false),
+		pendingRequests:    syncext.NewAtomic[int](0),
+		triggerChan:        make(chan bool),
+		cancelChan:         make(chan bool, 1),
+		execNowChan:        make(chan bool, 1),
+		lastRequestTime:    time.Now(),
+		initialRequestTime: time.Now(),
+		onExecutionStart:   make([]func(bool), 0),
+		onExecutionDone:    make([]func(), 0),
+		onRequest:          make([]func(int, bool), 0),
+	}
+}
+
+func (d *DelayedCombiningInvoker) Request() {
+	now := time.Now()
+
+	d.syncLock.Lock()
+	defer d.syncLock.Unlock()
+
+	if d.executorRunning.Get() {
+		d.lastRequestTime = now
+		d.pendingRequests.Update(func(v int) int { return v + 1 })
+
+		for _, fn := range d.onRequest {
+			_ = langext.RunPanicSafe(func() { fn(d.pendingRequests.Get(), true) })
+		}
+
+		d.triggerChan <- true
+	} else {
+		d.initialRequestTime = now
+		d.lastRequestTime = now
+
+		d.executorRunning.Set(true)
+		d.pendingRequests.Set(1)
+		syncext.ReadNonBlocking(d.triggerChan) // clear the channel
+		syncext.ReadNonBlocking(d.cancelChan)  // clear the channel
+		syncext.ReadNonBlocking(d.execNowChan) // clear the channel
+
+		for _, fn := range d.onRequest {
+			_ = langext.RunPanicSafe(func() { fn(d.pendingRequests.Get(), false) })
+		}
+
+		go d.run()
+	}
+}
+
+func (d *DelayedCombiningInvoker) run() {
+
+	needsExecutorRunningCleanup := true
+	defer func() {
+		if needsExecutorRunningCleanup {
+			d.syncLock.Lock()
+			d.executorRunning.Set(false)
+			d.syncLock.Unlock()
+		}
+	}()
+
+	for {
+		d.syncLock.Lock()
+		timeOut := max(min(d.maxDelay-time.Since(d.initialRequestTime), d.delay-time.Since(d.lastRequestTime)), 0)
+		d.syncLock.Unlock()
+
+		immediately := false
+
+		select {
+		case <-d.execNowChan:
+			// run immediately
+			immediately = true
+			break
+		case <-d.triggerChan:
+			// external trigger - needs to re-evaluate
+			break
+		case <-d.cancelChan:
+			// cancel
+			return
+		case <-time.After(timeOut):
+			// time elapsed - check for execution
+			break
+
+		}
+
+		d.syncLock.Lock()
+		execute := immediately || time.Since(d.lastRequestTime) >= d.delay || time.Since(d.initialRequestTime) >= d.maxDelay
+		if !execute {
+			d.syncLock.Unlock()
+			continue
+		}
+
+		d.pendingRequests.Set(0)
+
+		for _, fn := range d.onExecutionStart {
+			_ = langext.RunPanicSafe(func() { fn(immediately) })
+		}
+
+		// =================================================
+		_ = langext.RunPanicSafe(d.action)
+		// =================================================
+
+		d.executorRunning.Set(false) // ensure HasPendingRequests returns fals ein onExecutionDone listener
+		needsExecutorRunningCleanup = false
+
+		for _, fn := range d.onExecutionDone {
+			_ = langext.RunPanicSafe(fn)
+		}
+
+		d.syncLock.Unlock()
+
+		return
+	}
+}
+
+func (d *DelayedCombiningInvoker) CancelPendingRequests() {
+	d.syncLock.Lock()
+	defer d.syncLock.Unlock()
+
+	syncext.WriteNonBlocking(d.cancelChan, true)
+}
+
+func (d *DelayedCombiningInvoker) HasPendingRequests() bool {
+	return d.executorRunning.Get()
+}
+
+func (d *DelayedCombiningInvoker) CountPendingRequests() int {
+	return d.pendingRequests.Get()
+}
+
+func (d *DelayedCombiningInvoker) ExecuteNow() bool {
+	d.syncLock.Lock()
+	defer d.syncLock.Unlock()
+
+	if d.executorRunning.Get() {
+		syncext.WriteNonBlocking(d.execNowChan, true)
+		return true
+	} else {
+		return false
+	}
+}
+
+func (d *DelayedCombiningInvoker) WaitForCompletion(ctx context.Context) error {
+	return d.executorRunning.WaitWithContext(ctx, false)
+}
+
+func (d *DelayedCombiningInvoker) RegisterOnExecutionStart(fn func(immediately bool)) {
+	d.syncLock.Lock()
+	defer d.syncLock.Unlock()
+	d.onExecutionStart = append(d.onExecutionStart, fn)
+}
+
+func (d *DelayedCombiningInvoker) RegisterOnExecutionDone(fn func()) {
+	d.syncLock.Lock()
+	defer d.syncLock.Unlock()
+	d.onExecutionDone = append(d.onExecutionDone, fn)
+}
+
+func (d *DelayedCombiningInvoker) RegisterOnRequest(fn func(pending int, initial bool)) {
+	d.syncLock.Lock()
+	defer d.syncLock.Unlock()
+	d.onRequest = append(d.onRequest, fn)
+}

dataext/delayedCombiningInvoker_test.go

@@ -0,0 +1,182 @@
+package dataext
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func waitForCalls(t *testing.T, calls *int64, want int64, max time.Duration) {
+	t.Helper()
+	deadline := time.Now().Add(max)
+	for time.Now().Before(deadline) {
+		if atomic.LoadInt64(calls) >= want {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+func TestDelayedCombiningInvoker_SingleRequest(t *testing.T) {
+	var calls int64
+	d := NewDelayedCombiningInvoker(func() {
+		atomic.AddInt64(&calls, 1)
+	}, 20*time.Millisecond, 200*time.Millisecond)
+
+	d.Request()
+
+	waitForCalls(t, &calls, 1, 2*time.Second)
+	if c := atomic.LoadInt64(&calls); c != 1 {
+		t.Fatalf("calls=%d want 1", c)
+	}
+}
+
+func TestDelayedCombiningInvoker_TwoRequestsCombine(t *testing.T) {
+	var calls int64
+	d := NewDelayedCombiningInvoker(func() {
+		atomic.AddInt64(&calls, 1)
+	}, 50*time.Millisecond, 1*time.Second)
+
+	d.Request()
+	time.Sleep(10 * time.Millisecond)
+	d.Request()
+
+	waitForCalls(t, &calls, 1, 2*time.Second)
+	if c := atomic.LoadInt64(&calls); c != 1 {
+		t.Fatalf("calls=%d want 1 (should be combined)", c)
+	}
+}
+
+func TestDelayedCombiningInvoker_SequentialRuns(t *testing.T) {
+	var calls int64
+	d := NewDelayedCombiningInvoker(func() {
+		atomic.AddInt64(&calls, 1)
+	}, 20*time.Millisecond, 200*time.Millisecond)
+
+	d.Request()
+	waitForCalls(t, &calls, 1, 2*time.Second)
+	if c := atomic.LoadInt64(&calls); c != 1 {
+		t.Fatalf("after first wait calls=%d want 1", c)
+	}
+
+	// allow executorRunning to clear
+	time.Sleep(50 * time.Millisecond)
+
+	d.Request()
+	waitForCalls(t, &calls, 2, 2*time.Second)
+	if c := atomic.LoadInt64(&calls); c != 2 {
+		t.Fatalf("calls=%d want 2", c)
+	}
+}
+
+func TestDelayedCombiningInvoker_ExecuteNow(t *testing.T) {
+	var calls int64
+	d := NewDelayedCombiningInvoker(func() {
+		atomic.AddInt64(&calls, 1)
+	}, 5*time.Second, 30*time.Second)
+
+	d.Request()
+	if !d.HasPendingRequests() {
+		t.Fatal("should have pending requests")
+	}
+
+	if !d.ExecuteNow() {
+		t.Fatal("ExecuteNow should return true when running")
+	}
+
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if atomic.LoadInt64(&calls) >= 1 {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if c := atomic.LoadInt64(&calls); c != 1 {
+		t.Fatalf("calls=%d want 1 (ExecuteNow should fire well before delay)", c)
+	}
+
+	// allow internal state cleanup
+	for i := 0; i < 100; i++ {
+		if !d.HasPendingRequests() {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if d.ExecuteNow() {
+		t.Fatal("ExecuteNow should return false when no pending")
+	}
+}
+
+func TestDelayedCombiningInvoker_Cancel(t *testing.T) {
+	var calls int64
+	d := NewDelayedCombiningInvoker(func() {
+		atomic.AddInt64(&calls, 1)
+	}, 500*time.Millisecond, 5*time.Second)
+
+	d.Request()
+	d.CancelPendingRequests()
+
+	time.Sleep(200 * time.Millisecond)
+
+	if c := atomic.LoadInt64(&calls); c != 0 {
+		t.Fatalf("calls=%d want 0 after cancel", c)
+	}
+}
+
+func TestDelayedCombiningInvoker_HasAndCountPending(t *testing.T) {
+	d := NewDelayedCombiningInvoker(func() {
+		// no-op
+	}, 500*time.Millisecond, 5*time.Second)
+
+	if d.HasPendingRequests() {
+		t.Fatal("should not have pending before any Request")
+	}
+	if d.CountPendingRequests() != 0 {
+		t.Fatalf("count=%d want 0", d.CountPendingRequests())
+	}
+
+	d.Request()
+	if !d.HasPendingRequests() {
+		t.Fatal("should have pending")
+	}
+	if d.CountPendingRequests() < 1 {
+		t.Fatalf("count=%d want >=1", d.CountPendingRequests())
+	}
+	d.CancelPendingRequests()
+}
+
+func TestDelayedCombiningInvoker_Listeners(t *testing.T) {
+	var (
+		startCount   int64
+		doneCount    int64
+		requestCount int64
+	)
+
+	d := NewDelayedCombiningInvoker(func() {
+		// no-op
+	}, 20*time.Millisecond, 200*time.Millisecond)
+
+	d.RegisterOnExecutionStart(func(immediately bool) {
+		atomic.AddInt64(&startCount, 1)
+	})
+	d.RegisterOnExecutionDone(func() {
+		atomic.AddInt64(&doneCount, 1)
+	})
+	d.RegisterOnRequest(func(pending int, initial bool) {
+		atomic.AddInt64(&requestCount, 1)
+	})
+
+	d.Request()
+
+	waitForCalls(t, &doneCount, 1, 2*time.Second)
+
+	if atomic.LoadInt64(&startCount) != 1 {
+		t.Fatalf("startCount=%d want 1", startCount)
+	}
+	if atomic.LoadInt64(&doneCount) != 1 {
+		t.Fatalf("doneCount=%d want 1", doneCount)
+	}
+	if atomic.LoadInt64(&requestCount) != 1 {
+		t.Fatalf("requestCount=%d want 1", requestCount)
+	}
+}

dataext/lruMap.go

@@ -19,40 +19,38 @@ import (
 // There are also a bunch of unit tests to ensure that the cache is always in a consistent state
 //
 
-type LRUData interface{}
-
-type LRUMap struct {
+type LRUMap[TKey comparable, TData any] struct {
 	maxsize int
 	lock    sync.Mutex
 
-	cache map[string]*cacheNode
+	cache map[TKey]*cacheNode[TKey, TData]
 
-	lfuHead *cacheNode
-	lfuTail *cacheNode
+	lfuHead *cacheNode[TKey, TData]
+	lfuTail *cacheNode[TKey, TData]
 }
 
-type cacheNode struct {
-	key    string
-	data   LRUData
-	parent *cacheNode
-	child  *cacheNode
+type cacheNode[TKey comparable, TData any] struct {
+	key    TKey
+	data   TData
+	parent *cacheNode[TKey, TData]
+	child  *cacheNode[TKey, TData]
 }
 
-func NewLRUMap(size int) *LRUMap {
+func NewLRUMap[TKey comparable, TData any](size int) *LRUMap[TKey, TData] {
 	if size <= 2 && size != 0 {
 		panic("Size must be > 2  (or 0)")
 	}
 
-	return &LRUMap{
+	return &LRUMap[TKey, TData]{
 		maxsize: size,
 		lock:    sync.Mutex{},
-		cache:   make(map[string]*cacheNode, size+1),
+		cache:   make(map[TKey]*cacheNode[TKey, TData], size+1),
 		lfuHead: nil,
 		lfuTail: nil,
 	}
 }
 
-func (c *LRUMap) Put(key string, value LRUData) {
+func (c *LRUMap[TKey, TData]) Put(key TKey, value TData) {
 	if c.maxsize == 0 {
 		return // cache disabled
 	}
@@ -70,7 +68,7 @@ func (c *LRUMap) Put(key string, value LRUData) {
 	}
 
 	// key does not exist: insert into map and add to top of LFU
-	node = &cacheNode{
+	node = &cacheNode[TKey, TData]{
 		key:    key,
 		data:   value,
 		parent: nil,
@@ -95,9 +93,9 @@ func (c *LRUMap) Put(key string, value LRUData) {
 	}
 }
 
-func (c *LRUMap) TryGet(key string) (LRUData, bool) {
+func (c *LRUMap[TKey, TData]) TryGet(key TKey) (TData, bool) {
 	if c.maxsize == 0 {
-		return nil, false // cache disabled
+		return *new(TData), false // cache disabled
 	}
 
 	c.lock.Lock()
@@ -105,13 +103,13 @@ func (c *LRUMap) TryGet(key string) (LRUData, bool) {
 
 	val, ok := c.cache[key]
 	if !ok {
-		return nil, false
+		return *new(TData), false
 	}
 	c.moveNodeToTop(val)
 	return val.data, ok
 }
 
-func (c *LRUMap) moveNodeToTop(node *cacheNode) {
+func (c *LRUMap[TKey, TData]) moveNodeToTop(node *cacheNode[TKey, TData]) {
 	// (only called in critical section !)
 
 	if c.lfuHead == node { // fast case
@@ -144,7 +142,7 @@ func (c *LRUMap) moveNodeToTop(node *cacheNode) {
 	}
 }
 
-func (c *LRUMap) Size() int {
+func (c *LRUMap[TKey, TData]) Size() int {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	return len(c.cache)

dataext/lruMap_test.go

@@ -1,7 +1,7 @@
 package dataext
 
 import (
-	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
 	"math/rand"
 	"strconv"
 	"testing"
@@ -12,7 +12,7 @@ func init() {
 }
 
 func TestResultCache1(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key := randomKey()
@@ -39,7 +39,7 @@ func TestResultCache1(t *testing.T) {
 	if !ok {
 		t.Errorf("cache TryGet returned no value")
 	}
-	if !eq(cacheval, val) {
+	if cacheval != val {
 		t.Errorf("cache TryGet returned different value (%+v <> %+v)", cacheval, val)
 	}
 
@@ -50,7 +50,7 @@ func TestResultCache1(t *testing.T) {
 }
 
 func TestResultCache2(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -150,7 +150,7 @@ func TestResultCache2(t *testing.T) {
 }
 
 func TestResultCache3(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -160,20 +160,20 @@ func TestResultCache3(t *testing.T) {
 	cache.Put(key1, val1)
 	verifyLRUList(cache, t)
 
-	if val, ok := cache.TryGet(key1); !ok || !eq(val, val1) {
+	if val, ok := cache.TryGet(key1); !ok || val != val1 {
 		t.Errorf("Value in cache should be [val1]")
 	}
 
 	cache.Put(key1, val2)
 	verifyLRUList(cache, t)
 
-	if val, ok := cache.TryGet(key1); !ok || !eq(val, val2) {
+	if val, ok := cache.TryGet(key1); !ok || val != val2 {
 		t.Errorf("Value in cache should be [val2]")
 	}
 }
 
 // does a basic consistency check over the internal cache representation
-func verifyLRUList(cache *LRUMap, t *testing.T) {
+func verifyLRUList[TKey comparable, TData any](cache *LRUMap[TKey, TData], t *testing.T) {
 	size := 0
 
 	tailFound := false
@@ -250,23 +250,10 @@ func randomKey() string {
 	return strconv.FormatInt(rand.Int63(), 16)
 }
 
-func randomVal() LRUData {
+func randomVal() string {
 	v, err := langext.NewHexUUID()
 	if err != nil {
 		panic(err)
 	}
-	return &v
-}
-
-func eq(a LRUData, b LRUData) bool {
-	v1, ok1 := a.(*string)
-	v2, ok2 := b.(*string)
-	if ok1 && ok2 {
-		if v1 == nil || v2 == nil {
-			return false
-		}
-		return v1 == v2
-	}
-
-	return false
+	return v
 }

dataext/merge.go

@@ -14,7 +14,7 @@ func ObjectMerge[T1 any, T2 any](base T1, override T2) T1 {
 		fieldBase := reflBase.Field(i)
 		fieldOvrd := reflOvrd.Field(i)
 
-		if fieldBase.Kind() != reflect.Ptr || fieldOvrd.Kind() != reflect.Ptr {
+		if fieldBase.Kind() != reflect.Pointer || fieldOvrd.Kind() != reflect.Pointer {
 			continue
 		}
 

dataext/merge_test.go

@@ -1,7 +1,7 @@
 package dataext
 
 import (
-	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
@@ -25,17 +25,17 @@ func TestObjectMerge(t *testing.T) {
 
 	valueA := A{
 		Field1:   nil,
-		Field2:   langext.Ptr("99"),
-		Field3:   langext.Ptr(12.2),
+		Field2:   new("99"),
+		Field3:   new(12.2),
 		Field4:   nil,
 		OnlyA:    1,
 		DiffType: 2,
 	}
 
 	valueB := B{
-		Field1:   langext.Ptr(12),
+		Field1:   new(12),
 		Field2:   nil,
-		Field3:   langext.Ptr(13.2),
+		Field3:   new(13.2),
 		Field4:   nil,
 		OnlyB:    1,
 		DiffType: "X",
@@ -43,10 +43,10 @@ func TestObjectMerge(t *testing.T) {
 
 	valueMerge := ObjectMerge(valueA, valueB)
 
-	assertPtrEqual(t, "Field1", valueMerge.Field1, valueB.Field1)
-	assertPtrEqual(t, "Field2", valueMerge.Field2, valueA.Field2)
-	assertPtrEqual(t, "Field3", valueMerge.Field3, valueB.Field3)
-	assertPtrEqual(t, "Field4", valueMerge.Field4, nil)
+	tst.AssertIdentPtrEqual(t, "Field1", valueMerge.Field1, valueB.Field1)
+	tst.AssertIdentPtrEqual(t, "Field2", valueMerge.Field2, valueA.Field2)
+	tst.AssertIdentPtrEqual(t, "Field3", valueMerge.Field3, valueB.Field3)
+	tst.AssertIdentPtrEqual(t, "Field4", valueMerge.Field4, nil)
 
 }
 

dataext/multiMutex.go

@@ -0,0 +1,105 @@
+package dataext
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// MultiMutex is a simple map[key -> mutex]
+type MultiMutex[TKey comparable] struct {
+	mutextMap *SyncMap[TKey, *CASMutex]
+}
+
+func NewMultiMutex[TKey comparable]() *MultiMutex[TKey] {
+	return &MultiMutex[TKey]{
+		mutextMap: NewSyncMap[TKey, *CASMutex](),
+	}
+}
+
+// TryLockWithContext attempts to acquire the lock, blocking until resources
+// are available or ctx is done (timeout or cancellation).
+func (mm *MultiMutex[TKey]) TryLockWithContext(ctx context.Context, key TKey) bool {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.TryLockWithContext(ctx)
+}
+
+// Lock acquires the lock.
+// If it is currently held by others, Lock will wait until it has a chance to acquire it.
+func (mm *MultiMutex[TKey]) Lock(key TKey) {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	lck.Lock()
+}
+
+// TryLock attempts to acquire the lock without blocking.
+// Return false if someone is holding it now.
+func (mm *MultiMutex[TKey]) TryLock(key TKey) bool {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.TryLock()
+}
+
+// TryLockWithTimeout attempts to acquire the lock within a period of time.
+// Return false if spending time is more than duration and no chance to acquire it.
+func (mm *MultiMutex[TKey]) TryLockWithTimeout(key TKey, duration time.Duration) bool {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.TryLockWithTimeout(duration)
+}
+
+// Unlock releases the lock.
+func (mm *MultiMutex[TKey]) Unlock(key TKey) {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	lck.Unlock()
+}
+
+// RTryLockWithContext attempts to acquire the read lock, blocking until resources
+// are available or ctx is done (timeout or cancellation).
+func (mm *MultiMutex[TKey]) RTryLockWithContext(ctx context.Context, key TKey) bool {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.RTryLockWithContext(ctx)
+}
+
+// RLock acquires the read lock.
+// If it is currently held by others writing, RLock will wait until it has a chance to acquire it.
+func (mm *MultiMutex[TKey]) RLock(key TKey) {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	lck.RLock()
+}
+
+// RTryLock attempts to acquire the read lock without blocking.
+// Return false if someone is writing it now.
+func (mm *MultiMutex[TKey]) RTryLock(key TKey) bool {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.RTryLock()
+}
+
+// RTryLockWithTimeout attempts to acquire the read lock within a period of time.
+// Return false if spending time is more than duration and no chance to acquire it.
+func (mm *MultiMutex[TKey]) RTryLockWithTimeout(duration time.Duration, key TKey) bool {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.RTryLockWithTimeout(duration)
+}
+
+// RUnlock releases the read lock.
+func (mm *MultiMutex[TKey]) RUnlock(key TKey) {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	lck.RUnlock()
+}
+
+// RLocker returns a Locker interface that implements the Lock and Unlock methods
+// by calling CASMutex.RLock and CASMutex.RUnlock.
+func (mm *MultiMutex[TKey]) RLocker(key TKey) sync.Locker {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck.RLocker()
+}
+
+// Get returns a Locker interface
+func (mm *MultiMutex[TKey]) Get(key TKey) sync.Locker {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck
+}
+
+// GetCAS returns the underlying CASMutex
+func (mm *MultiMutex[TKey]) GetCAS(key TKey) *CASMutex {
+	lck := mm.mutextMap.GetAndSetIfNotContainsFunc(key, NewCASMutex)
+	return lck
+}

dataext/multiMutex_test.go

@@ -0,0 +1,89 @@
+package dataext
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestMultiMutex_LockDifferentKeys(t *testing.T) {
+	mm := NewMultiMutex[string]()
+	mm.Lock("a")
+	mm.Lock("b")
+	mm.Unlock("a")
+	mm.Unlock("b")
+}
+
+func TestMultiMutex_TryLockSameKey(t *testing.T) {
+	mm := NewMultiMutex[string]()
+	if !mm.TryLock("k") {
+		t.Fatal("TryLock should succeed first time")
+	}
+	if mm.TryLock("k") {
+		t.Fatal("TryLock should fail second time")
+	}
+	mm.Unlock("k")
+	if !mm.TryLock("k") {
+		t.Fatal("TryLock should succeed after unlock")
+	}
+	mm.Unlock("k")
+}
+
+func TestMultiMutex_TryLockDifferentKeys(t *testing.T) {
+	mm := NewMultiMutex[int]()
+	if !mm.TryLock(1) {
+		t.Fatal("TryLock(1) failed")
+	}
+	if !mm.TryLock(2) {
+		t.Fatal("TryLock(2) failed - different keys should be independent")
+	}
+	mm.Unlock(1)
+	mm.Unlock(2)
+}
+
+func TestMultiMutex_RLockMultiple(t *testing.T) {
+	mm := NewMultiMutex[string]()
+	if !mm.RTryLock("k") {
+		t.Fatal("first RTryLock failed")
+	}
+	if !mm.RTryLock("k") {
+		t.Fatal("second RTryLock failed")
+	}
+	mm.RUnlock("k")
+	mm.RUnlock("k")
+}
+
+func TestMultiMutex_TryLockWithTimeout(t *testing.T) {
+	mm := NewMultiMutex[string]()
+	mm.Lock("k")
+	if mm.TryLockWithTimeout("k", 10*time.Millisecond) {
+		t.Fatal("expected timeout failure")
+	}
+	mm.Unlock("k")
+}
+
+func TestMultiMutex_TryLockWithContext(t *testing.T) {
+	mm := NewMultiMutex[string]()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+	defer cancel()
+	if !mm.TryLockWithContext(ctx, "k") {
+		t.Fatal("TryLockWithContext should succeed on free key")
+	}
+	mm.Unlock("k")
+}
+
+func TestMultiMutex_GetAndGetCAS(t *testing.T) {
+	mm := NewMultiMutex[string]()
+	l := mm.Get("a")
+	if l == nil {
+		t.Fatal("Get returned nil")
+	}
+	cas := mm.GetCAS("a")
+	if cas == nil {
+		t.Fatal("GetCAS returned nil")
+	}
+	rl := mm.RLocker("a")
+	if rl == nil {
+		t.Fatal("RLocker returned nil")
+	}
+}

dataext/mutexSet.go

@@ -0,0 +1,53 @@
+package dataext
+
+import "sync"
+
+type MutexSet[T comparable] struct {
+	master sync.RWMutex
+	locks  map[T]*sync.RWMutex
+}
+
+func NewMutexSet[T comparable]() *MutexSet[T] {
+	return &MutexSet[T]{
+		master: sync.RWMutex{},
+		locks:  make(map[T]*sync.RWMutex),
+	}
+}
+
+func (ms *MutexSet[T]) get(key T) *sync.RWMutex {
+	ms.master.RLock()
+	if v, ok := ms.locks[key]; ok {
+		ms.master.RUnlock()
+		return v
+	}
+	ms.master.RUnlock()
+
+	// ---------
+
+	ms.master.Lock()
+	defer ms.master.Unlock()
+
+	if v, ok := ms.locks[key]; ok {
+		return v
+	}
+
+	m := &sync.RWMutex{}
+	ms.locks[key] = m
+	return m
+}
+
+func (ms *MutexSet[T]) Lock(key T) {
+	ms.get(key).Lock()
+}
+
+func (ms *MutexSet[T]) Unlock(key T) {
+	ms.get(key).Unlock()
+}
+
+func (ms *MutexSet[T]) RLock(key T) {
+	ms.get(key).RLock()
+}
+
+func (ms *MutexSet[T]) RUnlock(key T) {
+	ms.get(key).RUnlock()
+}

dataext/mutexSet_test.go

@@ -0,0 +1,51 @@
+package dataext
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestMutexSet_BasicLockUnlock(t *testing.T) {
+	ms := NewMutexSet[string]()
+	ms.Lock("a")
+	ms.Unlock("a")
+	ms.RLock("b")
+	ms.RUnlock("b")
+}
+
+func TestMutexSet_DifferentKeysIndependent(t *testing.T) {
+	ms := NewMutexSet[int]()
+	ms.Lock(1)
+	ms.Lock(2)
+	ms.Unlock(1)
+	ms.Unlock(2)
+}
+
+func TestMutexSet_SameKeyMutuallyExclusive(t *testing.T) {
+	ms := NewMutexSet[string]()
+	var counter int64
+	const n = 50
+	var wg sync.WaitGroup
+	wg.Add(n)
+	for i := 0; i < n; i++ {
+		go func() {
+			defer wg.Done()
+			ms.Lock("shared")
+			atomic.AddInt64(&counter, 1)
+			ms.Unlock("shared")
+		}()
+	}
+	wg.Wait()
+	if atomic.LoadInt64(&counter) != n {
+		t.Fatalf("got %d want %d", counter, n)
+	}
+}
+
+func TestMutexSet_RLockMultiple(t *testing.T) {
+	ms := NewMutexSet[string]()
+	ms.RLock("k")
+	ms.RLock("k")
+	ms.RUnlock("k")
+	ms.RUnlock("k")
+}

dataext/optional.go

@@ -0,0 +1,83 @@
+package dataext
+
+import (
+	"encoding/json"
+	"errors"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+)
+
+type JsonOpt[T any] struct {
+	isSet bool
+	value T
+}
+
+func NewJsonOpt[T any](v T) JsonOpt[T] {
+	return JsonOpt[T]{isSet: true, value: v}
+}
+
+func EmptyJsonOpt[T any]() JsonOpt[T] {
+	return JsonOpt[T]{isSet: false}
+}
+
+// MarshalJSON returns m as the JSON encoding of m.
+func (m JsonOpt[T]) MarshalJSON() ([]byte, error) {
+	if !m.isSet {
+		return []byte("null"), nil // actually this would be undefined - but undefined is not valid JSON
+	}
+
+	return json.Marshal(m.value)
+}
+
+// UnmarshalJSON sets *m to a copy of data.
+func (m *JsonOpt[T]) UnmarshalJSON(data []byte) error {
+	if m == nil {
+		return errors.New("JsonOpt: UnmarshalJSON on nil pointer")
+	}
+
+	m.isSet = true
+	return json.Unmarshal(data, &m.value)
+}
+
+func (m JsonOpt[T]) IsSet() bool {
+	return m.isSet
+}
+
+func (m JsonOpt[T]) IsUnset() bool {
+	return !m.isSet
+}
+
+func (m JsonOpt[T]) Value() (T, bool) {
+	if !m.isSet {
+		return *new(T), false
+	}
+	return m.value, true
+}
+
+func (m JsonOpt[T]) ValueOrNil() *T {
+	if !m.isSet {
+		return nil
+	}
+	return &m.value
+}
+
+func (m JsonOpt[T]) ValueDblPtrOrNil() **T {
+	if !m.isSet {
+		return nil
+	}
+	return langext.DblPtr(m.value)
+}
+
+func (m JsonOpt[T]) MustValue() T {
+	if !m.isSet {
+		panic("value not set")
+	}
+	return m.value
+}
+
+func (m JsonOpt[T]) IfSet(fn func(v T)) bool {
+	if !m.isSet {
+		return false
+	}
+	fn(m.value)
+	return true
+}

dataext/optional_test.go

@@ -0,0 +1,142 @@
+package dataext
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestJsonOpt_NewAndEmpty(t *testing.T) {
+	o := NewJsonOpt[int](42)
+	if !o.IsSet() {
+		t.Fatal("expected IsSet=true")
+	}
+	if o.IsUnset() {
+		t.Fatal("expected IsUnset=false")
+	}
+
+	e := EmptyJsonOpt[int]()
+	if e.IsSet() {
+		t.Fatal("expected IsSet=false")
+	}
+	if !e.IsUnset() {
+		t.Fatal("expected IsUnset=true")
+	}
+}
+
+func TestJsonOpt_Value(t *testing.T) {
+	o := NewJsonOpt[string]("hello")
+	v, ok := o.Value()
+	if !ok || v != "hello" {
+		t.Fatalf("got (%q,%v)", v, ok)
+	}
+
+	e := EmptyJsonOpt[string]()
+	v, ok = e.Value()
+	if ok || v != "" {
+		t.Fatalf("empty got (%q,%v)", v, ok)
+	}
+}
+
+func TestJsonOpt_ValueOrNil(t *testing.T) {
+	o := NewJsonOpt[int](7)
+	p := o.ValueOrNil()
+	if p == nil || *p != 7 {
+		t.Fatalf("expected ptr to 7")
+	}
+	e := EmptyJsonOpt[int]()
+	if e.ValueOrNil() != nil {
+		t.Fatal("expected nil")
+	}
+}
+
+func TestJsonOpt_ValueDblPtrOrNil(t *testing.T) {
+	o := NewJsonOpt[int](7)
+	p := o.ValueDblPtrOrNil()
+	if p == nil || *p == nil || **p != 7 {
+		t.Fatalf("expected double ptr to 7")
+	}
+	e := EmptyJsonOpt[int]()
+	if e.ValueDblPtrOrNil() != nil {
+		t.Fatal("expected nil")
+	}
+}
+
+func TestJsonOpt_MustValue(t *testing.T) {
+	o := NewJsonOpt[int](9)
+	if o.MustValue() != 9 {
+		t.Fatal("MustValue wrong")
+	}
+	defer func() {
+		if recover() == nil {
+			t.Fatal("expected panic")
+		}
+	}()
+	EmptyJsonOpt[int]().MustValue()
+}
+
+func TestJsonOpt_IfSet(t *testing.T) {
+	called := false
+	NewJsonOpt[int](1).IfSet(func(v int) {
+		called = true
+		if v != 1 {
+			t.Fatalf("v=%d", v)
+		}
+	})
+	if !called {
+		t.Fatal("IfSet did not invoke fn")
+	}
+
+	called = false
+	EmptyJsonOpt[int]().IfSet(func(v int) { called = true })
+	if called {
+		t.Fatal("IfSet invoked fn on empty")
+	}
+}
+
+func TestJsonOpt_MarshalJSON(t *testing.T) {
+	o := NewJsonOpt[int](5)
+	b, err := json.Marshal(o)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(b) != "5" {
+		t.Fatalf("got %s", b)
+	}
+
+	e := EmptyJsonOpt[int]()
+	b, err = json.Marshal(e)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(b) != "null" {
+		t.Fatalf("got %s", b)
+	}
+}
+
+func TestJsonOpt_UnmarshalJSON(t *testing.T) {
+	var o JsonOpt[int]
+	if err := json.Unmarshal([]byte("42"), &o); err != nil {
+		t.Fatal(err)
+	}
+	if !o.IsSet() {
+		t.Fatal("should be set")
+	}
+	if v, _ := o.Value(); v != 42 {
+		t.Fatalf("got %d", v)
+	}
+}
+
+func TestJsonOpt_StructWithJsonOpt(t *testing.T) {
+	type S struct {
+		A JsonOpt[int]    `json:"a"`
+		B JsonOpt[string] `json:"b"`
+	}
+	s := S{A: NewJsonOpt[int](1), B: EmptyJsonOpt[string]()}
+	b, err := json.Marshal(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(b) != `{"a":1,"b":null}` {
+		t.Fatalf("got %s", b)
+	}
+}

dataext/orderedMap.go

@@ -0,0 +1,187 @@
+package dataext
+
+import (
+	"fmt"
+	"iter"
+)
+
+// OrderedMap is like a normal map[TKey, TVal] - but its elements stay in order
+// NOT THREADSAFE !!!
+type OrderedMap[TKey comparable, TVal any] struct {
+	m map[TKey]*TVal
+	a []TKey
+}
+
+func NewOrderedMap[TKey comparable, TVal any](caps ...int) *OrderedMap[TKey, TVal] {
+	if len(caps) == 0 {
+		return &OrderedMap[TKey, TVal]{
+			m: make(map[TKey]*TVal),
+			a: make([]TKey, 0),
+		}
+	}
+
+	omcap := 0
+	for _, v := range caps {
+		omcap += v
+	}
+
+	return &OrderedMap[TKey, TVal]{
+		m: make(map[TKey]*TVal, omcap),
+		a: make([]TKey, 0, omcap),
+	}
+}
+
+func (o *OrderedMap[TKey, TVal]) Get(key TKey) (TVal, bool) {
+	v, ok := o.m[key]
+	if ok {
+		return *v, ok
+	}
+
+	return *new(TVal), false
+}
+
+func (o *OrderedMap[TKey, TVal]) GetOrNil(key TKey) *TVal {
+	v, ok := o.m[key]
+	if ok {
+		return v
+	}
+
+	return nil
+}
+
+func (o *OrderedMap[TKey, TVal]) GetOrDefault(key TKey, defaultVal TVal) TVal {
+	v, ok := o.m[key]
+	if ok {
+		return *v
+	}
+
+	return defaultVal
+}
+
+// Add adds the new value to the map
+// At the end of the map-ordering (even if key already exists, its then "moved" to the end)
+// returns true if key already existed
+func (o *OrderedMap[TKey, TVal]) Add(key TKey, val TVal) bool {
+	if _, ok := o.m[key]; ok {
+		o.remFromArray(key)
+		o.m[key] = &val
+		o.a = append(o.a, key)
+		return true
+	}
+
+	o.m[key] = &val
+	o.a = append(o.a, key)
+	return false
+}
+
+// AddOrReplace adds the new value to the map
+// Normally at the end of the map, but if teh key already exists, its only replaced
+// returns true if key already existed
+func (o *OrderedMap[TKey, TVal]) AddOrReplace(key TKey, val TVal) bool {
+	if _, ok := o.m[key]; ok {
+		o.m[key] = &val
+		return true
+	}
+
+	o.m[key] = &val
+	o.a = append(o.a, key)
+	return false
+}
+
+func (o *OrderedMap[TKey, TVal]) Remove(key TKey) bool {
+	if _, ok := o.m[key]; ok {
+		o.remFromArray(key)
+		delete(o.m, key)
+		return true
+	}
+
+	return false
+}
+
+func (o *OrderedMap[TKey, TVal]) Iterate() iter.Seq2[TKey, TVal] {
+	return func(yield func(TKey, TVal) bool) {
+		for _, v := range o.a {
+			if !yield(v, *o.m[v]) {
+				return
+			}
+		}
+	}
+}
+
+func (o *OrderedMap[TKey, TVal]) IterateValues() iter.Seq[TVal] {
+	return func(yield func(TVal) bool) {
+		for _, v := range o.a {
+			if !yield(*o.m[v]) {
+				return
+			}
+		}
+	}
+}
+
+func (o *OrderedMap[TKey, TVal]) IterateKeys() iter.Seq[TKey] {
+	return func(yield func(TKey) bool) {
+		for _, v := range o.a {
+			if !yield(v) {
+				return
+			}
+		}
+	}
+}
+
+func (o *OrderedMap[TKey, TVal]) Array() []TVal {
+	res := make([]TVal, len(o.a))
+	for i, v := range o.a {
+		res[i] = *o.m[v]
+	}
+
+	return res
+}
+
+func (o *OrderedMap[TKey, TVal]) Keys() []TKey {
+	res := make([]TKey, len(o.a))
+	for i, v := range o.a {
+		res[i] = v
+	}
+
+	return res
+}
+
+func (o *OrderedMap[TKey, TVal]) Clear() {
+	mapCap := max(len(o.m), cap(o.a))
+	o.m = make(map[TKey]*TVal, mapCap)
+	o.a = make([]TKey, 0, mapCap)
+}
+
+func (o *OrderedMap[TKey, TVal]) Size() int {
+	return len(o.a)
+}
+
+func (o *OrderedMap[TKey, TVal]) Capacity() int {
+	return min(cap(o.a), len(o.m))
+}
+
+func (o *OrderedMap[TKey, TVal]) Contains(key TKey) bool {
+	_, ok := o.m[key]
+	return ok
+}
+
+func (o *OrderedMap[TKey, TVal]) IndexOf(key TKey) int {
+	for i, v := range o.a {
+		if v == key {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func (o *OrderedMap[TKey, TVal]) remFromArray(key TKey) {
+	for i, v := range o.a {
+		if v == key {
+			o.a = append(o.a[:i], o.a[i+1:]...)
+			return
+		}
+	}
+
+	panic(fmt.Sprintf("Failed to remove key from OrderedMap -- key '%v' not found", key))
+}

dataext/orderedMap_test.go

@@ -0,0 +1,429 @@
+package dataext
+
+import (
+	"slices"
+	"testing"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+)
+
+func TestOrderedMapNew(t *testing.T) {
+	m := NewOrderedMap[string, int](4)
+
+	tst.AssertEqual(t, m.Size(), 0)
+	tst.AssertArrayEqual(t, m.Keys(), []string{})
+	tst.AssertArrayEqual(t, m.Array(), []int{})
+}
+
+func TestOrderedMapNewNoCap(t *testing.T) {
+	m := NewOrderedMap[string, int]()
+
+	tst.AssertEqual(t, m.Size(), 0)
+	tst.AssertArrayEqual(t, m.Keys(), []string{})
+	tst.AssertArrayEqual(t, m.Array(), []int{})
+}
+
+func TestOrderedMapNewWIthCap(t *testing.T) {
+	m := NewOrderedMap[string, int](4)
+
+	tst.AssertEqual(t, m.Size(), 0)
+	tst.AssertArrayEqual(t, m.Keys(), []string{})
+	tst.AssertArrayEqual(t, m.Array(), []int{})
+}
+
+func TestOrderedMapAddAndGet(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	tst.AssertFalse(t, m.Add("a", 1))
+	tst.AssertFalse(t, m.Add("b", 2))
+	tst.AssertFalse(t, m.Add("c", 3))
+
+	tst.AssertEqual(t, m.Size(), 3)
+
+	v, ok := m.Get("a")
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, v, 1)
+
+	v, ok = m.Get("b")
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, v, 2)
+
+	v, ok = m.Get("c")
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, v, 3)
+
+	v, ok = m.Get("missing")
+	tst.AssertFalse(t, ok)
+	tst.AssertEqual(t, v, 0)
+}
+
+func TestOrderedMapOrderPreserved(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	m.Add("first", 1)
+	m.Add("second", 2)
+	m.Add("third", 3)
+	m.Add("fourth", 4)
+
+	tst.AssertArrayEqual(t, m.Keys(), []string{"first", "second", "third", "fourth"})
+	tst.AssertArrayEqual(t, m.Array(), []int{1, 2, 3, 4})
+}
+
+func TestOrderedMapAddMovesExistingToEnd(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	tst.AssertTrue(t, m.Add("a", 10))
+
+	tst.AssertArrayEqual(t, m.Keys(), []string{"b", "c", "a"})
+	tst.AssertArrayEqual(t, m.Array(), []int{2, 3, 10})
+
+	v, ok := m.Get("a")
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, v, 10)
+}
+
+func TestOrderedMapAddOrReplaceKeepsOrder(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	tst.AssertTrue(t, m.AddOrReplace("b", 99))
+
+	tst.AssertArrayEqual(t, m.Keys(), []string{"a", "b", "c"})
+	tst.AssertArrayEqual(t, m.Array(), []int{1, 99, 3})
+
+	v, ok := m.Get("b")
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, v, 99)
+}
+
+func TestOrderedMapAddOrReplaceNew(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	tst.AssertFalse(t, m.AddOrReplace("a", 1))
+	tst.AssertFalse(t, m.AddOrReplace("b", 2))
+
+	tst.AssertArrayEqual(t, m.Keys(), []string{"a", "b"})
+	tst.AssertArrayEqual(t, m.Array(), []int{1, 2})
+}
+
+func TestOrderedMapGetOrNil(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 42)
+
+	v := m.GetOrNil("a")
+	if v == nil {
+		t.Errorf("expected non-nil pointer")
+		return
+	}
+	tst.AssertEqual(t, *v, 42)
+
+	v = m.GetOrNil("missing")
+	if v != nil {
+		t.Errorf("expected nil pointer")
+	}
+}
+
+func TestOrderedMapGetOrDefault(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 42)
+
+	tst.AssertEqual(t, m.GetOrDefault("a", -1), 42)
+	tst.AssertEqual(t, m.GetOrDefault("missing", -1), -1)
+}
+
+func TestOrderedMapRemove(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	tst.AssertTrue(t, m.Remove("b"))
+
+	tst.AssertEqual(t, m.Size(), 2)
+	tst.AssertArrayEqual(t, m.Keys(), []string{"a", "c"})
+	tst.AssertArrayEqual(t, m.Array(), []int{1, 3})
+
+	_, ok := m.Get("b")
+	tst.AssertFalse(t, ok)
+	tst.AssertFalse(t, m.Contains("b"))
+}
+
+func TestOrderedMapRemoveMissing(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	m.Add("a", 1)
+	tst.AssertFalse(t, m.Remove("missing"))
+
+	tst.AssertEqual(t, m.Size(), 1)
+	tst.AssertArrayEqual(t, m.Keys(), []string{"a"})
+}
+
+func TestOrderedMapRemoveFirstAndLast(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+	m.Add("d", 4)
+
+	tst.AssertTrue(t, m.Remove("a"))
+	tst.AssertTrue(t, m.Remove("d"))
+
+	tst.AssertArrayEqual(t, m.Keys(), []string{"b", "c"})
+	tst.AssertArrayEqual(t, m.Array(), []int{2, 3})
+}
+
+func TestOrderedMapContains(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+
+	tst.AssertTrue(t, m.Contains("a"))
+	tst.AssertFalse(t, m.Contains("b"))
+
+	m.Remove("a")
+	tst.AssertFalse(t, m.Contains("a"))
+}
+
+func TestOrderedMapIndexOf(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	tst.AssertEqual(t, m.IndexOf("a"), 0)
+	tst.AssertEqual(t, m.IndexOf("b"), 1)
+	tst.AssertEqual(t, m.IndexOf("c"), 2)
+	tst.AssertEqual(t, m.IndexOf("missing"), -1)
+
+	m.Add("a", 10) // moves to end
+	tst.AssertEqual(t, m.IndexOf("a"), 2)
+	tst.AssertEqual(t, m.IndexOf("b"), 0)
+	tst.AssertEqual(t, m.IndexOf("c"), 1)
+}
+
+func TestOrderedMapClear(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	m.Clear()
+
+	tst.AssertEqual(t, m.Size(), 0)
+	tst.AssertArrayEqual(t, m.Keys(), []string{})
+	tst.AssertArrayEqual(t, m.Array(), []int{})
+	tst.AssertFalse(t, m.Contains("a"))
+
+	m.Add("x", 99)
+	tst.AssertEqual(t, m.Size(), 1)
+	tst.AssertArrayEqual(t, m.Keys(), []string{"x"})
+}
+
+func TestOrderedMapSize(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	tst.AssertEqual(t, m.Size(), 0)
+
+	m.Add("a", 1)
+	tst.AssertEqual(t, m.Size(), 1)
+
+	m.Add("b", 2)
+	tst.AssertEqual(t, m.Size(), 2)
+
+	m.Add("a", 10) // replaces, size stays
+	tst.AssertEqual(t, m.Size(), 2)
+
+	m.AddOrReplace("b", 20) // replaces, size stays
+	tst.AssertEqual(t, m.Size(), 2)
+
+	m.Remove("a")
+	tst.AssertEqual(t, m.Size(), 1)
+}
+
+func TestOrderedMapIterate(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	got := make([]int, 0)
+	for v := range m.IterateValues() {
+		got = append(got, v)
+	}
+
+	tst.AssertArrayEqual(t, got, []int{1, 2, 3})
+}
+
+func TestOrderedMapIterateBreak(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	got := make([]int, 0)
+	for v := range m.IterateValues() {
+		got = append(got, v)
+		if v == 2 {
+			break
+		}
+	}
+
+	tst.AssertArrayEqual(t, got, []int{1, 2})
+}
+
+func TestOrderedMapIterateSeq2(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	gotKeys := make([]string, 0)
+	gotVals := make([]int, 0)
+	for k, v := range m.Iterate() {
+		gotKeys = append(gotKeys, k)
+		gotVals = append(gotVals, v)
+	}
+
+	tst.AssertArrayEqual(t, gotKeys, []string{"a", "b", "c"})
+	tst.AssertArrayEqual(t, gotVals, []int{1, 2, 3})
+}
+
+func TestOrderedMapIterateSeq2Break(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	gotKeys := make([]string, 0)
+	gotVals := make([]int, 0)
+	for k, v := range m.Iterate() {
+		gotKeys = append(gotKeys, k)
+		gotVals = append(gotVals, v)
+		if k == "b" {
+			break
+		}
+	}
+
+	tst.AssertArrayEqual(t, gotKeys, []string{"a", "b"})
+	tst.AssertArrayEqual(t, gotVals, []int{1, 2})
+}
+
+func TestOrderedMapIterateSeq2AfterReorder(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+	m.Add("a", 10) // moves "a" to end with new value
+
+	gotKeys := make([]string, 0)
+	gotVals := make([]int, 0)
+	for k, v := range m.Iterate() {
+		gotKeys = append(gotKeys, k)
+		gotVals = append(gotVals, v)
+	}
+
+	tst.AssertArrayEqual(t, gotKeys, []string{"b", "c", "a"})
+	tst.AssertArrayEqual(t, gotVals, []int{2, 3, 10})
+}
+
+func TestOrderedMapIterateKeys(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	got := make([]string, 0)
+	for k := range m.IterateKeys() {
+		got = append(got, k)
+	}
+
+	tst.AssertArrayEqual(t, got, []string{"a", "b", "c"})
+}
+
+func TestOrderedMapIterateKeysBreak(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+	m.Add("c", 3)
+
+	got := make([]string, 0)
+	for k := range m.IterateKeys() {
+		got = append(got, k)
+		if k == "b" {
+			break
+		}
+	}
+
+	tst.AssertArrayEqual(t, got, []string{"a", "b"})
+}
+
+func TestOrderedMapArrayIsCopy(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+
+	arr := m.Array()
+	arr[0] = 999
+
+	tst.AssertArrayEqual(t, m.Array(), []int{1, 2})
+}
+
+func TestOrderedMapKeysIsCopy(t *testing.T) {
+	m := NewOrderedMap[string, int](0)
+	m.Add("a", 1)
+	m.Add("b", 2)
+
+	keys := m.Keys()
+	keys[0] = "zzz"
+
+	tst.AssertArrayEqual(t, m.Keys(), []string{"a", "b"})
+}
+
+func TestOrderedMapIntKey(t *testing.T) {
+	m := NewOrderedMap[int, string](0)
+	m.Add(3, "three")
+	m.Add(1, "one")
+	m.Add(2, "two")
+
+	tst.AssertArrayEqual(t, m.Keys(), []int{3, 1, 2})
+	tst.AssertArrayEqual(t, m.Array(), []string{"three", "one", "two"})
+
+	v, ok := m.Get(1)
+	tst.AssertTrue(t, ok)
+	tst.AssertEqual(t, v, "one")
+}
+
+func TestOrderedMapStress(t *testing.T) {
+	m := NewOrderedMap[int, int](0)
+
+	for i := 0; i < 100; i++ {
+		m.Add(i, i*10)
+	}
+	tst.AssertEqual(t, m.Size(), 100)
+
+	for i := 0; i < 100; i++ {
+		v, ok := m.Get(i)
+		tst.AssertTrue(t, ok)
+		tst.AssertEqual(t, v, i*10)
+	}
+
+	for i := 0; i < 50; i++ {
+		m.Remove(i * 2)
+	}
+	tst.AssertEqual(t, m.Size(), 50)
+
+	expected := make([]int, 0, 50)
+	for i := 0; i < 50; i++ {
+		expected = append(expected, i*2+1)
+	}
+	keys := m.Keys()
+	slices.Sort(keys)
+	tst.AssertArrayEqual(t, keys, expected)
+}

dataext/pubsub.go

@@ -0,0 +1,267 @@
+package dataext
+
+import (
+	"context"
+	"iter"
+	"sync"
+	"time"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/syncext"
+	"github.com/rs/xid"
+)
+
+// PubSub is a simple Pub/Sub Broker
+// Clients can subscribe to a namespace and receive published messages on this namespace
+// Messages are broadcast to all subscribers
+type PubSub[TNamespace comparable, TData any] struct {
+	masterLock *sync.Mutex
+
+	subscriptions map[TNamespace][]*pubSubSubscription[TNamespace, TData]
+}
+
+type PubSubSubscription interface {
+	Unsubscribe()
+}
+
+type pubSubSubscription[TNamespace comparable, TData any] struct {
+	ID string
+
+	parent    *PubSub[TNamespace, TData]
+	namespace TNamespace
+
+	subLock *sync.Mutex
+
+	Func func(TData)
+	Chan chan TData
+
+	UnsubChan chan bool
+}
+
+func (p *pubSubSubscription[TNamespace, TData]) Unsubscribe() {
+	p.parent.unsubscribe(p)
+}
+
+func NewPubSub[TNamespace comparable, TData any](capacity int) *PubSub[TNamespace, TData] {
+	return &PubSub[TNamespace, TData]{
+		masterLock:    &sync.Mutex{},
+		subscriptions: make(map[TNamespace][]*pubSubSubscription[TNamespace, TData], capacity),
+	}
+}
+
+func (ps *PubSub[TNamespace, TData]) Namespaces() []TNamespace {
+	ps.masterLock.Lock()
+	defer ps.masterLock.Unlock()
+
+	return langext.MapKeyArr(ps.subscriptions)
+}
+
+func (ps *PubSub[TNamespace, TData]) SubscriberCount(ns TNamespace) int {
+	ps.masterLock.Lock()
+	defer ps.masterLock.Unlock()
+
+	return len(ps.subscriptions[ns])
+}
+
+// Publish sends `data` to all subscriber
+// But unbuffered - if one is currently not listening, we skip (the actualReceiver < subscriber)
+func (ps *PubSub[TNamespace, TData]) Publish(ns TNamespace, data TData) (subscriber int, actualReceiver int) {
+	ps.masterLock.Lock()
+	subs := langext.ArrCopy(ps.subscriptions[ns])
+	ps.masterLock.Unlock()
+
+	subscriber = len(subs)
+	actualReceiver = 0
+
+	for _, sub := range subs {
+		func() {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+				actualReceiver++
+			} else if sub.Chan != nil {
+				msgSent := syncext.WriteNonBlocking(sub.Chan, data)
+				if msgSent {
+					actualReceiver++
+				}
+			}
+		}()
+	}
+
+	return subscriber, actualReceiver
+}
+
+// PublishWithContext sends `data` to all subscriber
+// buffered - if one is currently not listening, we wait (but error out when the context runs out)
+func (ps *PubSub[TNamespace, TData]) PublishWithContext(ctx context.Context, ns TNamespace, data TData) (subscriber int, actualReceiver int, err error) {
+	ps.masterLock.Lock()
+	subs := langext.ArrCopy(ps.subscriptions[ns])
+	ps.masterLock.Unlock()
+
+	subscriber = len(subs)
+	actualReceiver = 0
+
+	for _, sub := range subs {
+		err := func() error {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+				actualReceiver++
+			} else if sub.Chan != nil {
+				err := syncext.WriteChannelWithContext(ctx, sub.Chan, data)
+				if err != nil {
+					return err
+				}
+				actualReceiver++
+			}
+
+			return nil
+		}()
+		if err != nil {
+			return subscriber, actualReceiver, err
+		}
+	}
+
+	return subscriber, actualReceiver, nil
+}
+
+// PublishWithTimeout sends `data` to all subscriber
+// buffered - if one is currently not listening, we wait (but wait at most `timeout` - if the timeout is exceeded then actualReceiver < subscriber)
+func (ps *PubSub[TNamespace, TData]) PublishWithTimeout(ns TNamespace, data TData, timeout time.Duration) (subscriber int, actualReceiver int) {
+	ps.masterLock.Lock()
+	subs := langext.ArrCopy(ps.subscriptions[ns])
+	ps.masterLock.Unlock()
+
+	subscriber = len(subs)
+	actualReceiver = 0
+
+	for _, sub := range subs {
+		func() {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+				actualReceiver++
+			} else if sub.Chan != nil {
+				ok := syncext.WriteChannelWithTimeout(sub.Chan, data, timeout)
+				if ok {
+					actualReceiver++
+				}
+			}
+		}()
+	}
+
+	return subscriber, actualReceiver
+}
+
+// PublishAsync sends `data` to all subscriber
+// does not wait for subscriber (this method returns immediately), waits at most {timeout} seconds on channels (async)
+func (ps *PubSub[TNamespace, TData]) PublishAsync(ns TNamespace, data TData, timeout time.Duration) (subscriber int) {
+	ps.masterLock.Lock()
+	subs := langext.ArrCopy(ps.subscriptions[ns])
+	ps.masterLock.Unlock()
+
+	subscriber = len(subs)
+
+	for _, sub := range subs {
+		go func() {
+			sub.subLock.Lock()
+			defer sub.subLock.Unlock()
+
+			if sub.Func != nil {
+				go func() { sub.Func(data) }()
+			} else if sub.Chan != nil {
+				_ = syncext.WriteChannelWithTimeout(sub.Chan, data, timeout)
+			}
+		}()
+	}
+
+	return subscriber
+}
+
+func (ps *PubSub[TNamespace, TData]) SubscribeByCallback(ns TNamespace, fn func(TData)) PubSubSubscription {
+	ps.masterLock.Lock()
+	defer ps.masterLock.Unlock()
+
+	sub := &pubSubSubscription[TNamespace, TData]{ID: xid.New().String(), namespace: ns, parent: ps, subLock: &sync.Mutex{}, Func: fn, UnsubChan: nil}
+
+	ps.subscriptions[ns] = append(ps.subscriptions[ns], sub)
+
+	return sub
+}
+
+func (ps *PubSub[TNamespace, TData]) SubscribeByChan(ns TNamespace, chanBufferSize int) (chan TData, PubSubSubscription) {
+	ps.masterLock.Lock()
+	defer ps.masterLock.Unlock()
+
+	msgCh := make(chan TData, chanBufferSize)
+
+	sub := &pubSubSubscription[TNamespace, TData]{ID: xid.New().String(), namespace: ns, parent: ps, subLock: &sync.Mutex{}, Chan: msgCh, UnsubChan: nil}
+
+	ps.subscriptions[ns] = append(ps.subscriptions[ns], sub)
+
+	return msgCh, sub
+}
+
+func (ps *PubSub[TNamespace, TData]) SubscribeByIter(ns TNamespace, chanBufferSize int) (iter.Seq[TData], PubSubSubscription) {
+	ps.masterLock.Lock()
+	defer ps.masterLock.Unlock()
+
+	msgCh := make(chan TData, chanBufferSize)
+	unsubChan := make(chan bool, 8)
+
+	sub := &pubSubSubscription[TNamespace, TData]{ID: xid.New().String(), namespace: ns, parent: ps, subLock: &sync.Mutex{}, Chan: msgCh, UnsubChan: unsubChan}
+
+	ps.subscriptions[ns] = append(ps.subscriptions[ns], sub)
+
+	iterFun := func(yield func(TData) bool) {
+		for {
+			select {
+			case msg := <-msgCh:
+				if !yield(msg) {
+					sub.Unsubscribe()
+					return
+				}
+			case <-sub.UnsubChan:
+				sub.Unsubscribe()
+				return
+			}
+		}
+	}
+
+	return iterFun, sub
+}
+
+func (ps *PubSub[TNamespace, TData]) unsubscribe(p *pubSubSubscription[TNamespace, TData]) {
+	ps.masterLock.Lock()
+	defer ps.masterLock.Unlock()
+
+	p.subLock.Lock()
+	defer p.subLock.Unlock()
+
+	if p.Chan != nil {
+		close(p.Chan)
+		p.Chan = nil
+	}
+	if p.UnsubChan != nil {
+		syncext.WriteNonBlocking(p.UnsubChan, true)
+		close(p.UnsubChan)
+		p.UnsubChan = nil
+	}
+
+	ps.subscriptions[p.namespace] = langext.ArrFilter(ps.subscriptions[p.namespace], func(v *pubSubSubscription[TNamespace, TData]) bool {
+		return v.ID != p.ID
+	})
+	if len(ps.subscriptions[p.namespace]) == 0 {
+		delete(ps.subscriptions, p.namespace)
+	}
+}

dataext/pubsub_test.go

@@ -0,0 +1,438 @@
+package dataext
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestNewPubSub(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+	if ps == nil {
+		t.Fatal("NewPubSub returned nil")
+	}
+	if ps.masterLock == nil {
+		t.Fatal("masterLock is nil")
+	}
+	if ps.subscriptions == nil {
+		t.Fatal("subscriptions is nil")
+	}
+}
+
+func TestPubSub_Namespaces(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	// Initially no namespaces
+	namespaces := ps.Namespaces()
+	if len(namespaces) != 0 {
+		t.Fatalf("Expected 0 namespaces, got %d", len(namespaces))
+	}
+
+	// Add a subscription to create a namespace
+	_, sub1 := ps.SubscribeByChan("test-ns1", 1)
+	defer sub1.Unsubscribe()
+
+	// Add another subscription to a different namespace
+	_, sub2 := ps.SubscribeByChan("test-ns2", 1)
+	defer sub2.Unsubscribe()
+
+	// Check namespaces
+	namespaces = ps.Namespaces()
+	if len(namespaces) != 2 {
+		t.Fatalf("Expected 2 namespaces, got %d", len(namespaces))
+	}
+
+	// Check if namespaces contain the expected values
+	found1, found2 := false, false
+	for _, ns := range namespaces {
+		if ns == "test-ns1" {
+			found1 = true
+		}
+		if ns == "test-ns2" {
+			found2 = true
+		}
+	}
+
+	if !found1 || !found2 {
+		t.Fatalf("Expected to find both namespaces, found ns1: %v, ns2: %v", found1, found2)
+	}
+}
+
+func TestPubSub_SubscribeByCallback(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	var received string
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	callback := func(msg string) {
+		received = msg
+		wg.Done()
+	}
+
+	sub := ps.SubscribeByCallback("test-ns", callback)
+	defer sub.Unsubscribe()
+
+	// Publish a message
+	subs, receivers := ps.Publish("test-ns", "hello")
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Wait for the callback to be executed
+	wg.Wait()
+
+	if received != "hello" {
+		t.Fatalf("Expected to receive 'hello', got '%s'", received)
+	}
+}
+
+func TestPubSub_SubscribeByChan(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	ch, sub := ps.SubscribeByChan("test-ns", 1)
+	defer sub.Unsubscribe()
+
+	// Publish a message
+	subs, receivers := ps.Publish("test-ns", "hello")
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Read from the channel with a timeout to avoid blocking
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+}
+
+func TestPubSub_SubscribeByIter(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	iterSeq, sub := ps.SubscribeByIter("test-ns", 1)
+	defer sub.Unsubscribe()
+
+	// Channel to communicate when message is received
+	done := make(chan bool)
+	goroutineDone := make(chan struct{})
+	received := false
+
+	// Start a goroutine to use the iterator
+	go func() {
+		defer close(goroutineDone)
+		for msg := range iterSeq {
+			if msg == "hello" {
+				received = true
+				done <- true
+				return // Stop iteration — triggers Unsubscribe via yield returning false
+			}
+		}
+	}()
+
+	// Give time for the iterator to start
+	time.Sleep(100 * time.Millisecond)
+
+	// Publish a message
+	ps.Publish("test-ns", "hello")
+
+	// Wait for the message to be received or timeout
+	select {
+	case <-done:
+		if !received {
+			t.Fatal("Message was received but not 'hello'")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Wait for the goroutine to fully exit so Unsubscribe (triggered by the
+	// iterator cleanup when yield returns false) has completed.
+	select {
+	case <-goroutineDone:
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for goroutine to finish")
+	}
+
+	subCount := ps.SubscriberCount("test-ns")
+	if subCount != 0 {
+		t.Fatalf("Expected 0 receivers, got %d", subCount)
+	}
+}
+
+func TestPubSub_Publish(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	// Test publishing to a namespace with no subscribers
+	subs, receivers := ps.Publish("empty-ns", "hello")
+	if subs != 0 {
+		t.Fatalf("Expected 0 subscribers, got %d", subs)
+	}
+	if receivers != 0 {
+		t.Fatalf("Expected 0 receivers, got %d", receivers)
+	}
+
+	// Add a subscriber
+	ch, sub := ps.SubscribeByChan("test-ns", 1)
+	defer sub.Unsubscribe()
+
+	// Publish a message
+	subs, receivers = ps.Publish("test-ns", "hello")
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Test non-blocking behavior with a full channel
+	// First fill the channel
+	ps.Publish("test-ns", "fill")
+
+	// Now publish again - this should not block but skip the receiver
+	subs, receivers = ps.Publish("test-ns", "overflow")
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	// The receiver count might be 0 if the channel is full
+
+	// Drain the channel
+	<-ch
+}
+
+func TestPubSub_PublishWithTimeout(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	// Add a subscriber with a channel
+	ch, sub := ps.SubscribeByChan("test-ns", 1)
+	defer sub.Unsubscribe()
+
+	// Publish with a timeout
+	subs, receivers := ps.PublishWithTimeout("test-ns", "hello", 100*time.Millisecond)
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Fill the channel
+	ps.Publish("test-ns", "fill")
+
+	// Test timeout behavior with a full channel
+	start := time.Now()
+	subs, receivers = ps.PublishWithTimeout("test-ns", "timeout-test", 50*time.Millisecond)
+	elapsed := time.Since(start)
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	// The receiver count should be 0 if the timeout occurred
+	if elapsed < 50*time.Millisecond {
+		t.Fatalf("Expected to wait at least 50ms, only waited %v", elapsed)
+	}
+
+	// Drain the channel
+	<-ch
+}
+
+func TestPubSub_PublishWithContext(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	// Add a subscriber with a channel
+	ch, sub := ps.SubscribeByChan("test-ns", 1)
+	defer sub.Unsubscribe()
+
+	// Create a context
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	// Publish with context
+	subs, receivers, err := ps.PublishWithContext(ctx, "test-ns", "hello")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Fill the channel
+	ps.Publish("test-ns", "fill")
+
+	// Test context cancellation with a full channel
+	ctx, cancel = context.WithCancel(context.Background())
+
+	// Cancel the context after a short delay
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		cancel()
+	}()
+
+	start := time.Now()
+	subs, receivers, err = ps.PublishWithContext(ctx, "test-ns", "context-test")
+	elapsed := time.Since(start)
+
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+
+	// Should get a context canceled error
+	if err == nil {
+		t.Fatal("Expected context canceled error, got nil")
+	}
+
+	if elapsed < 50*time.Millisecond {
+		t.Fatalf("Expected to wait at least 50ms, only waited %v", elapsed)
+	}
+
+	// Drain the channel
+	<-ch
+}
+
+func TestPubSub_Unsubscribe(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	// Add a subscriber
+	ch, sub := ps.SubscribeByChan("test-ns", 1)
+
+	// Publish a message
+	subs, receivers := ps.Publish("test-ns", "hello")
+	if subs != 1 {
+		t.Fatalf("Expected 1 subscriber, got %d", subs)
+	}
+	if receivers != 1 {
+		t.Fatalf("Expected 1 receiver, got %d", receivers)
+	}
+
+	// Verify the message was received
+	select {
+	case msg := <-ch:
+		if msg != "hello" {
+			t.Fatalf("Expected to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message")
+	}
+
+	// Unsubscribe
+	sub.Unsubscribe()
+
+	// Publish again
+	subs, receivers = ps.Publish("test-ns", "after-unsub")
+	if subs != 0 {
+		t.Fatalf("Expected 0 subscribers after unsubscribe, got %d", subs)
+	}
+	if receivers != 0 {
+		t.Fatalf("Expected 0 receivers after unsubscribe, got %d", receivers)
+	}
+
+	// Check that the namespace is removed
+	namespaces := ps.Namespaces()
+	if len(namespaces) != 0 {
+		t.Fatalf("Expected 0 namespaces after unsubscribe, got %d", len(namespaces))
+	}
+}
+
+func TestPubSub_MultipleSubscribers(t *testing.T) {
+	ps := NewPubSub[string, string](10)
+
+	// Add multiple subscribers
+	ch1, sub1 := ps.SubscribeByChan("test-ns", 1)
+	defer sub1.Unsubscribe()
+
+	ch2, sub2 := ps.SubscribeByChan("test-ns", 1)
+	defer sub2.Unsubscribe()
+
+	var received string
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	sub3 := ps.SubscribeByCallback("test-ns", func(msg string) {
+		received = msg
+		wg.Done()
+	})
+	defer sub3.Unsubscribe()
+
+	// Publish a message
+	subs, receivers := ps.Publish("test-ns", "hello")
+	if subs != 3 {
+		t.Fatalf("Expected 3 subscribers, got %d", subs)
+	}
+	if receivers != 3 {
+		t.Fatalf("Expected 3 receivers, got %d", receivers)
+	}
+
+	// Verify the message was received by all subscribers
+	select {
+	case msg := <-ch1:
+		if msg != "hello" {
+			t.Fatalf("Expected ch1 to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message on ch1")
+	}
+
+	select {
+	case msg := <-ch2:
+		if msg != "hello" {
+			t.Fatalf("Expected ch2 to receive 'hello', got '%s'", msg)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Timed out waiting for message on ch2")
+	}
+
+	// Wait for the callback
+	wg.Wait()
+
+	if received != "hello" {
+		t.Fatalf("Expected callback to receive 'hello', got '%s'", received)
+	}
+}

dataext/ringBuffer.go

@@ -0,0 +1,144 @@
+package dataext
+
+import "iter"
+
+type RingBuffer[T any] struct {
+	items    []T //
+	capacity int // max number of items the buffer can hold
+	size     int // how many items are in the buffer
+	head     int // ptr to next item
+}
+
+func NewRingBuffer[T any](capacity int) *RingBuffer[T] {
+	return &RingBuffer[T]{
+		items:    make([]T, capacity),
+		capacity: capacity,
+		size:     0,
+		head:     0,
+	}
+}
+
+func (rb *RingBuffer[T]) Push(item T) {
+	if rb.size < rb.capacity {
+		rb.size++
+	}
+	rb.items[rb.head] = item
+	rb.head = (rb.head + 1) % rb.capacity
+}
+
+func (rb *RingBuffer[T]) PushPop(item T) *T {
+	if rb.size < rb.capacity {
+		rb.size++
+		rb.items[rb.head] = item
+		rb.head = (rb.head + 1) % rb.capacity
+		return nil
+	} else {
+		prev := rb.items[rb.head]
+		rb.items[rb.head] = item
+		rb.head = (rb.head + 1) % rb.capacity
+		return &prev
+	}
+}
+
+func (rb *RingBuffer[T]) Peek() (T, bool) {
+	if rb.size == 0 {
+		return *new(T), false
+	}
+	return rb.items[(rb.head-1+rb.capacity)%rb.capacity], true
+}
+
+func (rb *RingBuffer[T]) Items() []T {
+	if rb.size < rb.capacity {
+		return rb.items[:rb.size]
+	}
+	return append(rb.items[rb.head:], rb.items[:rb.head]...)
+}
+
+func (rb *RingBuffer[T]) Size() int {
+	return rb.size
+}
+
+func (rb *RingBuffer[T]) Capacity() int {
+	return rb.capacity
+}
+
+func (rb *RingBuffer[T]) Clear() {
+	rb.size = 0
+	rb.head = 0
+}
+
+func (rb *RingBuffer[T]) IsFull() bool {
+	return rb.size == rb.capacity
+}
+
+func (rb *RingBuffer[T]) At(i int) T {
+	if i < 0 || i >= rb.size {
+		panic("Index out of bounds")
+	}
+	if rb.size < rb.capacity {
+		return rb.items[i]
+	}
+	return rb.items[(rb.head+i)%rb.capacity]
+}
+
+func (rb *RingBuffer[T]) Get(i int) (T, bool) {
+	if i < 0 || i >= rb.size {
+		return *new(T), false
+	}
+	if rb.size < rb.capacity {
+		return rb.items[i], true
+	}
+	return rb.items[(rb.head+i)%rb.capacity], true
+}
+
+func (rb *RingBuffer[T]) Iter() iter.Seq[T] {
+	return func(yield func(T) bool) {
+		for i := 0; i < rb.size; i++ {
+			if !yield(rb.At(i)) {
+				return
+			}
+		}
+	}
+}
+
+func (rb *RingBuffer[T]) Iter2() iter.Seq2[int, T] {
+	return func(yield func(int, T) bool) {
+		for i := 0; i < rb.size; i++ {
+			if !yield(i, rb.At(i)) {
+				return
+			}
+		}
+	}
+}
+
+func (rb *RingBuffer[T]) Remove(fnEqual func(v T) bool) int {
+	// Mike [2024-11-13]: I *really* tried to write an in-place algorithm to remove elements
+	//                    But after carful consideration, I left that as an exercise for future readers
+	//                    It is, suprisingly, non-trivial, especially because the head-ptr must be weirdly updated
+	//                    And out At() method does not work correctly with {head<>0 && size<capacity}
+
+	dc := 0
+	b := make([]T, rb.capacity)
+	bsize := 0
+
+	for i := 0; i < rb.size; i++ {
+		comp := rb.At(i)
+		if fnEqual(comp) {
+			dc++
+		} else {
+			b[bsize] = comp
+			bsize++
+		}
+	}
+
+	if dc == 0 {
+		return 0
+	}
+
+	rb.items = b
+	rb.size = bsize
+	rb.head = bsize % rb.capacity
+
+	return dc
+
+}

dataext/ringBuffer_test.go

@@ -0,0 +1,447 @@
+package dataext
+
+import "testing"
+
+func TestRingBufferPushAddsItem(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	if rb.Size() != 1 {
+		t.Errorf("Expected size 1, got %d", rb.Size())
+	}
+	if item, _ := rb.Peek(); item != 1 {
+		t.Errorf("Expected item 1, got %d", item)
+	}
+}
+
+func TestRingBufferPushPopReturnsOldestItem(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	if item := rb.PushPop(4); item == nil || *item != 1 {
+		t.Errorf("Expected item 1, got %v", item)
+	}
+}
+
+func TestRingBufferPeekReturnsLastPushedItem(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	if item, _ := rb.Peek(); item != 2 {
+		t.Errorf("Expected item 2, got %d", item)
+	}
+}
+
+func TestRingBufferOverflow1(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1) // overriden
+	rb.Push(2) // overriden
+	rb.Push(3)
+	rb.Push(9)
+	rb.Push(4)
+	rb.Push(5)
+	rb.Push(7)
+	if rb.Size() != 5 {
+		t.Errorf("Expected size 4, got %d", rb.Size())
+	}
+	expected := []int{3, 9, 4, 5, 7}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferItemsReturnsAllItems(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	items := rb.Items()
+	expected := []int{1, 2, 3}
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferClearEmptiesBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Clear()
+	if rb.Size() != 0 {
+		t.Errorf("Expected size 0, got %d", rb.Size())
+	}
+}
+
+func TestRingBufferIsFullReturnsTrueWhenFull(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	if !rb.IsFull() {
+		t.Errorf("Expected buffer to be full")
+	}
+}
+
+func TestRingBufferAtReturnsCorrectItem(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	if item := rb.At(1); item != 2 {
+		t.Errorf("Expected item 2, got %d", item)
+	}
+}
+
+func TestRingBufferGetReturnsCorrectItem(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	if item, ok := rb.Get(1); !ok || item != 2 {
+		t.Errorf("Expected item 2, got %d", item)
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	rb.Push(2)
+	rb.Push(4)
+	removed := rb.Remove(func(v int) bool { return v == 2 })
+	if removed != 2 {
+		t.Errorf("Expected 2 items removed, got %d", removed)
+	}
+	if rb.Size() != 3 {
+		t.Errorf("Expected size 3, got %d", rb.Size())
+	}
+	expected := []int{1, 3, 4}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems2(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	rb.Push(2)
+	rb.Push(4)
+	removed := rb.Remove(func(v int) bool { return v == 3 })
+	if removed != 1 {
+		t.Errorf("Expected 2 items removed, got %d", removed)
+	}
+	if rb.Size() != 4 {
+		t.Errorf("Expected size 3, got %d", rb.Size())
+	}
+	expected := []int{1, 2, 2, 4}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems3(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	rb.Push(9)
+	rb.Push(4)
+	removed := rb.Remove(func(v int) bool { return v == 3 })
+	if removed != 1 {
+		t.Errorf("Expected 2 items removed, got %d", removed)
+	}
+	if rb.Size() != 4 {
+		t.Errorf("Expected size 3, got %d", rb.Size())
+	}
+	expected := []int{1, 2, 9, 4}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems4(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1) // overriden
+	rb.Push(2) // overriden
+	rb.Push(3)
+	rb.Push(9)
+	rb.Push(4)
+	rb.Push(5)
+	rb.Push(7)
+	removed := rb.Remove(func(v int) bool { return v == 7 })
+	if removed != 1 {
+		t.Errorf("Expected 1 items removed, got %d", removed)
+	}
+	if rb.Size() != 4 {
+		t.Errorf("Expected size 4, got %d", rb.Size())
+	}
+	expected := []int{3, 9, 4, 5}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems5(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1) // overriden
+	rb.Push(2) // overriden
+	rb.Push(3)
+	rb.Push(9)
+	rb.Push(4)
+	rb.Push(5)
+	rb.Push(7)
+	removed := rb.Remove(func(v int) bool { return v == 3 })
+	if removed != 1 {
+		t.Errorf("Expected 1 items removed, got %d", removed)
+	}
+	if rb.Size() != 4 {
+		t.Errorf("Expected size 4, got %d", rb.Size())
+	}
+	expected := []int{9, 4, 5, 7}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems6(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1) // overriden
+	rb.Push(2) // overriden
+	rb.Push(3)
+	rb.Push(9)
+	rb.Push(4)
+	rb.Push(5)
+	rb.Push(7)
+	removed := rb.Remove(func(v int) bool { return v == 1 })
+	if removed != 0 {
+		t.Errorf("Expected 0 items removed, got %d", removed)
+	}
+	if rb.Size() != 5 {
+		t.Errorf("Expected size 5, got %d", rb.Size())
+	}
+	expected := []int{3, 9, 4, 5, 7}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+	if !rb.IsFull() {
+		t.Errorf("Expected buffer to not be full")
+	}
+}
+
+func TestRingBufferRemoveDeletesMatchingItems7(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1) // overriden
+	rb.Push(2) // overriden
+	rb.Push(3)
+	rb.Push(9)
+	rb.Push(4)
+	rb.Push(5)
+	rb.Push(7)
+	removed := rb.Remove(func(v int) bool { return v == 9 })
+	if removed != 1 {
+		t.Errorf("Expected 1 items removed, got %d", removed)
+	}
+	if rb.Size() != 4 {
+		t.Errorf("Expected size 4, got %d", rb.Size())
+	}
+	expected := []int{3, 4, 5, 7}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+	if rb.IsFull() {
+		t.Errorf("Expected buffer to not be full")
+	}
+}
+
+func TestRingBufferAddItemsToFullRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	rb.Push(4)
+	if rb.Size() != 3 {
+		t.Errorf("Expected size 3, got %d", rb.Size())
+	}
+	expected := []int{2, 3, 4}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferAddItemsToNonFullRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	if rb.Size() != 2 {
+		t.Errorf("Expected size 2, got %d", rb.Size())
+	}
+	expected := []int{1, 2}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveItemsFromNonFullRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	removed := rb.Remove(func(v int) bool { return v == 1 })
+	if removed != 1 {
+		t.Errorf("Expected 1 item removed, got %d", removed)
+	}
+	if rb.Size() != 1 {
+		t.Errorf("Expected size 1, got %d", rb.Size())
+	}
+	expected := []int{2}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveItemsFromFullRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	removed := rb.Remove(func(v int) bool { return v == 2 })
+	if removed != 1 {
+		t.Errorf("Expected 1 item removed, got %d", removed)
+	}
+	if rb.Size() != 2 {
+		t.Errorf("Expected size 2, got %d", rb.Size())
+	}
+	expected := []int{1, 3}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveMultipleItemsFromRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](5)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	rb.Push(2)
+	rb.Push(4)
+	removed := rb.Remove(func(v int) bool { return v == 2 })
+	if removed != 2 {
+		t.Errorf("Expected 2 items removed, got %d", removed)
+	}
+	if rb.Size() != 3 {
+		t.Errorf("Expected size 3, got %d", rb.Size())
+	}
+	expected := []int{1, 3, 4}
+	items := rb.Items()
+	for i, item := range items {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+	}
+}
+
+func TestRingBufferRemoveAllItemsFromRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	removed := rb.Remove(func(v int) bool { return true })
+	if removed != 3 {
+		t.Errorf("Expected 3 items removed, got %d", removed)
+	}
+	if rb.Size() != 0 {
+		t.Errorf("Expected size 0, got %d", rb.Size())
+	}
+}
+
+func TestRingBufferRemoveNoItemsFromRingBuffer(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	removed := rb.Remove(func(v int) bool { return false })
+	if removed != 0 {
+		t.Errorf("Expected 0 items removed, got %d", removed)
+	}
+	if rb.Size() != 3 {
+		t.Errorf("Expected size 3, got %d", rb.Size())
+	}
+}
+
+func TestRingBufferIteratesOverAllItems(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	expected := []int{1, 2, 3}
+	i := 0
+	for item := range rb.Iter() {
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+		i++
+	}
+	if i != len(expected) {
+		t.Errorf("Expected to iterate over %d items, but iterated over %d", len(expected), i)
+	}
+}
+
+func TestRingBufferIter2IteratesOverAllItemsWithIndices(t *testing.T) {
+	rb := NewRingBuffer[int](3)
+	rb.Push(1)
+	rb.Push(2)
+	rb.Push(3)
+	expected := []int{1, 2, 3}
+	i := 0
+	for index, item := range rb.Iter2() {
+		if index != i {
+			t.Errorf("Expected index %d, got %d", i, index)
+		}
+		if item != expected[i] {
+			t.Errorf("Expected item %d, got %d", expected[i], item)
+		}
+		i++
+	}
+	if i != len(expected) {
+		t.Errorf("Expected to iterate over %d items, but iterated over %d", len(expected), i)
+	}
+}

dataext/stack.go

@@ -0,0 +1,115 @@
+package dataext
+
+import (
+	"errors"
+	"sync"
+)
+
+var ErrEmptyStack = errors.New("stack is empty")
+
+type Stack[T any] struct {
+	lock *sync.Mutex
+	data []T
+}
+
+func NewStack[T any](threadsafe bool, initialCapacity int) *Stack[T] {
+	var lck *sync.Mutex = nil
+	if threadsafe {
+		lck = &sync.Mutex{}
+	}
+	return &Stack[T]{
+		lock: lck,
+		data: make([]T, 0, initialCapacity),
+	}
+}
+
+func (s *Stack[T]) Push(v T) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	s.data = append(s.data, v)
+}
+
+func (s *Stack[T]) Pop() (T, error) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+	if l == 0 {
+		return *new(T), ErrEmptyStack
+	}
+
+	result := s.data[l-1]
+	s.data = s.data[:l-1]
+
+	return result, nil
+}
+
+func (s *Stack[T]) OptPop() *T {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+	if l == 0 {
+		return nil
+	}
+
+	result := s.data[l-1]
+	s.data = s.data[:l-1]
+
+	return new(result)
+}
+
+func (s *Stack[T]) Peek() (T, error) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+
+	if l == 0 {
+		return *new(T), ErrEmptyStack
+	}
+
+	return s.data[l-1], nil
+}
+
+func (s *Stack[T]) OptPeek() *T {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+
+	if l == 0 {
+		return nil
+	}
+
+	return new(s.data[l-1])
+}
+
+func (s *Stack[T]) Length() int {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	return len(s.data)
+}
+
+func (s *Stack[T]) Empty() bool {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	return len(s.data) == 0
+}

dataext/stack_test.go

@@ -0,0 +1,98 @@
+package dataext
+
+import (
+	"errors"
+	"sync"
+	"testing"
+)
+
+func TestStack_PushPop(t *testing.T) {
+	s := NewStack[int](false, 4)
+	s.Push(1)
+	s.Push(2)
+	s.Push(3)
+
+	if s.Length() != 3 {
+		t.Fatalf("Length=%d", s.Length())
+	}
+	if s.Empty() {
+		t.Fatal("should not be empty")
+	}
+
+	v, err := s.Pop()
+	if err != nil || v != 3 {
+		t.Fatalf("Pop got (%d,%v)", v, err)
+	}
+	v, err = s.Pop()
+	if err != nil || v != 2 {
+		t.Fatalf("Pop got (%d,%v)", v, err)
+	}
+	v, err = s.Pop()
+	if err != nil || v != 1 {
+		t.Fatalf("Pop got (%d,%v)", v, err)
+	}
+}
+
+func TestStack_PopEmpty(t *testing.T) {
+	s := NewStack[int](false, 0)
+	_, err := s.Pop()
+	if !errors.Is(err, ErrEmptyStack) {
+		t.Fatalf("expected ErrEmptyStack, got %v", err)
+	}
+	if !s.Empty() {
+		t.Fatal("should be empty")
+	}
+}
+
+func TestStack_Peek(t *testing.T) {
+	s := NewStack[string](false, 0)
+	if _, err := s.Peek(); !errors.Is(err, ErrEmptyStack) {
+		t.Fatalf("expected ErrEmptyStack got %v", err)
+	}
+	s.Push("a")
+	s.Push("b")
+	v, err := s.Peek()
+	if err != nil || v != "b" {
+		t.Fatalf("Peek got (%q,%v)", v, err)
+	}
+	if s.Length() != 2 {
+		t.Fatal("Peek must not pop")
+	}
+}
+
+func TestStack_OptPopOptPeek(t *testing.T) {
+	s := NewStack[int](false, 0)
+	if s.OptPop() != nil {
+		t.Fatal("OptPop on empty should return nil")
+	}
+	if s.OptPeek() != nil {
+		t.Fatal("OptPeek on empty should return nil")
+	}
+	s.Push(7)
+	if p := s.OptPeek(); p == nil || *p != 7 {
+		t.Fatalf("OptPeek bad")
+	}
+	if p := s.OptPop(); p == nil || *p != 7 {
+		t.Fatalf("OptPop bad")
+	}
+	if !s.Empty() {
+		t.Fatal("should be empty after OptPop")
+	}
+}
+
+func TestStack_ThreadSafe(t *testing.T) {
+	s := NewStack[int](true, 0)
+	var wg sync.WaitGroup
+	const n = 200
+	wg.Add(n)
+	for i := 0; i < n; i++ {
+		go func(v int) {
+			defer wg.Done()
+			s.Push(v)
+		}(i)
+	}
+	wg.Wait()
+	if s.Length() != n {
+		t.Fatalf("Length=%d want %d", s.Length(), n)
+	}
+}

dataext/structHash.go

@@ -6,7 +6,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
 	"hash"
 	"io"
 	"reflect"
@@ -82,7 +82,7 @@ func binarize(writer io.Writer, dat reflect.Value, opt StructHashOptions) error
 
 	err = binary.Write(writer, binary.LittleEndian, uint8(dat.Kind()))
 	switch dat.Kind() {
-	case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice, reflect.Interface:
+	case reflect.Pointer, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice, reflect.Interface:
 		if dat.IsNil() {
 			err = binary.Write(writer, binary.LittleEndian, uint64(0))
 			if err != nil {

dataext/structHash_test.go

@@ -1,8 +1,7 @@
 package dataext
 
 import (
-	"encoding/hex"
-	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
 	"testing"
 )
 
@@ -18,14 +17,14 @@ func noErrStructHash(t *testing.T, dat any, opt ...StructHashOptions) []byte {
 
 func TestStructHashSimple(t *testing.T) {
 
-	assertEqual(t, "209bf774af36cc3a045c152d9f1269ef3684ad819c1359ee73ff0283a308fefa", noErrStructHash(t, "Hello"))
-	assertEqual(t, "c32f3626b981ae2997db656f3acad3f1dc9d30ef6b6d14296c023e391b25f71a", noErrStructHash(t, 0))
-	assertEqual(t, "01b781b03e9586b257d387057dfc70d9f06051e7d3c1e709a57e13cc8daf3e35", noErrStructHash(t, []byte{}))
-	assertEqual(t, "93e1dcd45c732fe0079b0fb3204c7c803f0921835f6bfee2e6ff263e73eed53c", noErrStructHash(t, []int{}))
-	assertEqual(t, "54f637a376aad55b3160d98ebbcae8099b70d91b9400df23fb3709855d59800a", noErrStructHash(t, []int{1, 2, 3}))
-	assertEqual(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", noErrStructHash(t, nil))
-	assertEqual(t, "349a7db91aa78fd30bbaa7c7f9c7bfb2fcfe72869b4861162a96713a852f60d3", noErrStructHash(t, []any{1, "", nil}))
-	assertEqual(t, "ca51aab87808bf0062a4a024de6aac0c2bad54275cc857a4944569f89fd245ad", noErrStructHash(t, struct{}{}))
+	tst.AssertHexEqual(t, "209bf774af36cc3a045c152d9f1269ef3684ad819c1359ee73ff0283a308fefa", noErrStructHash(t, "Hello"))
+	tst.AssertHexEqual(t, "c32f3626b981ae2997db656f3acad3f1dc9d30ef6b6d14296c023e391b25f71a", noErrStructHash(t, 0))
+	tst.AssertHexEqual(t, "01b781b03e9586b257d387057dfc70d9f06051e7d3c1e709a57e13cc8daf3e35", noErrStructHash(t, []byte{}))
+	tst.AssertHexEqual(t, "93e1dcd45c732fe0079b0fb3204c7c803f0921835f6bfee2e6ff263e73eed53c", noErrStructHash(t, []int{}))
+	tst.AssertHexEqual(t, "54f637a376aad55b3160d98ebbcae8099b70d91b9400df23fb3709855d59800a", noErrStructHash(t, []int{1, 2, 3}))
+	tst.AssertHexEqual(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", noErrStructHash(t, nil))
+	tst.AssertHexEqual(t, "349a7db91aa78fd30bbaa7c7f9c7bfb2fcfe72869b4861162a96713a852f60d3", noErrStructHash(t, []any{1, "", nil}))
+	tst.AssertHexEqual(t, "ca51aab87808bf0062a4a024de6aac0c2bad54275cc857a4944569f89fd245ad", noErrStructHash(t, struct{}{}))
 
 }
 
@@ -37,16 +36,16 @@ func TestStructHashSimpleStruct(t *testing.T) {
 		F3 *int
 	}
 
-	assertEqual(t, "a90bff751c70c738bb5cfc9b108e783fa9c19c0bc9273458e0aaee6e74aa1b92", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "a90bff751c70c738bb5cfc9b108e783fa9c19c0bc9273458e0aaee6e74aa1b92", noErrStructHash(t, t0{
 		F1: 10,
 		F2: []string{"1", "2", "3"},
 		F3: nil,
 	}))
 
-	assertEqual(t, "5d09090dc34ac59dd645f197a255f653387723de3afa1b614721ea5a081c675f", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "5d09090dc34ac59dd645f197a255f653387723de3afa1b614721ea5a081c675f", noErrStructHash(t, t0{
 		F1: 10,
 		F2: []string{"1", "2", "3"},
-		F3: langext.Ptr(99),
+		F3: new(99),
 	}))
 
 }
@@ -64,7 +63,7 @@ func TestStructHashLayeredStruct(t *testing.T) {
 		SV3 t1_1
 	}
 
-	assertEqual(t, "fd4ca071fb40a288fee4b7a3dfdaab577b30cb8f80f81ec511e7afd72dc3b469", noErrStructHash(t, t1_2{
+	tst.AssertHexEqual(t, "fd4ca071fb40a288fee4b7a3dfdaab577b30cb8f80f81ec511e7afd72dc3b469", noErrStructHash(t, t1_2{
 		SV1: nil,
 		SV2: nil,
 		SV3: t1_1{
@@ -73,7 +72,7 @@ func TestStructHashLayeredStruct(t *testing.T) {
 			F15: false,
 		},
 	}))
-	assertEqual(t, "3fbf7c67d8121deda075cc86319a4e32d71744feb2cebf89b43bc682f072a029", noErrStructHash(t, t1_2{
+	tst.AssertHexEqual(t, "3fbf7c67d8121deda075cc86319a4e32d71744feb2cebf89b43bc682f072a029", noErrStructHash(t, t1_2{
 		SV1: nil,
 		SV2: &t1_1{},
 		SV3: t1_1{
@@ -82,7 +81,7 @@ func TestStructHashLayeredStruct(t *testing.T) {
 			F15: true,
 		},
 	}))
-	assertEqual(t, "b1791ccd1b346c3ede5bbffda85555adcd8216b93ffca23f14fe175ec47c5104", noErrStructHash(t, t1_2{
+	tst.AssertHexEqual(t, "b1791ccd1b346c3ede5bbffda85555adcd8216b93ffca23f14fe175ec47c5104", noErrStructHash(t, t1_2{
 		SV1: &t1_1{},
 		SV2: &t1_1{},
 		SV3: t1_1{
@@ -101,7 +100,7 @@ func TestStructHashMap(t *testing.T) {
 		F2 map[string]int
 	}
 
-	assertEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
 		F1: 10,
 		F2: map[string]int{
 			"x": 1,
@@ -110,7 +109,7 @@ func TestStructHashMap(t *testing.T) {
 		},
 	}))
 
-	assertEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
 		F1: 10,
 		F2: map[string]int{
 			"a": 99,
@@ -128,16 +127,9 @@ func TestStructHashMap(t *testing.T) {
 	m3["x"] = 1
 	m3["a"] = 2
 
-	assertEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
+	tst.AssertHexEqual(t, "d50c53ad1fafb448c33fddd5aca01a86a2edf669ce2ecab07ba6fe877951d824", noErrStructHash(t, t0{
 		F1: 10,
 		F2: m3,
 	}))
 
 }
-
-func assertEqual(t *testing.T, expected string, actual []byte) {
-	actualStr := hex.EncodeToString(actual)
-	if actualStr != expected {
-		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actualStr, expected)
-	}
-}

dataext/syncMap.go

@@ -0,0 +1,256 @@
+package dataext
+
+import "sync"
+
+// SyncMap is a thread-safe map implementation for generic key-value pairs.
+// All functions aresafe to be called in parallel.
+type SyncMap[TKey comparable, TData any] struct {
+	data map[TKey]TData
+	lock sync.Mutex
+}
+
+func NewSyncMap[TKey comparable, TData any]() *SyncMap[TKey, TData] {
+	return &SyncMap[TKey, TData]{data: make(map[TKey]TData), lock: sync.Mutex{}}
+}
+
+// Set sets the value for the provided key
+func (s *SyncMap[TKey, TData]) Set(key TKey, data TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	s.data[key] = data
+}
+
+// SetIfNotContains sets the value for the provided key if it does not already exist
+func (s *SyncMap[TKey, TData]) SetIfNotContains(key TKey, data TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if _, existsInPreState := s.data[key]; existsInPreState {
+		return false
+	}
+
+	s.data[key] = data
+
+	return true
+}
+
+// SetIfNotContainsFunc sets the value for the provided key using the provided function
+func (s *SyncMap[TKey, TData]) SetIfNotContainsFunc(key TKey, data func() TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if _, existsInPreState := s.data[key]; existsInPreState {
+		return false
+	}
+
+	s.data[key] = data()
+
+	return true
+}
+
+// Get retrieves the value for the provided key
+func (s *SyncMap[TKey, TData]) Get(key TKey) (TData, bool) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		return v, true
+	} else {
+		return *new(TData), false
+	}
+}
+
+// GetAndSetIfNotContains returns the existing value if the key exists.
+// Otherwise, it sets the provided value and returns it.
+func (s *SyncMap[TKey, TData]) GetAndSetIfNotContains(key TKey, data TData) TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		return v
+	} else {
+		s.data[key] = data
+		return data
+	}
+}
+
+// GetAndSetIfNotContainsFunc returns the existing value if the key exists.
+// Otherwise, it calls the provided function to generate the value, sets it, and returns it.
+func (s *SyncMap[TKey, TData]) GetAndSetIfNotContainsFunc(key TKey, data func() TData) TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		return v
+	} else {
+		dataObj := data()
+		s.data[key] = dataObj
+		return dataObj
+	}
+}
+
+// Delete removes the entry with the provided key and returns true if the key existed before.
+func (s *SyncMap[TKey, TData]) Delete(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	delete(s.data, key)
+
+	return ok
+}
+
+// DeleteIf deletes all entries that match the provided function and returns the number of removed entries.
+func (s *SyncMap[TKey, TData]) DeleteIf(fn func(key TKey, data TData) bool) int {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	rm := 0
+	for k, v := range s.data {
+		if fn(k, v) {
+			delete(s.data, k)
+			rm++
+		}
+	}
+
+	return rm
+}
+
+// UpdateIfExists updates the value if the key exists, otherwise it does nothing.
+func (s *SyncMap[TKey, TData]) UpdateIfExists(key TKey, fn func(data TData) TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		s.data[key] = fn(v)
+		return true
+	} else {
+		return false
+	}
+}
+
+// UpdateOrInsert updates the value if the key exists, otherwise it inserts the provided `insertValue`.
+func (s *SyncMap[TKey, TData]) UpdateOrInsert(key TKey, fn func(data TData) TData, insertValue TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		s.data[key] = fn(v)
+		return true
+	} else {
+		s.data[key] = insertValue
+		return false
+	}
+}
+
+// Clear removes all entries from the map.
+func (s *SyncMap[TKey, TData]) Clear() {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[TKey]TData)
+}
+
+// Contains checks if the map contains the provided key.
+func (s *SyncMap[TKey, TData]) Contains(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	return ok
+}
+
+// GetAllKeys returns a copy (!) of all keys in the map.
+func (s *SyncMap[TKey, TData]) GetAllKeys() []TKey {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TKey, 0, len(s.data))
+
+	for k := range s.data {
+		r = append(r, k)
+	}
+
+	return r
+}
+
+// GetAllValues returns a copy (!) of all values in the map.
+func (s *SyncMap[TKey, TData]) GetAllValues() []TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TData, 0, len(s.data))
+
+	for _, v := range s.data {
+		r = append(r, v)
+	}
+
+	return r
+}
+
+// Count returns the number of entries in the map.
+func (s *SyncMap[TKey, TData]) Count() int {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	return len(s.data)
+}

dataext/syncMap_test.go

@@ -0,0 +1,176 @@
+package dataext
+
+import (
+	"sort"
+	"sync"
+	"testing"
+)
+
+func TestSyncMap_SetGet(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	m.Set("a", 1)
+	v, ok := m.Get("a")
+	if !ok || v != 1 {
+		t.Fatalf("got (%d,%v)", v, ok)
+	}
+	if _, ok := m.Get("missing"); ok {
+		t.Fatal("expected missing")
+	}
+}
+
+func TestSyncMap_SetIfNotContains(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	if !m.SetIfNotContains("a", 1) {
+		t.Fatal("first set should succeed")
+	}
+	if m.SetIfNotContains("a", 2) {
+		t.Fatal("second set should fail")
+	}
+	v, _ := m.Get("a")
+	if v != 1 {
+		t.Fatalf("expected unchanged got %d", v)
+	}
+}
+
+func TestSyncMap_SetIfNotContainsFunc(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	calls := 0
+	if !m.SetIfNotContainsFunc("a", func() int { calls++; return 5 }) {
+		t.Fatal("first should succeed")
+	}
+	if m.SetIfNotContainsFunc("a", func() int { calls++; return 6 }) {
+		t.Fatal("second should fail")
+	}
+	if calls != 1 {
+		t.Fatalf("calls=%d want 1", calls)
+	}
+}
+
+func TestSyncMap_GetAndSetIfNotContains(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	if v := m.GetAndSetIfNotContains("a", 10); v != 10 {
+		t.Fatalf("got %d", v)
+	}
+	if v := m.GetAndSetIfNotContains("a", 99); v != 10 {
+		t.Fatalf("got %d", v)
+	}
+}
+
+func TestSyncMap_GetAndSetIfNotContainsFunc(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	calls := 0
+	if v := m.GetAndSetIfNotContainsFunc("a", func() int { calls++; return 1 }); v != 1 {
+		t.Fatalf("got %d", v)
+	}
+	if v := m.GetAndSetIfNotContainsFunc("a", func() int { calls++; return 2 }); v != 1 {
+		t.Fatalf("got %d", v)
+	}
+	if calls != 1 {
+		t.Fatalf("calls=%d", calls)
+	}
+}
+
+func TestSyncMap_Delete(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	m.Set("a", 1)
+	if !m.Delete("a") {
+		t.Fatal("delete existing returned false")
+	}
+	if m.Delete("a") {
+		t.Fatal("delete missing returned true")
+	}
+}
+
+func TestSyncMap_DeleteIf(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	m.Set("a", 1)
+	m.Set("b", 2)
+	m.Set("c", 3)
+	rm := m.DeleteIf(func(k string, v int) bool { return v%2 == 1 })
+	if rm != 2 {
+		t.Fatalf("removed=%d", rm)
+	}
+	if m.Count() != 1 {
+		t.Fatalf("count=%d", m.Count())
+	}
+}
+
+func TestSyncMap_UpdateIfExists(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	if m.UpdateIfExists("a", func(v int) int { return v + 1 }) {
+		t.Fatal("should be false on missing key")
+	}
+	m.Set("a", 5)
+	if !m.UpdateIfExists("a", func(v int) int { return v + 1 }) {
+		t.Fatal("should be true on existing")
+	}
+	v, _ := m.Get("a")
+	if v != 6 {
+		t.Fatalf("v=%d", v)
+	}
+}
+
+func TestSyncMap_UpdateOrInsert(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	if m.UpdateOrInsert("a", func(v int) int { return v + 1 }, 100) {
+		t.Fatal("should return false on insert")
+	}
+	if v, _ := m.Get("a"); v != 100 {
+		t.Fatalf("v=%d", v)
+	}
+	if !m.UpdateOrInsert("a", func(v int) int { return v + 1 }, 100) {
+		t.Fatal("should return true on update")
+	}
+	if v, _ := m.Get("a"); v != 101 {
+		t.Fatalf("v=%d", v)
+	}
+}
+
+func TestSyncMap_ClearContains(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	m.Set("a", 1)
+	if !m.Contains("a") {
+		t.Fatal("Contains should be true")
+	}
+	m.Clear()
+	if m.Contains("a") {
+		t.Fatal("after Clear should be false")
+	}
+	if m.Count() != 0 {
+		t.Fatalf("count=%d", m.Count())
+	}
+}
+
+func TestSyncMap_GetAllKeysValues(t *testing.T) {
+	m := NewSyncMap[string, int]()
+	m.Set("a", 1)
+	m.Set("b", 2)
+	m.Set("c", 3)
+	keys := m.GetAllKeys()
+	sort.Strings(keys)
+	if len(keys) != 3 || keys[0] != "a" || keys[2] != "c" {
+		t.Fatalf("keys=%v", keys)
+	}
+	vals := m.GetAllValues()
+	sort.Ints(vals)
+	if len(vals) != 3 || vals[0] != 1 || vals[2] != 3 {
+		t.Fatalf("vals=%v", vals)
+	}
+}
+
+func TestSyncMap_Concurrent(t *testing.T) {
+	m := NewSyncMap[int, int]()
+	var wg sync.WaitGroup
+	const n = 200
+	wg.Add(n)
+	for i := 0; i < n; i++ {
+		go func(k int) {
+			defer wg.Done()
+			m.Set(k, k*2)
+		}(i)
+	}
+	wg.Wait()
+	if m.Count() != n {
+		t.Fatalf("count=%d want %d", m.Count(), n)
+	}
+}

dataext/syncRingSet.go

@@ -0,0 +1,143 @@
+package dataext
+
+import "sync"
+
+type SyncRingSet[TData comparable] struct {
+	data map[TData]bool
+	lock sync.Mutex
+	ring *RingBuffer[TData]
+}
+
+func NewSyncRingSet[TData comparable](capacity int) *SyncRingSet[TData] {
+	return &SyncRingSet[TData]{
+		data: make(map[TData]bool, capacity+1),
+		lock: sync.Mutex{},
+		ring: NewRingBuffer[TData](capacity),
+	}
+}
+
+// Add adds `value` to the set
+// returns true  if the value was actually inserted (value did not exist beforehand)
+// returns false if the value already existed
+func (s *SyncRingSet[TData]) Add(value TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	_, existsInPreState := s.data[value]
+	if existsInPreState {
+		return false
+	}
+
+	prev := s.ring.PushPop(value)
+
+	s.data[value] = true
+	if prev != nil {
+		delete(s.data, *prev)
+	}
+
+	return true
+}
+
+func (s *SyncRingSet[TData]) AddAll(values []TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	for _, value := range values {
+		_, existsInPreState := s.data[value]
+		if existsInPreState {
+			continue
+		}
+
+		prev := s.ring.PushPop(value)
+
+		s.data[value] = true
+		if prev != nil {
+			delete(s.data, *prev)
+		}
+	}
+}
+
+func (s *SyncRingSet[TData]) Remove(value TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	_, existsInPreState := s.data[value]
+	if !existsInPreState {
+		return false
+	}
+
+	delete(s.data, value)
+	s.ring.Remove(func(v TData) bool { return value == v })
+
+	return true
+}
+
+func (s *SyncRingSet[TData]) RemoveAll(values []TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	for _, value := range values {
+		delete(s.data, value)
+		s.ring.Remove(func(v TData) bool { return value == v })
+	}
+}
+
+func (s *SyncRingSet[TData]) Contains(value TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	_, ok := s.data[value]
+
+	return ok
+}
+
+func (s *SyncRingSet[TData]) Get() []TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	r := make([]TData, 0, len(s.data))
+
+	for k := range s.data {
+		r = append(r, k)
+	}
+
+	return r
+}
+
+// AddIfNotContains
+// returns true  if the value was actually added (value did not exist beforehand)
+// returns false if the value already existed
+func (s *SyncRingSet[TData]) AddIfNotContains(key TData) bool {
+	return s.Add(key)
+}
+
+// RemoveIfContains
+// returns true  if the value was actually removed (value did exist beforehand)
+// returns false if the value did not exist in the set
+func (s *SyncRingSet[TData]) RemoveIfContains(key TData) bool {
+	return s.Remove(key)
+}

dataext/syncRingSet_test.go

@@ -0,0 +1,84 @@
+package dataext
+
+import (
+	"sort"
+	"testing"
+)
+
+func TestSyncRingSet_AddAndContains(t *testing.T) {
+	s := NewSyncRingSet[int](3)
+	if !s.Add(1) {
+		t.Fatal("first Add(1) should be true")
+	}
+	if s.Add(1) {
+		t.Fatal("duplicate Add(1) should be false")
+	}
+	if !s.Contains(1) {
+		t.Fatal("expected Contains(1)")
+	}
+}
+
+func TestSyncRingSet_CapacityEvicts(t *testing.T) {
+	s := NewSyncRingSet[int](3)
+	s.Add(1)
+	s.Add(2)
+	s.Add(3)
+	s.Add(4) // should evict the oldest (1)
+	if s.Contains(1) {
+		t.Fatal("1 should have been evicted")
+	}
+	for _, v := range []int{2, 3, 4} {
+		if !s.Contains(v) {
+			t.Fatalf("expected %d", v)
+		}
+	}
+}
+
+func TestSyncRingSet_Remove(t *testing.T) {
+	s := NewSyncRingSet[string](3)
+	s.Add("a")
+	s.Add("b")
+	if !s.Remove("a") {
+		t.Fatal("remove existing failed")
+	}
+	if s.Remove("a") {
+		t.Fatal("remove missing returned true")
+	}
+	if s.Contains("a") {
+		t.Fatal("a should be gone")
+	}
+}
+
+func TestSyncRingSet_AddAllRemoveAll(t *testing.T) {
+	s := NewSyncRingSet[int](10)
+	s.AddAll([]int{1, 2, 3, 2})
+	out := s.Get()
+	sort.Ints(out)
+	if len(out) != 3 {
+		t.Fatalf("got %v", out)
+	}
+
+	s.RemoveAll([]int{1, 99})
+	if s.Contains(1) {
+		t.Fatal("1 should be removed")
+	}
+	if !s.Contains(2) || !s.Contains(3) {
+		t.Fatal("2/3 should remain")
+	}
+}
+
+func TestSyncRingSet_AddIfNotContainsRemoveIfContains(t *testing.T) {
+	s := NewSyncRingSet[string](5)
+	if !s.AddIfNotContains("x") {
+		t.Fatal("first should succeed")
+	}
+	if s.AddIfNotContains("x") {
+		t.Fatal("second should fail")
+	}
+	if !s.RemoveIfContains("x") {
+		t.Fatal("remove existing failed")
+	}
+	if s.RemoveIfContains("x") {
+		t.Fatal("remove missing returned true")
+	}
+}

dataext/syncSet.go

@@ -2,31 +2,41 @@ package dataext
 
 import "sync"
 
-type SyncStringSet struct {
-	data map[string]bool
+type SyncSet[TData comparable] struct {
+	data map[TData]bool
 	lock sync.Mutex
 }
 
-func (s *SyncStringSet) Add(value string) bool {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	if s.data == nil {
-		s.data = make(map[string]bool)
-	}
-
-	_, ok := s.data[value]
-	s.data[value] = true
-
-	return !ok
+func NewSyncSet[TData comparable]() *SyncSet[TData] {
+	return &SyncSet[TData]{data: make(map[TData]bool), lock: sync.Mutex{}}
 }
 
-func (s *SyncStringSet) AddAll(values []string) {
+// Add adds `value` to the set
+// returns true  if the value was actually inserted (value did not exist beforehand)
+// returns false if the value already existed
+func (s *SyncSet[TData]) Add(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
+	}
+
+	_, existsInPreState := s.data[value]
+	if existsInPreState {
+		return false
+	}
+
+	s.data[value] = true
+	return true
+}
+
+func (s *SyncSet[TData]) AddAll(values []TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
 	}
 
 	for _, value := range values {
@@ -34,12 +44,42 @@ func (s *SyncStringSet) AddAll(values []string) {
 	}
 }
 
-func (s *SyncStringSet) Contains(value string) bool {
+func (s *SyncSet[TData]) Remove(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
+	}
+
+	_, existsInPreState := s.data[value]
+	if !existsInPreState {
+		return false
+	}
+
+	delete(s.data, value)
+	return true
+}
+
+func (s *SyncSet[TData]) RemoveAll(values []TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
+	}
+
+	for _, value := range values {
+		delete(s.data, value)
+	}
+}
+
+func (s *SyncSet[TData]) Contains(value TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TData]bool)
 	}
 
 	_, ok := s.data[value]
@@ -47,15 +87,15 @@ func (s *SyncStringSet) Contains(value string) bool {
 	return ok
 }
 
-func (s *SyncStringSet) Get() []string {
+func (s *SyncSet[TData]) Get() []TData {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
-	r := make([]string, 0, len(s.data))
+	r := make([]TData, 0, len(s.data))
 
 	for k := range s.data {
 		r = append(r, k)
@@ -63,3 +103,17 @@ func (s *SyncStringSet) Get() []string {
 
 	return r
 }
+
+// AddIfNotContains
+// returns true  if the value was actually added (value did not exist beforehand)
+// returns false if the value already existed
+func (s *SyncSet[TData]) AddIfNotContains(key TData) bool {
+	return s.Add(key)
+}
+
+// RemoveIfContains
+// returns true  if the value was actually removed (value did exist beforehand)
+// returns false if the value did not exist in the set
+func (s *SyncSet[TData]) RemoveIfContains(key TData) bool {
+	return s.Remove(key)
+}

dataext/syncSet_test.go

@@ -0,0 +1,82 @@
+package dataext
+
+import (
+	"sort"
+	"testing"
+)
+
+func TestSyncSet_Add(t *testing.T) {
+	s := NewSyncSet[string]()
+	if !s.Add("a") {
+		t.Fatal("first add should be true")
+	}
+	if s.Add("a") {
+		t.Fatal("duplicate add should be false")
+	}
+	if !s.Contains("a") {
+		t.Fatal("Contains a should be true")
+	}
+}
+
+func TestSyncSet_AddAll(t *testing.T) {
+	s := NewSyncSet[int]()
+	s.AddAll([]int{1, 2, 3, 2})
+	if !s.Contains(1) || !s.Contains(2) || !s.Contains(3) {
+		t.Fatal("missing items")
+	}
+	if len(s.Get()) != 3 {
+		t.Fatalf("got len %d", len(s.Get()))
+	}
+}
+
+func TestSyncSet_Remove(t *testing.T) {
+	s := NewSyncSet[string]()
+	s.Add("a")
+	if !s.Remove("a") {
+		t.Fatal("remove existing failed")
+	}
+	if s.Remove("a") {
+		t.Fatal("remove missing returned true")
+	}
+	if s.Contains("a") {
+		t.Fatal("still contains after remove")
+	}
+}
+
+func TestSyncSet_RemoveAll(t *testing.T) {
+	s := NewSyncSet[int]()
+	s.AddAll([]int{1, 2, 3})
+	s.RemoveAll([]int{1, 2, 99})
+	if s.Contains(1) || s.Contains(2) {
+		t.Fatal("should be removed")
+	}
+	if !s.Contains(3) {
+		t.Fatal("3 should remain")
+	}
+}
+
+func TestSyncSet_Get(t *testing.T) {
+	s := NewSyncSet[int]()
+	s.AddAll([]int{3, 1, 2})
+	out := s.Get()
+	sort.Ints(out)
+	if len(out) != 3 || out[0] != 1 || out[2] != 3 {
+		t.Fatalf("out=%v", out)
+	}
+}
+
+func TestSyncSet_AddIfNotContainsRemoveIfContains(t *testing.T) {
+	s := NewSyncSet[string]()
+	if !s.AddIfNotContains("x") {
+		t.Fatal("first AddIfNotContains failed")
+	}
+	if s.AddIfNotContains("x") {
+		t.Fatal("second AddIfNotContains succeeded")
+	}
+	if !s.RemoveIfContains("x") {
+		t.Fatal("RemoveIfContains failed")
+	}
+	if s.RemoveIfContains("x") {
+		t.Fatal("RemoveIfContains on missing succeeded")
+	}
+}

dataext/tuple.go

@@ -0,0 +1,241 @@
+package dataext
+
+type ValueGroup interface {
+	TupleLength() int
+	TupleValues() []any
+}
+
+// ----------------------------------------------------------------------------
+
+type Single[T1 any] struct {
+	V1 T1
+}
+
+func (s Single[T1]) TupleLength() int {
+	return 1
+}
+
+func (s Single[T1]) TupleValues() []any {
+	return []any{s.V1}
+}
+
+func NewSingle[T1 any](v1 T1) Single[T1] {
+	return Single[T1]{V1: v1}
+}
+
+func NewTuple1[T1 any](v1 T1) Single[T1] {
+	return Single[T1]{V1: v1}
+}
+
+// ----------------------------------------------------------------------------
+
+type Tuple[T1 any, T2 any] struct {
+	V1 T1
+	V2 T2
+}
+
+func (t Tuple[T1, T2]) TupleLength() int {
+	return 2
+}
+
+func (t Tuple[T1, T2]) TupleValues() []any {
+	return []any{t.V1, t.V2}
+}
+
+func NewTuple[T1 any, T2 any](v1 T1, v2 T2) Tuple[T1, T2] {
+	return Tuple[T1, T2]{V1: v1, V2: v2}
+}
+
+func NewTuple2[T1 any, T2 any](v1 T1, v2 T2) Tuple[T1, T2] {
+	return Tuple[T1, T2]{V1: v1, V2: v2}
+}
+
+// ----------------------------------------------------------------------------
+
+type Triple[T1 any, T2 any, T3 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+}
+
+func (t Triple[T1, T2, T3]) TupleLength() int {
+	return 3
+}
+
+func (t Triple[T1, T2, T3]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3}
+}
+
+func NewTriple[T1 any, T2 any, T3 any](v1 T1, v2 T2, v3 T3) Triple[T1, T2, T3] {
+	return Triple[T1, T2, T3]{V1: v1, V2: v2, V3: v3}
+}
+
+func NewTuple3[T1 any, T2 any, T3 any](v1 T1, v2 T2, v3 T3) Triple[T1, T2, T3] {
+	return Triple[T1, T2, T3]{V1: v1, V2: v2, V3: v3}
+}
+
+// ----------------------------------------------------------------------------
+
+type Quadruple[T1 any, T2 any, T3 any, T4 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+}
+
+func (t Quadruple[T1, T2, T3, T4]) TupleLength() int {
+	return 4
+}
+
+func (t Quadruple[T1, T2, T3, T4]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4}
+}
+
+func NewQuadruple[T1 any, T2 any, T3 any, T4 any](v1 T1, v2 T2, v3 T3, v4 T4) Quadruple[T1, T2, T3, T4] {
+	return Quadruple[T1, T2, T3, T4]{V1: v1, V2: v2, V3: v3, V4: v4}
+}
+
+func NewTuple4[T1 any, T2 any, T3 any, T4 any](v1 T1, v2 T2, v3 T3, v4 T4) Quadruple[T1, T2, T3, T4] {
+	return Quadruple[T1, T2, T3, T4]{V1: v1, V2: v2, V3: v3, V4: v4}
+}
+
+// ----------------------------------------------------------------------------
+
+type Quintuple[T1 any, T2 any, T3 any, T4 any, T5 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+}
+
+func (t Quintuple[T1, T2, T3, T4, T5]) TupleLength() int {
+	return 5
+}
+
+func (t Quintuple[T1, T2, T3, T4, T5]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5}
+
+}
+
+func NewQuintuple[T1 any, T2 any, T3 any, T4 any, T5 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5) Quintuple[T1, T2, T3, T4, T5] {
+	return Quintuple[T1, T2, T3, T4, T5]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5}
+}
+
+func NewTuple5[T1 any, T2 any, T3 any, T4 any, T5 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5) Quintuple[T1, T2, T3, T4, T5] {
+	return Quintuple[T1, T2, T3, T4, T5]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5}
+}
+
+// ----------------------------------------------------------------------------
+
+type Sextuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+}
+
+func (t Sextuple[T1, T2, T3, T4, T5, T6]) TupleLength() int {
+	return 6
+}
+
+func (t Sextuple[T1, T2, T3, T4, T5, T6]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6}
+
+}
+
+func NewSextuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6) Sextuple[T1, T2, T3, T4, T5, T6] {
+	return Sextuple[T1, T2, T3, T4, T5, T6]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6}
+}
+
+func NewTuple6[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6) Sextuple[T1, T2, T3, T4, T5, T6] {
+	return Sextuple[T1, T2, T3, T4, T5, T6]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6}
+}
+
+// ----------------------------------------------------------------------------
+
+type Septuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+	V7 T7
+}
+
+func (t Septuple[T1, T2, T3, T4, T5, T6, T7]) TupleLength() int {
+	return 7
+}
+
+func (t Septuple[T1, T2, T3, T4, T5, T6, T7]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6, t.V7}
+}
+
+func NewSeptuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6, v7 T7) Septuple[T1, T2, T3, T4, T5, T6, T7] {
+	return Septuple[T1, T2, T3, T4, T5, T6, T7]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6, V7: v7}
+}
+
+func NewTuple7[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6, v7 T7) Septuple[T1, T2, T3, T4, T5, T6, T7] {
+	return Septuple[T1, T2, T3, T4, T5, T6, T7]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6, V7: v7}
+}
+
+// ----------------------------------------------------------------------------
+
+type Octuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+	V7 T7
+	V8 T8
+}
+
+func (t Octuple[T1, T2, T3, T4, T5, T6, T7, T8]) TupleLength() int {
+	return 8
+}
+
+func (t Octuple[T1, T2, T3, T4, T5, T6, T7, T8]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6, t.V7, t.V8}
+}
+
+func NewOctuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6, v7 T7, v8 T8) Octuple[T1, T2, T3, T4, T5, T6, T7, T8] {
+	return Octuple[T1, T2, T3, T4, T5, T6, T7, T8]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6, V7: v7, V8: v8}
+}
+
+func NewTuple8[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6, v7 T7, v8 T8) Octuple[T1, T2, T3, T4, T5, T6, T7, T8] {
+	return Octuple[T1, T2, T3, T4, T5, T6, T7, T8]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6, V7: v7, V8: v8}
+}
+
+// ----------------------------------------------------------------------------
+
+type Nonuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any, T9 any] struct {
+	V1 T1
+	V2 T2
+	V3 T3
+	V4 T4
+	V5 T5
+	V6 T6
+	V7 T7
+	V8 T8
+	V9 T9
+}
+
+func (t Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9]) TupleLength() int {
+	return 9
+}
+
+func (t Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9]) TupleValues() []any {
+	return []any{t.V1, t.V2, t.V3, t.V4, t.V5, t.V6, t.V7, t.V8, t.V9}
+}
+
+func NewNonuple[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any, T9 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6, v7 T7, v8 T8, v9 T9) Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9] {
+	return Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6, V7: v7, V8: v8, V9: v9}
+}
+func NewTuple9[T1 any, T2 any, T3 any, T4 any, T5 any, T6 any, T7 any, T8 any, T9 any](v1 T1, v2 T2, v3 T3, v4 T4, v5 T5, v6 T6, v7 T7, v8 T8, v9 T9) Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9] {
+	return Nonuple[T1, T2, T3, T4, T5, T6, T7, T8, T9]{V1: v1, V2: v2, V3: v3, V4: v4, V5: v5, V6: v6, V7: v7, V8: v8, V9: v9}
+}

dataext/tuple_test.go

@@ -0,0 +1,136 @@
+package dataext
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSingle(t *testing.T) {
+	s := NewSingle[int](7)
+	if s.V1 != 7 {
+		t.Fatalf("V1=%d", s.V1)
+	}
+	if s.TupleLength() != 1 {
+		t.Fatalf("len=%d", s.TupleLength())
+	}
+	if !reflect.DeepEqual(s.TupleValues(), []any{7}) {
+		t.Fatalf("values=%v", s.TupleValues())
+	}
+	if NewTuple1[int](7).V1 != 7 {
+		t.Fatal("NewTuple1 mismatch")
+	}
+}
+
+func TestTuple(t *testing.T) {
+	tp := NewTuple[int, string](1, "two")
+	if tp.V1 != 1 || tp.V2 != "two" {
+		t.Fatal("values wrong")
+	}
+	if tp.TupleLength() != 2 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(tp.TupleValues(), []any{1, "two"}) {
+		t.Fatalf("values=%v", tp.TupleValues())
+	}
+	if NewTuple2[int, string](1, "two") != tp {
+		t.Fatal("NewTuple2 mismatch")
+	}
+}
+
+func TestTriple(t *testing.T) {
+	tr := NewTriple[int, string, bool](1, "x", true)
+	if tr.TupleLength() != 3 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(tr.TupleValues(), []any{1, "x", true}) {
+		t.Fatalf("values=%v", tr.TupleValues())
+	}
+	if NewTuple3[int, string, bool](1, "x", true) != tr {
+		t.Fatal("NewTuple3 mismatch")
+	}
+}
+
+func TestQuadruple(t *testing.T) {
+	q := NewQuadruple[int, int, int, int](1, 2, 3, 4)
+	if q.TupleLength() != 4 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(q.TupleValues(), []any{1, 2, 3, 4}) {
+		t.Fatalf("values=%v", q.TupleValues())
+	}
+	if NewTuple4[int, int, int, int](1, 2, 3, 4) != q {
+		t.Fatal("NewTuple4 mismatch")
+	}
+}
+
+func TestQuintuple(t *testing.T) {
+	q := NewQuintuple[int, int, int, int, int](1, 2, 3, 4, 5)
+	if q.TupleLength() != 5 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(q.TupleValues(), []any{1, 2, 3, 4, 5}) {
+		t.Fatalf("values=%v", q.TupleValues())
+	}
+	if NewTuple5[int, int, int, int, int](1, 2, 3, 4, 5) != q {
+		t.Fatal("NewTuple5 mismatch")
+	}
+}
+
+func TestSextuple(t *testing.T) {
+	s := NewSextuple[int, int, int, int, int, int](1, 2, 3, 4, 5, 6)
+	if s.TupleLength() != 6 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(s.TupleValues(), []any{1, 2, 3, 4, 5, 6}) {
+		t.Fatalf("values=%v", s.TupleValues())
+	}
+	if NewTuple6[int, int, int, int, int, int](1, 2, 3, 4, 5, 6) != s {
+		t.Fatal("NewTuple6 mismatch")
+	}
+}
+
+func TestSeptuple(t *testing.T) {
+	s := NewSeptuple[int, int, int, int, int, int, int](1, 2, 3, 4, 5, 6, 7)
+	if s.TupleLength() != 7 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(s.TupleValues(), []any{1, 2, 3, 4, 5, 6, 7}) {
+		t.Fatalf("values=%v", s.TupleValues())
+	}
+	if NewTuple7[int, int, int, int, int, int, int](1, 2, 3, 4, 5, 6, 7) != s {
+		t.Fatal("NewTuple7 mismatch")
+	}
+}
+
+func TestOctuple(t *testing.T) {
+	o := NewOctuple[int, int, int, int, int, int, int, int](1, 2, 3, 4, 5, 6, 7, 8)
+	if o.TupleLength() != 8 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(o.TupleValues(), []any{1, 2, 3, 4, 5, 6, 7, 8}) {
+		t.Fatalf("values=%v", o.TupleValues())
+	}
+	if NewTuple8[int, int, int, int, int, int, int, int](1, 2, 3, 4, 5, 6, 7, 8) != o {
+		t.Fatal("NewTuple8 mismatch")
+	}
+}
+
+func TestNonuple(t *testing.T) {
+	n := NewNonuple[int, int, int, int, int, int, int, int, int](1, 2, 3, 4, 5, 6, 7, 8, 9)
+	if n.TupleLength() != 9 {
+		t.Fatal("len wrong")
+	}
+	if !reflect.DeepEqual(n.TupleValues(), []any{1, 2, 3, 4, 5, 6, 7, 8, 9}) {
+		t.Fatalf("values=%v", n.TupleValues())
+	}
+	if NewTuple9[int, int, int, int, int, int, int, int, int](1, 2, 3, 4, 5, 6, 7, 8, 9) != n {
+		t.Fatal("NewTuple9 mismatch")
+	}
+}
+
+func TestValueGroupInterface(t *testing.T) {
+	var vg ValueGroup = NewTuple[int, string](1, "a")
+	if vg.TupleLength() != 2 {
+		t.Fatal("interface length wrong")
+	}
+}