v0.0.93

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +1,7 @@
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="GoMixedReceiverTypes" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
     <inspection_tool class="LanguageDetectionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
     <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
       <option name="processCode" value="true" />

_data/version.sh

@@ -7,6 +7,20 @@ set -o pipefail  # Return value of a pipeline is the value of the last (rightmos
 IFS=$'\n\t'      # Set $IFS to only newline and tab.
 
 
+function black() { echo -e "\x1B[30m $1 \x1B[0m"; }
+function red()   { echo -e "\x1B[31m $1 \x1B[0m"; }
+function green() { echo -e "\x1B[32m $1 \x1B[0m"; }
+function yellow(){ echo -e "\x1B[33m $1 \x1B[0m"; }
+function blue()  { echo -e "\x1B[34m $1 \x1B[0m"; }
+function purple(){ echo -e "\x1B[35m $1 \x1B[0m"; }
+function cyan()  { echo -e "\x1B[36m $1 \x1B[0m"; }
+function white() { echo -e "\x1B[37m $1 \x1B[0m"; }
+
+if [ "$( git rev-parse --abbrev-ref HEAD )" != "master" ]; then
+  >&2 red "[ERROR] Can only create versions of <master>"
+  exit 1
+fi
+
 curr_vers=$(git describe --tags --abbrev=0 | sed 's/v//g')
 
 next_ver=$(echo "$curr_vers" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{if(length($NF+1)>length($NF))$(NF-1)++; $NF=sprintf("%0*d", length($NF), ($NF+1)%(10^length($NF))); print}')
@@ -18,7 +32,13 @@ echo ""
 
 git add --verbose .
 
-git commit -a -m "v${next_ver}"
+msg="v${next_ver}"
+
+if [ $# -gt 0 ]; then
+  msg="$1"
+fi
+
+git commit -a -m "${msg}"
 
 git tag "v${next_ver}"
 

cmdext/builder.go

@@ -0,0 +1,93 @@
+package cmdext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"time"
+)
+
+type CommandRunner struct {
+	program          string
+	args             []string
+	timeout          *time.Duration
+	env              []string
+	listener         []CommandListener
+	enforceExitCodes *[]int
+	enforceNoTimeout bool
+}
+
+func Runner(program string) *CommandRunner {
+	return &CommandRunner{
+		program:          program,
+		args:             make([]string, 0),
+		timeout:          nil,
+		env:              make([]string, 0),
+		listener:         make([]CommandListener, 0),
+		enforceExitCodes: nil,
+		enforceNoTimeout: false,
+	}
+}
+
+func (r *CommandRunner) Arg(arg string) *CommandRunner {
+	r.args = append(r.args, arg)
+	return r
+}
+
+func (r *CommandRunner) Args(arg []string) *CommandRunner {
+	r.args = append(r.args, arg...)
+	return r
+}
+
+func (r *CommandRunner) Timeout(timeout time.Duration) *CommandRunner {
+	r.timeout = &timeout
+	return r
+}
+
+func (r *CommandRunner) Env(key, value string) *CommandRunner {
+	r.env = append(r.env, fmt.Sprintf("%s=%s", key, value))
+	return r
+}
+
+func (r *CommandRunner) RawEnv(env string) *CommandRunner {
+	r.env = append(r.env, env)
+	return r
+}
+
+func (r *CommandRunner) Envs(env []string) *CommandRunner {
+	r.env = append(r.env, env...)
+	return r
+}
+
+func (r *CommandRunner) EnsureExitcode(arg ...int) *CommandRunner {
+	r.enforceExitCodes = langext.Ptr(langext.ForceArray(arg))
+	return r
+}
+
+func (r *CommandRunner) FailOnExitCode() *CommandRunner {
+	r.enforceExitCodes = langext.Ptr([]int{0})
+	return r
+}
+
+func (r *CommandRunner) FailOnTimeout() *CommandRunner {
+	r.enforceNoTimeout = true
+	return r
+}
+
+func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
+	r.listener = append(r.listener, lstr)
+	return r
+}
+
+func (r *CommandRunner) ListenStdout(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStdoutLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) ListenStderr(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStderrLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) Run() (CommandResult, error) {
+	return run(*r)
+}

cmdext/cmdrunner.go

@@ -0,0 +1,154 @@
+package cmdext
+
+import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
+	"os/exec"
+	"time"
+)
+
+var ErrExitCode = errors.New("process exited with an unexpected exitcode")
+var ErrTimeout = errors.New("process did not exit after the specified timeout")
+
+type CommandResult struct {
+	StdOut          string
+	StdErr          string
+	StdCombined     string
+	ExitCode        int
+	CommandTimedOut bool
+}
+
+func run(opt CommandRunner) (CommandResult, error) {
+	cmd := exec.Command(opt.program, opt.args...)
+
+	cmd.Env = append(cmd.Env, opt.env...)
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	preader := pipeReader{
+		lineBufferSize: langext.Ptr(128 * 1024 * 1024), // 128MB max size of a single line, is hopefully enough....
+		stdout:         stdoutPipe,
+		stderr:         stderrPipe,
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return CommandResult{}, err
+	}
+
+	type resultObj struct {
+		stdout      string
+		stderr      string
+		stdcombined string
+		err         error
+	}
+
+	outputChan := make(chan resultObj)
+	go func() {
+		// we need to first fully read the pipes and then call Wait
+		// see https://pkg.go.dev/os/exec#Cmd.StdoutPipe
+
+		stdout, stderr, stdcombined, err := preader.Read(opt.listener)
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+			_ = cmd.Process.Kill()
+			return
+		}
+
+		err = cmd.Wait()
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+		} else {
+			outputChan <- resultObj{stdout, stderr, stdcombined, nil}
+		}
+
+	}()
+
+	var timeoutChan <-chan time.Time = make(chan time.Time, 1)
+	if opt.timeout != nil {
+		timeoutChan = time.After(*opt.timeout)
+	}
+
+	select {
+
+	case <-timeoutChan:
+		_ = cmd.Process.Kill()
+		for _, lstr := range opt.listener {
+			lstr.Timeout()
+		}
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
+			// most of the time the cmd.Process.Kill() should also ahve finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		}
+
+	case outobj := <-outputChan:
+		if exiterr, ok := outobj.err.(*exec.ExitError); ok {
+			excode := exiterr.ExitCode()
+			for _, lstr := range opt.listener {
+				lstr.Finished(excode)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        excode,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(excode, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		} else if err != nil {
+			return CommandResult{}, err
+		} else {
+			for _, lstr := range opt.listener {
+				lstr.Finished(0)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        0,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(0, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		}
+	}
+}

cmdext/cmdrunner_test.go

@@ -0,0 +1,323 @@
+package cmdext
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestStdout(t *testing.T) {
+
+	res1, err := Runner("printf").Arg("hello").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "hello" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "hello\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStderr(t *testing.T) {
+
+	res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStdcombined(t *testing.T) {
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"1\", file=sys.stderr, flush=True); time.sleep(0.1); print(\"2\", file=sys.stdout, flush=True); time.sleep(0.1); print(\"3\", file=sys.stderr, flush=True)").
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "1\n3\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "2\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "1\n2\n3\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialRead(t *testing.T) {
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", flush=True); time.sleep(5); print(\"cant see me\", flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadStderr(t *testing.T) {
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, flush=True); time.sleep(5); print(\"cant see me\", file=sys.stderr, flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStdout(t *testing.T) {
+
+	res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stdout, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "message101" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStderr(t *testing.T) {
+
+	res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "message101" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushed(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", end=''); time.sleep(5); print(\"cant see me\", end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushedStderr(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, end=''); time.sleep(5); print(\"cant see me\", file=sys.stderr, end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestListener(t *testing.T) {
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys;" +
+			"import time;" +
+			"print(\"message 1\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 2\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 3\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 4\", file=sys.stderr, flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 5\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"final\");").
+		ListenStdout(func(s string) { fmt.Printf("@@STDOUT <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		ListenStderr(func(s string) { fmt.Printf("@@STDERR <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		Timeout(10 * time.Second).
+		Run()
+
+	if err != nil {
+		panic(err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+}
+
+func TestLongStdout(t *testing.T) {
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"X\" * 125001 + \"\\n\"); print(\"Y\" * 125001 + \"\\n\"); print(\"Z\" * 125001 + \"\\n\");").
+		Timeout(5000 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if len(res1.StdOut) != 375009 {
+		t.Errorf("len(res1.StdOut) == '%v'", len(res1.StdOut))
+	}
+
+}
+
+func TestFailOnTimeout(t *testing.T) {
+
+	_, err := Runner("sleep").Arg("2").Timeout(200 * time.Millisecond).FailOnTimeout().Run()
+	if err != ErrTimeout {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestFailOnExitcode(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).FailOnExitCode().Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode1(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).EnsureExitcode(1).Run()
+	if err != nil {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode2(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200*time.Millisecond).EnsureExitcode(0, 2, 3).Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}

cmdext/helper.go

@@ -0,0 +1,12 @@
+package cmdext
+
+import "time"
+
+func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+	b := Runner(program)
+	b = b.Args(args)
+	if timeout != nil {
+		b = b.Timeout(*timeout)
+	}
+	return b.Run()
+}

cmdext/listener.go

@@ -0,0 +1,57 @@
+package cmdext
+
+type CommandListener interface {
+	ReadRawStdout([]byte)
+	ReadRawStderr([]byte)
+
+	ReadStdoutLine(string)
+	ReadStderrLine(string)
+
+	Finished(int)
+	Timeout()
+}
+
+type genericCommandListener struct {
+	_readRawStdout  *func([]byte)
+	_readRawStderr  *func([]byte)
+	_readStdoutLine *func(string)
+	_readStderrLine *func(string)
+	_finished       *func(int)
+	_timeout        *func()
+}
+
+func (g genericCommandListener) ReadRawStdout(v []byte) {
+	if g._readRawStdout != nil {
+		(*g._readRawStdout)(v)
+	}
+}
+
+func (g genericCommandListener) ReadRawStderr(v []byte) {
+	if g._readRawStderr != nil {
+		(*g._readRawStderr)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStdoutLine(v string) {
+	if g._readStdoutLine != nil {
+		(*g._readStdoutLine)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStderrLine(v string) {
+	if g._readStderrLine != nil {
+		(*g._readStderrLine)(v)
+	}
+}
+
+func (g genericCommandListener) Finished(v int) {
+	if g._finished != nil {
+		(*g._finished)(v)
+	}
+}
+
+func (g genericCommandListener) Timeout() {
+	if g._timeout != nil {
+		(*g._timeout)()
+	}
+}

cmdext/pipereader.go

@@ -0,0 +1,158 @@
+package cmdext
+
+import (
+	"bufio"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
+	"io"
+	"sync"
+)
+
+type pipeReader struct {
+	lineBufferSize *int
+	stdout         io.ReadCloser
+	stderr         io.ReadCloser
+}
+
+// Read ready stdout and stdin until finished
+// also splits both pipes into lines and calld the listener
+func (pr *pipeReader) Read(listener []CommandListener) (string, string, string, error) {
+	type combevt struct {
+		line string
+		stop bool
+	}
+
+	errch := make(chan error, 8)
+
+	wg := sync.WaitGroup{}
+
+	// [1] read raw stdout
+
+	wg.Add(1)
+	stdoutBufferReader, stdoutBufferWriter := io.Pipe()
+	stdout := ""
+	go func() {
+		buf := make([]byte, 128)
+		for true {
+			n, out := pr.stdout.Read(buf)
+			if n > 0 {
+				txt := string(buf[:n])
+				stdout += txt
+				_, _ = stdoutBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStdout(buf[:n])
+				}
+			}
+			if out == io.EOF {
+				break
+			}
+			if out != nil {
+				errch <- out
+				break
+			}
+		}
+		_ = stdoutBufferWriter.Close()
+		wg.Done()
+	}()
+
+	// [2] read raw stderr
+
+	wg.Add(1)
+	stderrBufferReader, stderrBufferWriter := io.Pipe()
+	stderr := ""
+	go func() {
+		buf := make([]byte, 128)
+		for true {
+			n, err := pr.stderr.Read(buf)
+
+			if n > 0 {
+				txt := string(buf[:n])
+				stderr += txt
+				_, _ = stderrBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStderr(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stderrBufferWriter.Close()
+		wg.Done()
+	}()
+
+	combch := make(chan combevt, 32)
+
+	// [3] collect stdout line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stdoutBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStdoutLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [4] collect stderr line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stderrBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStderrLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [5] combine stdcombined
+
+	wg.Add(1)
+	stdcombined := ""
+	go func() {
+		stopctr := 0
+		for stopctr < 2 {
+			vvv := <-combch
+			if vvv.stop {
+				stopctr++
+			} else {
+				stdcombined += vvv.line + "\n" // this comes from bufio.Scanner and has no newlines...
+			}
+		}
+		wg.Done()
+	}()
+
+	// wait for all (5) goroutines to finish
+	wg.Wait()
+
+	if err, ok := syncext.ReadNonBlocking(errch); ok {
+		return "", "", "", err
+	}
+
+	return stdout, stderr, stdcombined, nil
+}

confext/confParser.go

@@ -14,9 +14,21 @@ import (
 // ApplyEnvOverrides overrides field values from environment variables
 //
 // fields must be tagged with `env:"env_key"`
-func ApplyEnvOverrides[T any](c *T) error {
+//
-
+// only works on exported fields
+//
+// fields without an env tag are ignored
+// fields with an `env:"-"` tag are ignore
+//
+// sub-structs are recursively parsed (if they have an env tag) and the env-variable keys are delimited by the delim parameter
+// sub-structs with `env:""` are also parsed, but the delimited is skipped (they are handled as if they were one level higher)
+func ApplyEnvOverrides[T any](prefix string, c *T, delim string) error {
 	rval := reflect.ValueOf(c).Elem()
+
+	return processEnvOverrides(rval, delim, prefix)
+}
+
+func processEnvOverrides(rval reflect.Value, delim string, prefix string) error {
 	rtyp := rval.Type()
 
 	for i := 0; i < rtyp.NumField(); i++ {
@@ -24,113 +36,148 @@ func ApplyEnvOverrides[T any](c *T) error {
 		rsfield := rtyp.Field(i)
 		rvfield := rval.Field(i)
 
-		envkey := rsfield.Tag.Get("env")
+		if !rsfield.IsExported() {
-		if envkey == "" {
 			continue
 		}
 
-		envval, efound := os.LookupEnv(envkey)
+		if rvfield.Kind() == reflect.Struct {
+
+			envkey, found := rsfield.Tag.Lookup("env")
+			if !found || envkey == "-" {
+				continue
+			}
+
+			subPrefix := prefix
+			if envkey != "" {
+				subPrefix = subPrefix + envkey + delim
+			}
+
+			err := processEnvOverrides(rvfield, delim, subPrefix)
+			if err != nil {
+				return err
+			}
+		}
+
+		envkey := rsfield.Tag.Get("env")
+		if envkey == "" || envkey == "-" {
+			continue
+		}
+
+		fullEnvKey := prefix + envkey
+
+		envval, efound := os.LookupEnv(fullEnvKey)
 		if !efound {
 			continue
 		}
 
-		if rvfield.Type() == reflect.TypeOf("") {
+		if rvfield.Type().Kind() == reflect.Pointer {
 
-			rvfield.Set(reflect.ValueOf(envval))
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type().Elem())
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
 			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", envkey, envval))
+				return err
 			}
 
-			rvfield.Set(reflect.ValueOf(int(envint)))
+			// converts reflect.Value to pointer
+			ptrval := reflect.New(rvfield.Type().Elem())
+			ptrval.Elem().Set(newval)
 
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
+			rvfield.Set(ptrval)
 
-		} else if rvfield.Type() == reflect.TypeOf(int64(0)) {
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
-
-			envint, err := strconv.ParseInt(envval, 10, 64)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int64(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int32(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 32)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int32(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(int8(0)) {
-
-			envint, err := strconv.ParseInt(envval, 10, 8)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(int8(envint)))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, envval)
-
-		} else if rvfield.Type() == reflect.TypeOf(time.Duration(0)) {
-
-			dur, err := timeext.ParseDurationShortString(envval)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(dur))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, dur.String())
-
-		} else if rvfield.Type() == reflect.TypeOf(time.UnixMilli(0)) {
-
-			tim, err := time.Parse(time.RFC3339Nano, envval)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", envkey, envval))
-			}
-
-			rvfield.Set(reflect.ValueOf(tim))
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", envkey, tim.String())
-
-		} else if rvfield.Type().ConvertibleTo(reflect.TypeOf(int(0))) {
-
-			envint, err := strconv.ParseInt(envval, 10, 8)
-			if err != nil {
-				return errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvfield.Type().Name(), envkey, envval))
-			}
-
-			envcvl := reflect.ValueOf(envint).Convert(rvfield.Type())
-
-			rvfield.Set(envcvl)
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%v'\n", envkey, envcvl.Interface())
-
-		} else if rvfield.Type().ConvertibleTo(reflect.TypeOf("")) {
-
-			envcvl := reflect.ValueOf(envval).Convert(rvfield.Type())
-
-			rvfield.Set(envcvl)
-
-			fmt.Printf("[CONF] Overwrite config '%s' with '%v'\n", envkey, envcvl.Interface())
 
 		} else {
-			return errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvfield.Kind().String(), rvfield.Type().String()))
+
+			newval, err := parseEnvToValue(envval, fullEnvKey, rvfield.Type())
+			if err != nil {
+				return err
 			}
+
+			rvfield.Set(newval)
+			fmt.Printf("[CONF] Overwrite config '%s' with '%s'\n", fullEnvKey, envval)
+
+		}
+
 	}
 
 	return nil
 }
+
+func parseEnvToValue(envval string, fullEnvKey string, rvtype reflect.Type) (reflect.Value, error) {
+	if rvtype == reflect.TypeOf("") {
+
+		return reflect.ValueOf(envval), nil
+
+	} else if rvtype == reflect.TypeOf(int(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, bits.UintSize)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int64(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 64)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int64 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int64(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int32(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 32)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int32(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(int8(0)) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to int32 (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(int8(envint)), nil
+
+	} else if rvtype == reflect.TypeOf(time.Duration(0)) {
+
+		dur, err := timeext.ParseDurationShortString(envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to duration (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(dur), nil
+
+	} else if rvtype == reflect.TypeOf(time.UnixMilli(0)) {
+
+		tim, err := time.Parse(time.RFC3339Nano, envval)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to time.time (value := '%s')", fullEnvKey, envval))
+		}
+
+		return reflect.ValueOf(tim), nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf(int(0))) {
+
+		envint, err := strconv.ParseInt(envval, 10, 8)
+		if err != nil {
+			return reflect.Value{}, errors.New(fmt.Sprintf("Failed to parse env-config variable '%s' to <%s, ,int> (value := '%s')", rvtype.Name(), fullEnvKey, envval))
+		}
+
+		envcvl := reflect.ValueOf(envint).Convert(rvtype)
+
+		return envcvl, nil
+
+	} else if rvtype.ConvertibleTo(reflect.TypeOf("")) {
+
+		envcvl := reflect.ValueOf(envval).Convert(rvtype)
+		return envcvl, nil
+
+	} else {
+		return reflect.Value{}, errors.New(fmt.Sprintf("Unknown kind/type in config: [ %s | %s ]", rvtype.Kind().String(), rvtype.String()))
+	}
+}

confext/confParser_test.go

@@ -1,6 +1,7 @@
 package confext
 
 import (
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
 	"testing"
 	"time"
 )
@@ -40,7 +41,7 @@ func TestApplyEnvOverridesNoop(t *testing.T) {
 
 	output := input
 
-	err := ApplyEnvOverrides(&output)
+	err := ApplyEnvOverrides(&output, ".")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
@@ -92,7 +93,7 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 	t.Setenv("TEST_V8", "1min4s")
 	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
 
-	err := ApplyEnvOverrides(&data)
+	err := ApplyEnvOverrides(&data, ".")
 	if err != nil {
 		t.Errorf("%v", err)
 		t.FailNow()
@@ -109,8 +110,168 @@ func TestApplyEnvOverridesSimple(t *testing.T) {
 	assertEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
 }
 
+func TestApplyEnvOverridesRecursive(t *testing.T) {
+
+	type subdata struct {
+		V1 int           `env:"SUB_V1"`
+		VX string        ``
+		V2 string        `env:"SUB_V2"`
+		V8 time.Duration `env:"SUB_V3"`
+		V9 time.Time     `env:"SUB_V4"`
+	}
+
+	type testdata struct {
+		V1   int     `env:"TEST_V1"`
+		VX   string  ``
+		Sub1 subdata ``
+		Sub2 subdata `env:"TEST_V2"`
+		Sub3 subdata `env:"TEST_V3"`
+		Sub4 subdata `env:""`
+		V5   string  `env:"-"`
+	}
+
+	data := testdata{
+		V1: 1,
+		VX: "2",
+		V5: "no",
+		Sub1: subdata{
+			V1: 3,
+			VX: "4",
+			V2: "5",
+			V8: 6 * time.Second,
+			V9: time.Date(2000, 1, 7, 1, 1, 1, 0, time.UTC),
+		},
+		Sub2: subdata{
+			V1: 8,
+			VX: "9",
+			V2: "10",
+			V8: 11 * time.Second,
+			V9: time.Date(2000, 1, 12, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+		Sub3: subdata{
+			V1: 13,
+			VX: "14",
+			V2: "15",
+			V8: 16 * time.Second,
+			V9: time.Date(2000, 1, 17, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+		Sub4: subdata{
+			V1: 18,
+			VX: "19",
+			V2: "20",
+			V8: 21 * time.Second,
+			V9: time.Date(2000, 1, 22, 1, 1, 1, 0, timeext.TimezoneBerlin),
+		},
+	}
+
+	t.Setenv("TEST_V1", "999")
+	t.Setenv("-", "yes")
+
+	t.Setenv("TEST_V2_SUB_V1", "846")
+	t.Setenv("TEST_V2_SUB_V2", "222_hello_world")
+	t.Setenv("TEST_V2_SUB_V3", "1min4s")
+	t.Setenv("TEST_V2_SUB_V4", "2009-11-10T23:00:00Z")
+
+	t.Setenv("TEST_V3_SUB_V1", "33846")
+	t.Setenv("TEST_V3_SUB_V2", "33_hello_world")
+	t.Setenv("TEST_V3_SUB_V3", "33min4s")
+	t.Setenv("TEST_V3_SUB_V4", "2033-11-10T23:00:00Z")
+
+	t.Setenv("SUB_V1", "11")
+	t.Setenv("SUB_V2", "22")
+	t.Setenv("SUB_V3", "33min")
+	t.Setenv("SUB_V4", "2044-01-01T00:00:00Z")
+
+	err := ApplyEnvOverrides(&data, "_")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	assertEqual(t, data.V1, 999)
+	assertEqual(t, data.VX, "2")
+	assertEqual(t, data.V5, "no")
+	assertEqual(t, data.Sub1.V1, 3)
+	assertEqual(t, data.Sub1.VX, "4")
+	assertEqual(t, data.Sub1.V2, "5")
+	assertEqual(t, data.Sub1.V8, time.Second*6)
+	assertEqual(t, data.Sub1.V9, time.Unix(947206861, 0).UTC())
+	assertEqual(t, data.Sub2.V1, 846)
+	assertEqual(t, data.Sub2.VX, "9")
+	assertEqual(t, data.Sub2.V2, "222_hello_world")
+	assertEqual(t, data.Sub2.V8, time.Second*64)
+	assertEqual(t, data.Sub2.V9, time.Unix(1257894000, 0).UTC())
+	assertEqual(t, data.Sub3.V1, 33846)
+	assertEqual(t, data.Sub3.VX, "14")
+	assertEqual(t, data.Sub3.V2, "33_hello_world")
+	assertEqual(t, data.Sub3.V8, time.Second*1984)
+	assertEqual(t, data.Sub3.V9, time.Unix(2015276400, 0).UTC())
+	assertEqual(t, data.Sub4.V1, 11)
+	assertEqual(t, data.Sub4.VX, "19")
+	assertEqual(t, data.Sub4.V2, "22")
+	assertEqual(t, data.Sub4.V8, time.Second*1980)
+	assertEqual(t, data.Sub4.V9, time.Unix(2335219200, 0).UTC())
+}
+
+func TestApplyEnvOverridesPointer(t *testing.T) {
+
+	type aliasint int
+	type aliasstring string
+
+	type testdata struct {
+		V1 *int           `env:"TEST_V1"`
+		VX *string        ``
+		V2 *string        `env:"TEST_V2"`
+		V3 *int8          `env:"TEST_V3"`
+		V4 *int32         `env:"TEST_V4"`
+		V5 *int64         `env:"TEST_V5"`
+		V6 *aliasint      `env:"TEST_V6"`
+		VY *aliasint      ``
+		V7 *aliasstring   `env:"TEST_V7"`
+		V8 *time.Duration `env:"TEST_V8"`
+		V9 *time.Time     `env:"TEST_V9"`
+	}
+
+	data := testdata{}
+
+	t.Setenv("TEST_V1", "846")
+	t.Setenv("TEST_V2", "hello_world")
+	t.Setenv("TEST_V3", "6")
+	t.Setenv("TEST_V4", "333")
+	t.Setenv("TEST_V5", "-937")
+	t.Setenv("TEST_V6", "070")
+	t.Setenv("TEST_V7", "AAAAAA")
+	t.Setenv("TEST_V8", "1min4s")
+	t.Setenv("TEST_V9", "2009-11-10T23:00:00Z")
+
+	err := ApplyEnvOverrides(&data, ".")
+	if err != nil {
+		t.Errorf("%v", err)
+		t.FailNow()
+	}
+
+	assertPtrEqual(t, data.V1, 846)
+	assertPtrEqual(t, data.V2, "hello_world")
+	assertPtrEqual(t, data.V3, 6)
+	assertPtrEqual(t, data.V4, 333)
+	assertPtrEqual(t, data.V5, -937)
+	assertPtrEqual(t, data.V6, 70)
+	assertPtrEqual(t, data.V7, "AAAAAA")
+	assertPtrEqual(t, data.V8, time.Second*64)
+	assertPtrEqual(t, data.V9, time.Unix(1257894000, 0).UTC())
+}
+
 func assertEqual[T comparable](t *testing.T, actual T, expected T) {
 	if actual != expected {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
+
+func assertPtrEqual[T comparable](t *testing.T, actual *T, expected T) {
+	if actual == nil {
+		t.Errorf("values differ: Actual: NIL, Expected: '%v'", expected)
+	}
+	if *actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

cryptext/aes.go

@@ -0,0 +1,132 @@
+package cryptext
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"golang.org/x/crypto/scrypt"
+	"io"
+)
+
+// https://stackoverflow.com/a/18819040/1761622
+
+type aesPayload struct {
+	Salt    []byte `json:"s"`
+	IV      []byte `json:"i"`
+	Data    []byte `json:"d"`
+	Rounds  int    `json:"r"`
+	Version uint   `json:"v"`
+}
+
+func EncryptAESSimple(password []byte, data []byte, rounds int) (string, error) {
+
+	salt := make([]byte, 8)
+	_, err := io.ReadFull(rand.Reader, salt)
+	if err != nil {
+		return "", err
+	}
+
+	key, err := scrypt.Key(password, salt, rounds, 8, 1, 32)
+	if err != nil {
+		return "", err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	h := sha256.New()
+	h.Write(data)
+	checksum := h.Sum(nil)
+	if len(checksum) != 32 {
+		return "", errors.New("wrong cs size")
+	}
+
+	ciphertext := make([]byte, 32+len(data))
+
+	iv := make([]byte, aes.BlockSize)
+	_, err = io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return "", err
+	}
+
+	combinedData := make([]byte, 0, 32+len(data))
+	combinedData = append(combinedData, checksum...)
+	combinedData = append(combinedData, data...)
+
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext, combinedData)
+
+	pl := aesPayload{
+		Salt:    salt,
+		IV:      iv,
+		Data:    ciphertext,
+		Version: 1,
+		Rounds:  rounds,
+	}
+
+	jbin, err := json.Marshal(pl)
+	if err != nil {
+		return "", err
+	}
+
+	res := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(jbin)
+
+	return res, nil
+}
+
+func DecryptAESSimple(password []byte, encText string) ([]byte, error) {
+
+	jbin, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(encText)
+	if err != nil {
+		return nil, err
+	}
+
+	var pl aesPayload
+	err = json.Unmarshal(jbin, &pl)
+	if err != nil {
+		return nil, err
+	}
+
+	if pl.Version != 1 {
+		return nil, errors.New("unsupported version")
+	}
+
+	key, err := scrypt.Key(password, pl.Salt, pl.Rounds, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	dest := make([]byte, len(pl.Data))
+
+	cfb := cipher.NewCFBDecrypter(block, pl.IV)
+	cfb.XORKeyStream(dest, pl.Data)
+
+	if len(dest) < 32 {
+		return nil, errors.New("payload too small")
+	}
+
+	chck := dest[:32]
+	data := dest[32:]
+
+	h := sha256.New()
+	h.Write(data)
+	chck2 := h.Sum(nil)
+
+	if !bytes.Equal(chck, chck2) {
+		return nil, errors.New("checksum mismatch")
+	}
+
+	return data, nil
+}

cryptext/aes_test.go

@@ -0,0 +1,35 @@
+package cryptext
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestEncryptAESSimple(t *testing.T) {
+
+	pw := []byte("hunter12")
+
+	str1 := []byte("Hello World")
+
+	str2, err := EncryptAESSimple(pw, str1, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%s\n", str2)
+
+	str3, err := DecryptAESSimple(pw, str2)
+	if err != nil {
+		panic(err)
+	}
+
+	assertEqual(t, string(str1), string(str3))
+
+	str4, err := EncryptAESSimple(pw, str3, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	assertNotEqual(t, string(str2), string(str4))
+
+}

cryptext/hash_test.go

@@ -23,3 +23,9 @@ func assertEqual(t *testing.T, actual string, expected string) {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
+
+func assertNotEqual(t *testing.T, actual string, expected string) {
+	if actual == expected {
+		t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

cryptext/passHash.go

@@ -0,0 +1,365 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/totpext"
+	"golang.org/x/crypto/bcrypt"
+	"strconv"
+	"strings"
+)
+
+const LatestPassHashVersion = 4
+
+// PassHash
+// - [v0]: plaintext password ( `0|...` )
+// - [v1]: sha256(plaintext)
+// - [v2]: seed | sha256<seed>(plaintext)
+// - [v3]: seed | sha256<seed>(plaintext) | [hex(totp)]
+// - [v4]: bcrypt(plaintext) | [hex(totp)]
+type PassHash string
+
+func (ph PassHash) Valid() bool {
+	_, _, _, _, _, valid := ph.Data()
+	return valid
+}
+
+func (ph PassHash) HasTOTP() bool {
+	_, _, _, otp, _, _ := ph.Data()
+	return otp
+}
+
+func (ph PassHash) Data() (_version int, _seed []byte, _payload []byte, _totp bool, _totpsecret []byte, _valid bool) {
+
+	split := strings.Split(string(ph), "|")
+	if len(split) == 0 {
+		return -1, nil, nil, false, nil, false
+	}
+
+	version, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return -1, nil, nil, false, nil, false
+	}
+
+	if version == 0 {
+		if len(split) != 2 {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), nil, []byte(split[1]), false, nil, true
+	}
+
+	if version == 1 {
+		if len(split) != 2 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), nil, payload, false, nil, true
+	}
+
+	//
+	if version == 2 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		seed, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[2])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		return int(version), seed, payload, false, nil, true
+	}
+
+	if version == 3 {
+		if len(split) != 4 {
+			return -1, nil, nil, false, nil, false
+		}
+		seed, err := base64.RawStdEncoding.DecodeString(split[1])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		payload, err := base64.RawStdEncoding.DecodeString(split[2])
+		if err != nil {
+			return -1, nil, nil, false, nil, false
+		}
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[3] != "0" {
+			totpsecret, err = hex.DecodeString(split[3])
+			totp = true
+		}
+		return int(version), seed, payload, totp, totpsecret, true
+	}
+
+	if version == 4 {
+		if len(split) != 3 {
+			return -1, nil, nil, false, nil, false
+		}
+		payload := []byte(split[1])
+		totp := false
+		totpsecret := make([]byte, 0)
+		if split[2] != "0" {
+			totpsecret, err = hex.DecodeString(split[3])
+			totp = true
+		}
+		return int(version), nil, payload, totp, totpsecret, true
+	}
+
+	return -1, nil, nil, false, nil, false
+}
+
+func (ph PassHash) Verify(plainpass string, totp *string) bool {
+	version, seed, payload, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return false
+	}
+
+	if hastotp && totp == nil {
+		return false
+	}
+
+	if version == 0 {
+		return langext.ArrEqualsExact([]byte(plainpass), payload)
+	}
+
+	if version == 1 {
+		return langext.ArrEqualsExact(hash256(plainpass), payload)
+	}
+
+	if version == 2 {
+		return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload)
+	}
+
+	if version == 3 {
+		if !hastotp {
+			return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload)
+		} else {
+			return langext.ArrEqualsExact(hash256Seeded(plainpass, seed), payload) && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	if version == 4 {
+		if !hastotp {
+			return bcrypt.CompareHashAndPassword(payload, []byte(plainpass)) == nil
+		} else {
+			return bcrypt.CompareHashAndPassword(payload, []byte(plainpass)) == nil && totpext.Validate(totpsecret, *totp)
+		}
+	}
+
+	return false
+}
+
+func (ph PassHash) NeedsPasswordUpgrade() bool {
+	version, _, _, _, _, valid := ph.Data()
+	return valid && version < LatestPassHashVersion
+}
+
+func (ph PassHash) Upgrade(plainpass string) (PassHash, error) {
+	version, _, _, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid password")
+	}
+	if version == LatestPassHashVersion {
+		return ph, nil
+	}
+	if hastotp {
+		return HashPassword(plainpass, totpsecret)
+	} else {
+		return HashPassword(plainpass, nil)
+	}
+}
+
+func (ph PassHash) ClearTOTP() (PassHash, error) {
+	version, _, _, _, _, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return ph, nil
+	}
+
+	if version == 1 {
+		return ph, nil
+	}
+
+	if version == 2 {
+		return ph, nil
+	}
+
+	if version == 3 {
+		split := strings.Split(string(ph), "|")
+		split[3] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 4 {
+		split := strings.Split(string(ph), "|")
+		split[2] = "0"
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) WithTOTP(totpSecret []byte) (PassHash, error) {
+	version, _, _, _, _, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 1 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 2 {
+		return "", errors.New("version does not support totp, needs upgrade")
+	}
+
+	if version == 3 {
+		split := strings.Split(string(ph), "|")
+		split[3] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	if version == 4 {
+		split := strings.Split(string(ph), "|")
+		split[2] = hex.EncodeToString(totpSecret)
+		return PassHash(strings.Join(split, "|")), nil
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) Change(newPlainPass string) (PassHash, error) {
+	version, _, _, hastotp, totpsecret, valid := ph.Data()
+	if !valid {
+		return "", errors.New("invalid PassHash")
+	}
+
+	if version == 0 {
+		return HashPasswordV0(newPlainPass)
+	}
+
+	if version == 1 {
+		return HashPasswordV1(newPlainPass)
+	}
+
+	if version == 2 {
+		return HashPasswordV2(newPlainPass)
+	}
+
+	if version == 3 {
+		return HashPasswordV3(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	if version == 4 {
+		return HashPasswordV4(newPlainPass, langext.Conditional(hastotp, totpsecret, nil))
+	}
+
+	return "", errors.New("unknown version")
+}
+
+func (ph PassHash) String() string {
+	return string(ph)
+}
+
+func HashPassword(plainpass string, totpSecret []byte) (PassHash, error) {
+	return HashPasswordV4(plainpass, totpSecret)
+}
+
+func HashPasswordV4(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	payload, err := bcrypt.GenerateFromPassword([]byte(plainpass), bcrypt.MinCost)
+	if err != nil {
+		return "", err
+	}
+
+	return PassHash(fmt.Sprintf("4|%s|%s", string(payload), strtotp)), nil
+}
+
+func HashPasswordV3(plainpass string, totpSecret []byte) (PassHash, error) {
+	var strtotp string
+
+	if totpSecret == nil {
+		strtotp = "0"
+	} else {
+		strtotp = hex.EncodeToString(totpSecret)
+	}
+
+	seed, err := newSeed()
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash256Seeded(plainpass, seed)
+
+	return PassHash(fmt.Sprintf("3|%s|%s|%s",
+		base64.RawStdEncoding.EncodeToString(seed),
+		base64.RawStdEncoding.EncodeToString(checksum),
+		strtotp)), nil
+}
+
+func HashPasswordV2(plainpass string) (PassHash, error) {
+	seed, err := newSeed()
+	if err != nil {
+		return "", err
+	}
+
+	checksum := hash256Seeded(plainpass, seed)
+
+	return PassHash(fmt.Sprintf("2|%s|%s", base64.RawStdEncoding.EncodeToString(seed), base64.RawStdEncoding.EncodeToString(checksum))), nil
+}
+
+func HashPasswordV1(plainpass string) (PassHash, error) {
+	return PassHash(fmt.Sprintf("1|%s", base64.RawStdEncoding.EncodeToString(hash256(plainpass)))), nil
+}
+
+func HashPasswordV0(plainpass string) (PassHash, error) {
+	return PassHash(fmt.Sprintf("0|%s", plainpass)), nil
+}
+
+func hash256(s string) []byte {
+	h := sha256.New()
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func hash256Seeded(s string, seed []byte) []byte {
+	h := sha256.New()
+	h.Write(seed)
+	h.Write([]byte(s))
+	bs := h.Sum(nil)
+	return bs
+}
+
+func newSeed() ([]byte, error) {
+	secret := make([]byte, 32)
+	_, err := rand.Read(secret)
+	if err != nil {
+		return nil, err
+	}
+	return secret, nil
+}

dataext/bufferedReadCloser.go

@@ -8,10 +8,10 @@ import (
 type brcMode int
 
 const (
-	modeSourceReading  = 0
+	modeSourceReading  brcMode = 0
-	modeSourceFinished = 1
+	modeSourceFinished brcMode = 1
-	modeBufferReading  = 2
+	modeBufferReading  brcMode = 2
-	modeBufferFinished = 3
+	modeBufferFinished brcMode = 3
 )
 
 type BufferedReadCloser interface {

dataext/lruMap.go

@@ -19,40 +19,38 @@ import (
 // There are also a bunch of unit tests to ensure that the cache is always in a consistent state
 //
 
-type LRUData interface{}
+type LRUMap[TKey comparable, TData any] struct {
-
-type LRUMap struct {
 	maxsize int
 	lock    sync.Mutex
 
-	cache map[string]*cacheNode
+	cache map[TKey]*cacheNode[TKey, TData]
 
-	lfuHead *cacheNode
+	lfuHead *cacheNode[TKey, TData]
-	lfuTail *cacheNode
+	lfuTail *cacheNode[TKey, TData]
 }
 
-type cacheNode struct {
+type cacheNode[TKey comparable, TData any] struct {
-	key    string
+	key    TKey
-	data   LRUData
+	data   TData
-	parent *cacheNode
+	parent *cacheNode[TKey, TData]
-	child  *cacheNode
+	child  *cacheNode[TKey, TData]
 }
 
-func NewLRUMap(size int) *LRUMap {
+func NewLRUMap[TKey comparable, TData any](size int) *LRUMap[TKey, TData] {
 	if size <= 2 && size != 0 {
 		panic("Size must be > 2  (or 0)")
 	}
 
-	return &LRUMap{
+	return &LRUMap[TKey, TData]{
 		maxsize: size,
 		lock:    sync.Mutex{},
-		cache:   make(map[string]*cacheNode, size+1),
+		cache:   make(map[TKey]*cacheNode[TKey, TData], size+1),
 		lfuHead: nil,
 		lfuTail: nil,
 	}
 }
 
-func (c *LRUMap) Put(key string, value LRUData) {
+func (c *LRUMap[TKey, TData]) Put(key TKey, value TData) {
 	if c.maxsize == 0 {
 		return // cache disabled
 	}
@@ -70,7 +68,7 @@ func (c *LRUMap) Put(key string, value LRUData) {
 	}
 
 	// key does not exist: insert into map and add to top of LFU
-	node = &cacheNode{
+	node = &cacheNode[TKey, TData]{
 		key:    key,
 		data:   value,
 		parent: nil,
@@ -95,9 +93,9 @@ func (c *LRUMap) Put(key string, value LRUData) {
 	}
 }
 
-func (c *LRUMap) TryGet(key string) (LRUData, bool) {
+func (c *LRUMap[TKey, TData]) TryGet(key TKey) (TData, bool) {
 	if c.maxsize == 0 {
-		return nil, false // cache disabled
+		return *new(TData), false // cache disabled
 	}
 
 	c.lock.Lock()
@@ -105,13 +103,13 @@ func (c *LRUMap) TryGet(key string) (LRUData, bool) {
 
 	val, ok := c.cache[key]
 	if !ok {
-		return nil, false
+		return *new(TData), false
 	}
 	c.moveNodeToTop(val)
 	return val.data, ok
 }
 
-func (c *LRUMap) moveNodeToTop(node *cacheNode) {
+func (c *LRUMap[TKey, TData]) moveNodeToTop(node *cacheNode[TKey, TData]) {
 	// (only called in critical section !)
 
 	if c.lfuHead == node { // fast case
@@ -144,7 +142,7 @@ func (c *LRUMap) moveNodeToTop(node *cacheNode) {
 	}
 }
 
-func (c *LRUMap) Size() int {
+func (c *LRUMap[TKey, TData]) Size() int {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	return len(c.cache)

dataext/lruMap_test.go

@@ -12,7 +12,7 @@ func init() {
 }
 
 func TestResultCache1(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key := randomKey()
@@ -39,7 +39,7 @@ func TestResultCache1(t *testing.T) {
 	if !ok {
 		t.Errorf("cache TryGet returned no value")
 	}
-	if !eq(cacheval, val) {
+	if cacheval != val {
 		t.Errorf("cache TryGet returned different value (%+v <> %+v)", cacheval, val)
 	}
 
@@ -50,7 +50,7 @@ func TestResultCache1(t *testing.T) {
 }
 
 func TestResultCache2(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -150,7 +150,7 @@ func TestResultCache2(t *testing.T) {
 }
 
 func TestResultCache3(t *testing.T) {
-	cache := NewLRUMap(8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -160,20 +160,20 @@ func TestResultCache3(t *testing.T) {
 	cache.Put(key1, val1)
 	verifyLRUList(cache, t)
 
-	if val, ok := cache.TryGet(key1); !ok || !eq(val, val1) {
+	if val, ok := cache.TryGet(key1); !ok || val != val1 {
 		t.Errorf("Value in cache should be [val1]")
 	}
 
 	cache.Put(key1, val2)
 	verifyLRUList(cache, t)
 
-	if val, ok := cache.TryGet(key1); !ok || !eq(val, val2) {
+	if val, ok := cache.TryGet(key1); !ok || val != val2 {
 		t.Errorf("Value in cache should be [val2]")
 	}
 }
 
 // does a basic consistency check over the internal cache representation
-func verifyLRUList(cache *LRUMap, t *testing.T) {
+func verifyLRUList[TKey comparable, TData any](cache *LRUMap[TKey, TData], t *testing.T) {
 	size := 0
 
 	tailFound := false
@@ -250,23 +250,10 @@ func randomKey() string {
 	return strconv.FormatInt(rand.Int63(), 16)
 }
 
-func randomVal() LRUData {
+func randomVal() string {
 	v, err := langext.NewHexUUID()
 	if err != nil {
 		panic(err)
 	}
-	return &v
+	return v
-}
-
-func eq(a LRUData, b LRUData) bool {
-	v1, ok1 := a.(*string)
-	v2, ok2 := b.(*string)
-	if ok1 && ok2 {
-		if v1 == nil || v2 == nil {
-			return false
-		}
-		return v1 == v2
-	}
-
-	return false
 }

dataext/stack.go

@@ -0,0 +1,116 @@
+package dataext
+
+import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"sync"
+)
+
+var ErrEmptyStack = errors.New("stack is empty")
+
+type Stack[T any] struct {
+	lock *sync.Mutex
+	data []T
+}
+
+func NewStack[T any](threadsafe bool, initialCapacity int) *Stack[T] {
+	var lck *sync.Mutex = nil
+	if threadsafe {
+		lck = &sync.Mutex{}
+	}
+	return &Stack[T]{
+		lock: lck,
+		data: make([]T, 0, initialCapacity),
+	}
+}
+
+func (s *Stack[T]) Push(v T) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	s.data = append(s.data, v)
+}
+
+func (s *Stack[T]) Pop() (T, error) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+	if l == 0 {
+		return *new(T), ErrEmptyStack
+	}
+
+	result := s.data[l-1]
+	s.data = s.data[:l-1]
+
+	return result, nil
+}
+
+func (s *Stack[T]) OptPop() *T {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+	if l == 0 {
+		return nil
+	}
+
+	result := s.data[l-1]
+	s.data = s.data[:l-1]
+
+	return langext.Ptr(result)
+}
+
+func (s *Stack[T]) Peek() (T, error) {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+
+	if l == 0 {
+		return *new(T), ErrEmptyStack
+	}
+
+	return s.data[l-1], nil
+}
+
+func (s *Stack[T]) OptPeek() *T {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	l := len(s.data)
+
+	if l == 0 {
+		return nil
+	}
+
+	return langext.Ptr(s.data[l-1])
+}
+
+func (s *Stack[T]) Length() int {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	return len(s.data)
+}
+
+func (s *Stack[T]) Empty() bool {
+	if s.lock != nil {
+		s.lock.Lock()
+		defer s.lock.Unlock()
+	}
+
+	return len(s.data) == 0
+}

dataext/syncSet.go

@@ -2,17 +2,17 @@ package dataext
 
 import "sync"
 
-type SyncStringSet struct {
+type SyncSet[TData comparable] struct {
-	data map[string]bool
+	data map[TData]bool
 	lock sync.Mutex
 }
 
-func (s *SyncStringSet) Add(value string) bool {
+func (s *SyncSet[TData]) Add(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
 	_, ok := s.data[value]
@@ -21,12 +21,12 @@ func (s *SyncStringSet) Add(value string) bool {
 	return !ok
 }
 
-func (s *SyncStringSet) AddAll(values []string) {
+func (s *SyncSet[TData]) AddAll(values []TData) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
 	for _, value := range values {
@@ -34,12 +34,12 @@ func (s *SyncStringSet) AddAll(values []string) {
 	}
 }
 
-func (s *SyncStringSet) Contains(value string) bool {
+func (s *SyncSet[TData]) Contains(value TData) bool {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
 	_, ok := s.data[value]
@@ -47,15 +47,15 @@ func (s *SyncStringSet) Contains(value string) bool {
 	return ok
 }
 
-func (s *SyncStringSet) Get() []string {
+func (s *SyncSet[TData]) Get() []TData {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
 	if s.data == nil {
-		s.data = make(map[string]bool)
+		s.data = make(map[TData]bool)
 	}
 
-	r := make([]string, 0, len(s.data))
+	r := make([]TData, 0, len(s.data))
 
 	for k := range s.data {
 		r = append(r, k)

go.mod

@@ -3,8 +3,12 @@ module gogs.mikescher.com/BlackForestBytes/goext
 go 1.19
 
 require (
-	golang.org/x/sys v0.1.0
+	golang.org/x/sys v0.3.0
-	golang.org/x/term v0.1.0
+	golang.org/x/term v0.3.0
 )
 
-require github.com/jmoiron/sqlx v1.3.5 // indirect
+require (
+	github.com/jmoiron/sqlx v1.3.5 // indirect
+	go.mongodb.org/mongo-driver v1.11.1 // indirect
+	golang.org/x/crypto v0.4.0 // indirect
+)

go.sum

@@ -1,9 +1,50 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
+go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
+golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

langext/array.go

@@ -265,6 +265,46 @@ func ArrMap[T1 any, T2 any](arr []T1, conv func(v T1) T2) []T2 {
 	return r
 }
 
+func ArrMapExt[T1 any, T2 any](arr []T1, conv func(idx int, v T1) T2) []T2 {
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i] = conv(i, v)
+	}
+	return r
+}
+
+func ArrMapErr[T1 any, T2 any](arr []T1, conv func(v T1) (T2, error)) ([]T2, error) {
+	var err error
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i], err = conv(v)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return r, nil
+}
+
+func ArrFilterMap[T1 any, T2 any](arr []T1, filter func(v T1) bool, conv func(v T1) T2) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v := range arr {
+		if filter(v) {
+			r = append(r, conv(v))
+		}
+	}
+	return r
+}
+
+func ArrFilter[T any](arr []T, filter func(v T) bool) []T {
+	r := make([]T, 0, len(arr))
+	for _, v := range arr {
+		if filter(v) {
+			r = append(r, v)
+		}
+	}
+	return r
+}
+
 func ArrSum[T NumberConstraint](arr []T) T {
 	var r T = 0
 	for _, v := range arr {
@@ -272,3 +312,19 @@ func ArrSum[T NumberConstraint](arr []T) T {
 	}
 	return r
 }
+
+func ArrFlatten[T1 any, T2 any](arr []T1, conv func(v T1) []T2) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v1 := range arr {
+		r = append(r, conv(v1)...)
+	}
+	return r
+}
+
+func ArrFlattenDirect[T1 any](arr [][]T1) []T1 {
+	r := make([]T1, 0, len(arr))
+	for _, v1 := range arr {
+		r = append(r, v1...)
+	}
+	return r
+}

langext/base58.go

@@ -0,0 +1,178 @@
+package langext
+
+import (
+	"bytes"
+	"errors"
+	"math/big"
+)
+
+// shamelessly stolen from https://github.com/btcsuite/
+
+type B58Encoding struct {
+	bigRadix     [11]*big.Int
+	bigRadix10   *big.Int
+	alphabet     string
+	alphabetIdx0 byte
+	b58          [256]byte
+}
+
+var Base58DefaultEncoding = newBase58Encoding("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
+var Base58FlickrEncoding = newBase58Encoding("123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ")
+var Base58RippleEncoding = newBase58Encoding("rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz")
+var Base58BitcoinEncoding = newBase58Encoding("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
+
+func newBase58Encoding(alphabet string) *B58Encoding {
+	bigRadix10 := big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58 * 58 * 58 * 58)
+	enc := &B58Encoding{
+		alphabet:     alphabet,
+		alphabetIdx0: '1',
+		bigRadix: [...]*big.Int{
+			big.NewInt(0),
+			big.NewInt(58),
+			big.NewInt(58 * 58),
+			big.NewInt(58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58 * 58),
+			big.NewInt(58 * 58 * 58 * 58 * 58 * 58 * 58 * 58 * 58),
+			bigRadix10,
+		},
+		bigRadix10: bigRadix10,
+	}
+
+	b58 := make([]byte, 0, 256)
+
+	for i := byte(0); i < 32; i++ {
+		for j := byte(0); j < 8; j++ {
+
+			b := i*8 + j
+
+			idx := bytes.IndexByte([]byte(alphabet), b)
+			if idx == -1 {
+				b58 = append(b58, 255)
+			} else {
+				b58 = append(b58, byte(idx))
+			}
+
+		}
+	}
+
+	enc.b58 = *((*[256]byte)(b58))
+
+	return enc
+}
+
+func (enc *B58Encoding) EncodeString(src string) (string, error) {
+	v, err := enc.Encode([]byte(src))
+	if err != nil {
+		return "", err
+	}
+	return string(v), nil
+}
+
+func (enc *B58Encoding) Encode(src []byte) ([]byte, error) {
+	x := new(big.Int)
+	x.SetBytes(src)
+
+	// maximum length of output is log58(2^(8*len(b))) == len(b) * 8 / log(58)
+	maxlen := int(float64(len(src))*1.365658237309761) + 1
+	answer := make([]byte, 0, maxlen)
+	mod := new(big.Int)
+	for x.Sign() > 0 {
+		// Calculating with big.Int is slow for each iteration.
+		//    x, mod = x / 58, x % 58
+		//
+		// Instead we can try to do as much calculations on int64.
+		//    x, mod = x / 58^10, x % 58^10
+		//
+		// Which will give us mod, which is 10 digit base58 number.
+		// We'll loop that 10 times to convert to the answer.
+
+		x.DivMod(x, enc.bigRadix10, mod)
+		if x.Sign() == 0 {
+			// When x = 0, we need to ensure we don't add any extra zeros.
+			m := mod.Int64()
+			for m > 0 {
+				answer = append(answer, enc.alphabet[m%58])
+				m /= 58
+			}
+		} else {
+			m := mod.Int64()
+			for i := 0; i < 10; i++ {
+				answer = append(answer, enc.alphabet[m%58])
+				m /= 58
+			}
+		}
+	}
+
+	// leading zero bytes
+	for _, i := range src {
+		if i != 0 {
+			break
+		}
+		answer = append(answer, enc.alphabetIdx0)
+	}
+
+	// reverse
+	alen := len(answer)
+	for i := 0; i < alen/2; i++ {
+		answer[i], answer[alen-1-i] = answer[alen-1-i], answer[i]
+	}
+
+	return answer, nil
+}
+
+func (enc *B58Encoding) DecodeString(src string) (string, error) {
+	v, err := enc.Decode([]byte(src))
+	if err != nil {
+		return "", err
+	}
+	return string(v), nil
+}
+
+func (enc *B58Encoding) Decode(src []byte) ([]byte, error) {
+	answer := big.NewInt(0)
+	scratch := new(big.Int)
+
+	for t := src; len(t) > 0; {
+		n := len(t)
+		if n > 10 {
+			n = 10
+		}
+
+		total := uint64(0)
+		for _, v := range t[:n] {
+			if v > 255 {
+				return []byte{}, errors.New("invalid char in input")
+			}
+
+			tmp := enc.b58[v]
+			if tmp == 255 {
+				return []byte{}, errors.New("invalid char in input")
+			}
+			total = total*58 + uint64(tmp)
+		}
+
+		answer.Mul(answer, enc.bigRadix[n])
+		scratch.SetUint64(total)
+		answer.Add(answer, scratch)
+
+		t = t[n:]
+	}
+
+	tmpval := answer.Bytes()
+
+	var numZeros int
+	for numZeros = 0; numZeros < len(src); numZeros++ {
+		if src[numZeros] != enc.alphabetIdx0 {
+			break
+		}
+	}
+	flen := numZeros + len(tmpval)
+	val := make([]byte, flen)
+	copy(val[numZeros:], tmpval)
+
+	return val, nil
+}

langext/base58_test.go

@@ -0,0 +1,67 @@
+package langext
+
+import (
+	"testing"
+)
+
+func _encStr(t *testing.T, enc *B58Encoding, v string) string {
+	v, err := enc.EncodeString(v)
+	if err != nil {
+		t.Error(err)
+	}
+	return v
+}
+
+func _decStr(t *testing.T, enc *B58Encoding, v string) string {
+	v, err := enc.DecodeString(v)
+	if err != nil {
+		t.Error(err)
+	}
+	return v
+}
+
+func TestBase58DefaultEncoding(t *testing.T) {
+	assertEqual(t, _encStr(t, Base58DefaultEncoding, "Hello"), "9Ajdvzr")
+	assertEqual(t, _encStr(t, Base58DefaultEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX")
+}
+
+func TestBase58DefaultDecoding(t *testing.T) {
+	assertEqual(t, _decStr(t, Base58DefaultEncoding, "9Ajdvzr"), "Hello")
+	assertEqual(t, _decStr(t, Base58DefaultEncoding, "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestBase58RippleEncoding(t *testing.T) {
+	assertEqual(t, _encStr(t, Base58RippleEncoding, "Hello"), "9wjdvzi")
+	assertEqual(t, _encStr(t, Base58RippleEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "h3as3SMcJu26nokqPxhkUVCnd3Qwdgbd4Cp3gfReYrsFi7N44bMy11jqnGj1iJHEnzeZCq1huJM7JHifVbi7hXB7ZpEA9DVtqt894reXucNWSNZ26XVaAhy1GSWqGdFeYTJCrMdDzTg3vCcQV55CJjZX")
+}
+
+func TestBase58RippleDecoding(t *testing.T) {
+	assertEqual(t, _decStr(t, Base58RippleEncoding, "9wjdvzi"), "Hello")
+	assertEqual(t, _decStr(t, Base58RippleEncoding, "h3as3SMcJu26nokqPxhkUVCnd3Qwdgbd4Cp3gfReYrsFi7N44bMy11jqnGj1iJHEnzeZCq1huJM7JHifVbi7hXB7ZpEA9DVtqt894reXucNWSNZ26XVaAhy1GSWqGdFeYTJCrMdDzTg3vCcQV55CJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestBase58BitcoinEncoding(t *testing.T) {
+	assertEqual(t, _encStr(t, Base58BitcoinEncoding, "Hello"), "9Ajdvzr")
+	assertEqual(t, _encStr(t, Base58BitcoinEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX")
+}
+
+func TestBase58BitcoinDecoding(t *testing.T) {
+	assertEqual(t, _decStr(t, Base58BitcoinEncoding, "9Ajdvzr"), "Hello")
+	assertEqual(t, _decStr(t, Base58BitcoinEncoding, "48638SMcJuah5okqPx4kCVf5d8QAdgbdNf28g7ReY13prUENNbMyssjq5GjsrJHF5zeZfqs4uJMUJHr7VbrU4XBUZ2Fw9DVtqtn9N1eXucEWSEZahXV6w4ysGSWqGdpeYTJf1MdDzTg8vfcQViifJjZX"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func TestBase58FlickrEncoding(t *testing.T) {
+	assertEqual(t, _encStr(t, Base58FlickrEncoding, "Hello"), "9aJCVZR")
+	assertEqual(t, _encStr(t, Base58FlickrEncoding, "If debugging is the process of removing software bugs, then programming must be the process of putting them in."), "48638rmBiUzG5NKQoX4KcuE5C8paCFACnE28F7qDx13PRtennAmYSSJQ5gJSRihf5ZDyEQS4UimtihR7uARt4wbty2fW9duTQTM9n1DwUBevreyzGwu6W4YSgrvQgCPDxsiE1mCdZsF8VEBpuHHEiJyw")
+}
+
+func TestBase58FlickrDecoding(t *testing.T) {
+	assertEqual(t, _decStr(t, Base58FlickrEncoding, "9aJCVZR"), "Hello")
+	assertEqual(t, _decStr(t, Base58FlickrEncoding, "48638rmBiUzG5NKQoX4KcuE5C8paCFACnE28F7qDx13PRtennAmYSSJQ5gJSRihf5ZDyEQS4UimtihR7uARt4wbty2fW9duTQTM9n1DwUBevreyzGwu6W4YSgrvQgCPDxsiE1mCdZsF8VEBpuHHEiJyw"), "If debugging is the process of removing software bugs, then programming must be the process of putting them in.")
+}
+
+func assertEqual(t *testing.T, actual string, expected string) {
+	if actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

langext/bool.go

@@ -15,3 +15,35 @@ func Conditional[T any](v bool, resTrue T, resFalse T) T {
 		return resFalse
 	}
 }
+
+func ConditionalFn00[T any](v bool, resTrue T, resFalse T) T {
+	if v {
+		return resTrue
+	} else {
+		return resFalse
+	}
+}
+
+func ConditionalFn10[T any](v bool, resTrue func() T, resFalse T) T {
+	if v {
+		return resTrue()
+	} else {
+		return resFalse
+	}
+}
+
+func ConditionalFn01[T any](v bool, resTrue T, resFalse func() T) T {
+	if v {
+		return resTrue
+	} else {
+		return resFalse()
+	}
+}
+
+func ConditionalFn11[T any](v bool, resTrue func() T, resFalse func() T) T {
+	if v {
+		return resTrue()
+	} else {
+		return resFalse()
+	}
+}

langext/maps.go

@@ -7,3 +7,11 @@ func MapKeyArr[T comparable, V any](v map[T]V) []T {
 	}
 	return result
 }
+
+func ArrToMap[T comparable, V any](a []V, keyfunc func(V) T) map[T]V {
+	result := make(map[T]V, len(a))
+	for _, v := range a {
+		result[keyfunc(v)] = v
+	}
+	return result
+}

langext/pointer.go

@@ -34,3 +34,7 @@ func IsNil(i interface{}) bool {
 	}
 	return false
 }
+
+func PtrEquals[T comparable](v1 *T, v2 *T) bool {
+	return (v1 == nil && v2 == nil) || (v1 != nil && v2 != nil && *v1 == *v2)
+}

langext/sort.go

@@ -37,3 +37,21 @@ func IsSliceSorted[T any](arr []T, less func(v1, v2 T) bool) bool {
 		return less(arr[i1], arr[i2])
 	})
 }
+
+func SortBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+}
+
+func SortByStable[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+}
+
+func IsSortedBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+}

rext/wrapper.go

@@ -0,0 +1,130 @@
+package rext
+
+import "regexp"
+
+type Regex interface {
+	IsMatch(haystack string) bool
+	MatchFirst(haystack string) (RegexMatch, bool)
+	MatchAll(haystack string) []RegexMatch
+	ReplaceAll(haystack string, repl string, literal bool) string
+	ReplaceAllFunc(haystack string, repl func(string) string) string
+	RemoveAll(haystack string) string
+	GroupCount() int
+}
+
+type regexWrapper struct {
+	rex      *regexp.Regexp
+	subnames []string
+}
+
+type RegexMatch struct {
+	haystack        string
+	submatchesIndex []int
+	subnames        []string
+}
+
+type RegexMatchGroup struct {
+	haystack string
+	start    int
+	end      int
+}
+
+func W(rex *regexp.Regexp) Regex {
+	return &regexWrapper{rex: rex, subnames: rex.SubexpNames()}
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+
+func (w *regexWrapper) IsMatch(haystack string) bool {
+	return w.rex.MatchString(haystack)
+}
+
+func (w *regexWrapper) MatchFirst(haystack string) (RegexMatch, bool) {
+	res := w.rex.FindStringSubmatchIndex(haystack)
+	if res == nil {
+		return RegexMatch{}, false
+	}
+
+	return RegexMatch{haystack: haystack, submatchesIndex: res, subnames: w.subnames}, true
+}
+
+func (w *regexWrapper) MatchAll(haystack string) []RegexMatch {
+	resarr := w.rex.FindAllStringSubmatchIndex(haystack, -1)
+
+	matches := make([]RegexMatch, 0, len(resarr))
+	for _, res := range resarr {
+		matches = append(matches, RegexMatch{haystack: haystack, submatchesIndex: res, subnames: w.subnames})
+	}
+
+	return matches
+}
+
+func (w *regexWrapper) ReplaceAll(haystack string, repl string, literal bool) string {
+	if literal {
+		// do not expand placeholder aka $1, $2, ...
+		return w.rex.ReplaceAllLiteralString(haystack, repl)
+	} else {
+		return w.rex.ReplaceAllString(haystack, repl)
+	}
+}
+
+func (w *regexWrapper) ReplaceAllFunc(haystack string, repl func(string) string) string {
+	return w.rex.ReplaceAllStringFunc(haystack, repl)
+}
+
+func (w *regexWrapper) RemoveAll(haystack string) string {
+	return w.rex.ReplaceAllLiteralString(haystack, "")
+}
+
+// GroupCount returns the amount of groups in this match, does not count group-0 (whole match)
+func (w *regexWrapper) GroupCount() int {
+	return len(w.subnames) - 1
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+
+func (m RegexMatch) FullMatch() RegexMatchGroup {
+	return RegexMatchGroup{haystack: m.haystack, start: m.submatchesIndex[0], end: m.submatchesIndex[1]}
+}
+
+// GroupCount returns the amount of groups in this match, does not count group-0 (whole match)
+func (m RegexMatch) GroupCount() int {
+	return len(m.subnames) - 1
+}
+
+// GroupByIndex returns the value of a matched group (group 0 == whole match)
+func (m RegexMatch) GroupByIndex(idx int) RegexMatchGroup {
+	return RegexMatchGroup{haystack: m.haystack, start: m.submatchesIndex[idx*2], end: m.submatchesIndex[idx*2+1]}
+}
+
+// GroupByName returns the value of a matched group (group 0 == whole match)
+func (m RegexMatch) GroupByName(name string) RegexMatchGroup {
+	for idx, subname := range m.subnames {
+		if subname == name {
+			return RegexMatchGroup{haystack: m.haystack, start: m.submatchesIndex[idx*2], end: m.submatchesIndex[idx*2+1]}
+		}
+	}
+	panic("failed to find regex-group by name")
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+
+func (g RegexMatchGroup) Value() string {
+	return g.haystack[g.start:g.end]
+}
+
+func (g RegexMatchGroup) Start() int {
+	return g.start
+}
+
+func (g RegexMatchGroup) End() int {
+	return g.end
+}
+
+func (g RegexMatchGroup) Range() (int, int) {
+	return g.start, g.end
+}
+
+func (g RegexMatchGroup) Length() int {
+	return g.end - g.start
+}

rfctime/interface.go

@@ -0,0 +1,101 @@
+package rfctime
+
+import "time"
+
+type RFCTime interface {
+	AnyTime
+
+	Time() time.Time
+	Serialize() string
+
+	After(u AnyTime) bool
+	Before(u AnyTime) bool
+	Equal(u AnyTime) bool
+
+	Sub(u AnyTime) time.Duration
+}
+
+type AnyTime interface {
+	MarshalJSON() ([]byte, error)
+
+	MarshalBinary() ([]byte, error)
+
+	GobEncode() ([]byte, error)
+
+	MarshalText() ([]byte, error)
+
+	IsZero() bool
+	Date() (year int, month time.Month, day int)
+	Year() int
+	Month() time.Month
+	Day() int
+	Weekday() time.Weekday
+	ISOWeek() (year, week int)
+	Clock() (hour, min, sec int)
+	Hour() int
+	Minute() int
+	Second() int
+	Nanosecond() int
+	YearDay() int
+	Unix() int64
+	UnixMilli() int64
+	UnixMicro() int64
+	UnixNano() int64
+	Format(layout string) string
+	GoString() string
+	String() string
+
+	Location() *time.Location
+}
+
+type RFCDuration interface {
+	Time() time.Time
+	Serialize() string
+
+	UnmarshalJSON(bytes []byte) error
+	MarshalJSON() ([]byte, error)
+
+	MarshalBinary() ([]byte, error)
+	UnmarshalBinary(data []byte) error
+
+	GobEncode() ([]byte, error)
+	GobDecode(data []byte) error
+
+	MarshalText() ([]byte, error)
+	UnmarshalText(data []byte) error
+
+	After(u AnyTime) bool
+	Before(u AnyTime) bool
+	Equal(u AnyTime) bool
+	IsZero() bool
+	Date() (year int, month time.Month, day int)
+	Year() int
+	Month() time.Month
+	Day() int
+	Weekday() time.Weekday
+	ISOWeek() (year, week int)
+	Clock() (hour, min, sec int)
+	Hour() int
+	Minute() int
+	Second() int
+	Nanosecond() int
+	YearDay() int
+	Sub(u AnyTime) time.Duration
+	Unix() int64
+	UnixMilli() int64
+	UnixMicro() int64
+	UnixNano() int64
+	Format(layout string) string
+	GoString() string
+	String() string
+}
+
+func tt(v AnyTime) time.Time {
+	if r, ok := v.(time.Time); ok {
+		return r
+	}
+	if r, ok := v.(RFCTime); ok {
+		return r.Time()
+	}
+	return time.Unix(0, v.UnixNano()).In(v.Location())
+}

rfctime/interface_test.go

@@ -0,0 +1,50 @@
+package rfctime
+
+import (
+	"testing"
+	"time"
+)
+
+func TestAnyTimeInterface(t *testing.T) {
+
+	var v AnyTime
+
+	v = NowRFC3339Nano()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowRFC3339()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowUnix()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowUnixMilli()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowUnixNano()
+	assertEqual(t, v.String(), v.String())
+
+	v = time.Now()
+	assertEqual(t, v.String(), v.String())
+
+}
+
+func TestRFCTimeInterface(t *testing.T) {
+	var v RFCTime
+
+	v = NowRFC3339Nano()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowRFC3339()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowUnix()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowUnixMilli()
+	assertEqual(t, v.String(), v.String())
+
+	v = NowUnixNano()
+	assertEqual(t, v.String(), v.String())
+
+}

rfctime/rfc3339.go

@@ -0,0 +1,207 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"time"
+)
+
+type RFC3339Time time.Time
+
+func (t RFC3339Time) Time() time.Time {
+	return time.Time(t)
+}
+
+func (t RFC3339Time) MarshalBinary() ([]byte, error) {
+	return (time.Time)(t).MarshalBinary()
+}
+
+func (t *RFC3339Time) UnmarshalBinary(data []byte) error {
+	return (*time.Time)(t).UnmarshalBinary(data)
+}
+
+func (t RFC3339Time) GobEncode() ([]byte, error) {
+	return (time.Time)(t).GobEncode()
+}
+
+func (t *RFC3339Time) GobDecode(data []byte) error {
+	return (*time.Time)(t).GobDecode(data)
+}
+
+func (t *RFC3339Time) UnmarshalJSON(data []byte) error {
+	str := ""
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	t0, err := time.Parse(t.FormatStr(), str)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339Time(t0)
+	return nil
+}
+
+func (t RFC3339Time) MarshalJSON() ([]byte, error) {
+	str := t.Time().Format(t.FormatStr())
+	return json.Marshal(str)
+}
+
+func (t RFC3339Time) MarshalText() ([]byte, error) {
+	b := make([]byte, 0, len(t.FormatStr()))
+	return t.Time().AppendFormat(b, t.FormatStr()), nil
+}
+
+func (t *RFC3339Time) UnmarshalText(data []byte) error {
+	var err error
+	v, err := time.Parse(t.FormatStr(), string(data))
+	if err != nil {
+		return err
+	}
+	tt := RFC3339Time(v)
+	*t = tt
+	return nil
+}
+
+func (t *RFC3339Time) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt != bsontype.DateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339Time", bt))
+	}
+	var tt time.Time
+	err := bson.Unmarshal(data, &tt)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339Time(tt)
+	return nil
+}
+
+func (t RFC3339Time) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t RFC3339Time) Serialize() string {
+	return t.Time().Format(t.FormatStr())
+}
+
+func (t RFC3339Time) FormatStr() string {
+	return time.RFC3339
+}
+
+func (t RFC3339Time) After(u AnyTime) bool {
+	return t.Time().After(tt(u))
+}
+
+func (t RFC3339Time) Before(u AnyTime) bool {
+	return t.Time().Before(tt(u))
+}
+
+func (t RFC3339Time) Equal(u AnyTime) bool {
+	return t.Time().Equal(tt(u))
+}
+
+func (t RFC3339Time) IsZero() bool {
+	return t.Time().IsZero()
+}
+
+func (t RFC3339Time) Date() (year int, month time.Month, day int) {
+	return t.Time().Date()
+}
+
+func (t RFC3339Time) Year() int {
+	return t.Time().Year()
+}
+
+func (t RFC3339Time) Month() time.Month {
+	return t.Time().Month()
+}
+
+func (t RFC3339Time) Day() int {
+	return t.Time().Day()
+}
+
+func (t RFC3339Time) Weekday() time.Weekday {
+	return t.Time().Weekday()
+}
+
+func (t RFC3339Time) ISOWeek() (year, week int) {
+	return t.Time().ISOWeek()
+}
+
+func (t RFC3339Time) Clock() (hour, min, sec int) {
+	return t.Time().Clock()
+}
+
+func (t RFC3339Time) Hour() int {
+	return t.Time().Hour()
+}
+
+func (t RFC3339Time) Minute() int {
+	return t.Time().Minute()
+}
+
+func (t RFC3339Time) Second() int {
+	return t.Time().Second()
+}
+
+func (t RFC3339Time) Nanosecond() int {
+	return t.Time().Nanosecond()
+}
+
+func (t RFC3339Time) YearDay() int {
+	return t.Time().YearDay()
+}
+
+func (t RFC3339Time) Add(d time.Duration) RFC3339Time {
+	return RFC3339Time(t.Time().Add(d))
+}
+
+func (t RFC3339Time) Sub(u AnyTime) time.Duration {
+	return t.Time().Sub(tt(u))
+}
+
+func (t RFC3339Time) AddDate(years int, months int, days int) RFC3339Time {
+	return RFC3339Time(t.Time().AddDate(years, months, days))
+}
+
+func (t RFC3339Time) Unix() int64 {
+	return t.Time().Unix()
+}
+
+func (t RFC3339Time) UnixMilli() int64 {
+	return t.Time().UnixMilli()
+}
+
+func (t RFC3339Time) UnixMicro() int64 {
+	return t.Time().UnixMicro()
+}
+
+func (t RFC3339Time) UnixNano() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t RFC3339Time) Format(layout string) string {
+	return t.Time().Format(layout)
+}
+
+func (t RFC3339Time) GoString() string {
+	return t.Time().GoString()
+}
+
+func (t RFC3339Time) String() string {
+	return t.Time().String()
+}
+
+func (t RFC3339Time) Location() *time.Location {
+	return t.Time().Location()
+}
+
+func NewRFC3339(t time.Time) RFC3339Time {
+	return RFC3339Time(t)
+}
+
+func NowRFC3339() RFC3339Time {
+	return RFC3339Time(time.Now())
+}

rfctime/rfc3339Nano.go

@@ -0,0 +1,207 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"time"
+)
+
+type RFC3339NanoTime time.Time
+
+func (t RFC3339NanoTime) Time() time.Time {
+	return time.Time(t)
+}
+
+func (t RFC3339NanoTime) MarshalBinary() ([]byte, error) {
+	return (time.Time)(t).MarshalBinary()
+}
+
+func (t *RFC3339NanoTime) UnmarshalBinary(data []byte) error {
+	return (*time.Time)(t).UnmarshalBinary(data)
+}
+
+func (t RFC3339NanoTime) GobEncode() ([]byte, error) {
+	return (time.Time)(t).GobEncode()
+}
+
+func (t *RFC3339NanoTime) GobDecode(data []byte) error {
+	return (*time.Time)(t).GobDecode(data)
+}
+
+func (t *RFC3339NanoTime) UnmarshalJSON(data []byte) error {
+	str := ""
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	t0, err := time.Parse(t.FormatStr(), str)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339NanoTime(t0)
+	return nil
+}
+
+func (t RFC3339NanoTime) MarshalJSON() ([]byte, error) {
+	str := t.Time().Format(t.FormatStr())
+	return json.Marshal(str)
+}
+
+func (t RFC3339NanoTime) MarshalText() ([]byte, error) {
+	b := make([]byte, 0, len(t.FormatStr()))
+	return t.Time().AppendFormat(b, t.FormatStr()), nil
+}
+
+func (t *RFC3339NanoTime) UnmarshalText(data []byte) error {
+	var err error
+	v, err := time.Parse(t.FormatStr(), string(data))
+	if err != nil {
+		return err
+	}
+	tt := RFC3339NanoTime(v)
+	*t = tt
+	return nil
+}
+
+func (t *RFC3339NanoTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt != bsontype.DateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339NanoTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339NanoTime(tt)
+	return nil
+}
+
+func (t RFC3339NanoTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t RFC3339NanoTime) Serialize() string {
+	return t.Time().Format(t.FormatStr())
+}
+
+func (t RFC3339NanoTime) FormatStr() string {
+	return time.RFC3339Nano
+}
+
+func (t RFC3339NanoTime) After(u AnyTime) bool {
+	return t.Time().After(tt(u))
+}
+
+func (t RFC3339NanoTime) Before(u AnyTime) bool {
+	return t.Time().Before(tt(u))
+}
+
+func (t RFC3339NanoTime) Equal(u AnyTime) bool {
+	return t.Time().Equal(tt(u))
+}
+
+func (t RFC3339NanoTime) IsZero() bool {
+	return t.Time().IsZero()
+}
+
+func (t RFC3339NanoTime) Date() (year int, month time.Month, day int) {
+	return t.Time().Date()
+}
+
+func (t RFC3339NanoTime) Year() int {
+	return t.Time().Year()
+}
+
+func (t RFC3339NanoTime) Month() time.Month {
+	return t.Time().Month()
+}
+
+func (t RFC3339NanoTime) Day() int {
+	return t.Time().Day()
+}
+
+func (t RFC3339NanoTime) Weekday() time.Weekday {
+	return t.Time().Weekday()
+}
+
+func (t RFC3339NanoTime) ISOWeek() (year, week int) {
+	return t.Time().ISOWeek()
+}
+
+func (t RFC3339NanoTime) Clock() (hour, min, sec int) {
+	return t.Time().Clock()
+}
+
+func (t RFC3339NanoTime) Hour() int {
+	return t.Time().Hour()
+}
+
+func (t RFC3339NanoTime) Minute() int {
+	return t.Time().Minute()
+}
+
+func (t RFC3339NanoTime) Second() int {
+	return t.Time().Second()
+}
+
+func (t RFC3339NanoTime) Nanosecond() int {
+	return t.Time().Nanosecond()
+}
+
+func (t RFC3339NanoTime) YearDay() int {
+	return t.Time().YearDay()
+}
+
+func (t RFC3339NanoTime) Add(d time.Duration) RFC3339NanoTime {
+	return RFC3339NanoTime(t.Time().Add(d))
+}
+
+func (t RFC3339NanoTime) Sub(u AnyTime) time.Duration {
+	return t.Time().Sub(tt(u))
+}
+
+func (t RFC3339NanoTime) AddDate(years int, months int, days int) RFC3339NanoTime {
+	return RFC3339NanoTime(t.Time().AddDate(years, months, days))
+}
+
+func (t RFC3339NanoTime) Unix() int64 {
+	return t.Time().Unix()
+}
+
+func (t RFC3339NanoTime) UnixMilli() int64 {
+	return t.Time().UnixMilli()
+}
+
+func (t RFC3339NanoTime) UnixMicro() int64 {
+	return t.Time().UnixMicro()
+}
+
+func (t RFC3339NanoTime) UnixNano() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t RFC3339NanoTime) Format(layout string) string {
+	return t.Time().Format(layout)
+}
+
+func (t RFC3339NanoTime) GoString() string {
+	return t.Time().GoString()
+}
+
+func (t RFC3339NanoTime) String() string {
+	return t.Time().String()
+}
+
+func (t RFC3339NanoTime) Location() *time.Location {
+	return t.Time().Location()
+}
+
+func NewRFC3339Nano(t time.Time) RFC3339NanoTime {
+	return RFC3339NanoTime(t)
+}
+
+func NowRFC3339Nano() RFC3339NanoTime {
+	return RFC3339NanoTime(time.Now())
+}

rfctime/rfc3339Nano_test.go

@@ -0,0 +1,52 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestRoundtrip(t *testing.T) {
+
+	type Wrap struct {
+		Value RFC3339NanoTime `json:"v"`
+	}
+
+	val1 := NewRFC3339Nano(time.Unix(0, 1675951556820915171))
+	w1 := Wrap{val1}
+
+	jstr1, err := json.Marshal(w1)
+	if err != nil {
+		panic(err)
+	}
+
+	if string(jstr1) != "{\"v\":\"2023-02-09T15:05:56.820915171+01:00\"}" {
+		t.Errorf(string(jstr1))
+		t.Errorf("repr differs")
+	}
+
+	w2 := Wrap{}
+
+	err = json.Unmarshal(jstr1, &w2)
+	if err != nil {
+		panic(err)
+	}
+
+	jstr2, err := json.Marshal(w2)
+	if err != nil {
+		panic(err)
+	}
+
+	assertEqual(t, string(jstr1), string(jstr2))
+
+	if !w1.Value.Equal(&w2.Value) {
+		t.Errorf("time differs")
+	}
+
+}
+
+func assertEqual(t *testing.T, actual string, expected string) {
+	if actual != expected {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

rfctime/seconds.go

@@ -0,0 +1,59 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"time"
+)
+
+type SecondsF64 time.Duration
+
+func (d SecondsF64) Duration() time.Duration {
+	return time.Duration(d)
+}
+
+func (d SecondsF64) String() string {
+	return d.Duration().String()
+}
+
+func (d SecondsF64) Nanoseconds() int64 {
+	return d.Duration().Nanoseconds()
+}
+
+func (d SecondsF64) Microseconds() int64 {
+	return d.Duration().Microseconds()
+}
+
+func (d SecondsF64) Milliseconds() int64 {
+	return d.Duration().Milliseconds()
+}
+
+func (d SecondsF64) Seconds() float64 {
+	return d.Duration().Seconds()
+}
+
+func (d SecondsF64) Minutes() float64 {
+	return d.Duration().Minutes()
+}
+
+func (d SecondsF64) Hours() float64 {
+	return d.Duration().Hours()
+}
+
+func (d *SecondsF64) UnmarshalJSON(data []byte) error {
+	var secs float64 = 0
+	if err := json.Unmarshal(data, &secs); err != nil {
+		return err
+	}
+	*d = SecondsF64(timeext.FromSeconds(secs))
+	return nil
+}
+
+func (d SecondsF64) MarshalJSON() ([]byte, error) {
+	secs := d.Seconds()
+	return json.Marshal(secs)
+}
+
+func NewSecondsF64(t time.Duration) SecondsF64 {
+	return SecondsF64(t)
+}

rfctime/unix.go

@@ -0,0 +1,180 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+type UnixTime time.Time
+
+func (t UnixTime) Time() time.Time {
+	return time.Time(t)
+}
+
+func (t UnixTime) MarshalBinary() ([]byte, error) {
+	return (time.Time)(t).MarshalBinary()
+}
+
+func (t *UnixTime) UnmarshalBinary(data []byte) error {
+	return (*time.Time)(t).UnmarshalBinary(data)
+}
+
+func (t UnixTime) GobEncode() ([]byte, error) {
+	return (time.Time)(t).GobEncode()
+}
+
+func (t *UnixTime) GobDecode(data []byte) error {
+	return (*time.Time)(t).GobDecode(data)
+}
+
+func (t *UnixTime) UnmarshalJSON(data []byte) error {
+	str := ""
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	t0, err := strconv.ParseInt(str, 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = UnixTime(time.Unix(t0, 0))
+	return nil
+}
+
+func (t UnixTime) MarshalJSON() ([]byte, error) {
+	str := strconv.FormatInt(t.Time().Unix(), 10)
+	return json.Marshal(str)
+}
+
+func (t UnixTime) MarshalText() ([]byte, error) {
+	return []byte(strconv.FormatInt(t.Time().Unix(), 10)), nil
+}
+
+func (t *UnixTime) UnmarshalText(data []byte) error {
+	t0, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = UnixTime(time.Unix(t0, 0))
+	return nil
+}
+
+func (t UnixTime) Serialize() string {
+	return strconv.FormatInt(t.Time().Unix(), 10)
+}
+
+func (t UnixTime) After(u AnyTime) bool {
+	return t.Time().After(tt(u))
+}
+
+func (t UnixTime) Before(u AnyTime) bool {
+	return t.Time().Before(tt(u))
+}
+
+func (t UnixTime) Equal(u AnyTime) bool {
+	return t.Time().Equal(tt(u))
+}
+
+func (t UnixTime) IsZero() bool {
+	return t.Time().IsZero()
+}
+
+func (t UnixTime) Date() (year int, month time.Month, day int) {
+	return t.Time().Date()
+}
+
+func (t UnixTime) Year() int {
+	return t.Time().Year()
+}
+
+func (t UnixTime) Month() time.Month {
+	return t.Time().Month()
+}
+
+func (t UnixTime) Day() int {
+	return t.Time().Day()
+}
+
+func (t UnixTime) Weekday() time.Weekday {
+	return t.Time().Weekday()
+}
+
+func (t UnixTime) ISOWeek() (year, week int) {
+	return t.Time().ISOWeek()
+}
+
+func (t UnixTime) Clock() (hour, min, sec int) {
+	return t.Time().Clock()
+}
+
+func (t UnixTime) Hour() int {
+	return t.Time().Hour()
+}
+
+func (t UnixTime) Minute() int {
+	return t.Time().Minute()
+}
+
+func (t UnixTime) Second() int {
+	return t.Time().Second()
+}
+
+func (t UnixTime) Nanosecond() int {
+	return t.Time().Nanosecond()
+}
+
+func (t UnixTime) YearDay() int {
+	return t.Time().YearDay()
+}
+
+func (t UnixTime) Add(d time.Duration) UnixTime {
+	return UnixTime(t.Time().Add(d))
+}
+
+func (t UnixTime) Sub(u AnyTime) time.Duration {
+	return t.Time().Sub(tt(u))
+}
+
+func (t UnixTime) AddDate(years int, months int, days int) UnixTime {
+	return UnixTime(t.Time().AddDate(years, months, days))
+}
+
+func (t UnixTime) Unix() int64 {
+	return t.Time().Unix()
+}
+
+func (t UnixTime) UnixMilli() int64 {
+	return t.Time().UnixMilli()
+}
+
+func (t UnixTime) UnixMicro() int64 {
+	return t.Time().UnixMicro()
+}
+
+func (t UnixTime) UnixNano() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t UnixTime) Format(layout string) string {
+	return t.Time().Format(layout)
+}
+
+func (t UnixTime) GoString() string {
+	return t.Time().GoString()
+}
+
+func (t UnixTime) String() string {
+	return t.Time().String()
+}
+
+func (t UnixTime) Location() *time.Location {
+	return t.Time().Location()
+}
+
+func NewUnix(t time.Time) UnixTime {
+	return UnixTime(t)
+}
+
+func NowUnix() UnixTime {
+	return UnixTime(time.Now())
+}

rfctime/unixMilli.go

@@ -0,0 +1,180 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+type UnixMilliTime time.Time
+
+func (t UnixMilliTime) Time() time.Time {
+	return time.Time(t)
+}
+
+func (t UnixMilliTime) MarshalBinary() ([]byte, error) {
+	return (time.Time)(t).MarshalBinary()
+}
+
+func (t *UnixMilliTime) UnmarshalBinary(data []byte) error {
+	return (*time.Time)(t).UnmarshalBinary(data)
+}
+
+func (t UnixMilliTime) GobEncode() ([]byte, error) {
+	return (time.Time)(t).GobEncode()
+}
+
+func (t *UnixMilliTime) GobDecode(data []byte) error {
+	return (*time.Time)(t).GobDecode(data)
+}
+
+func (t *UnixMilliTime) UnmarshalJSON(data []byte) error {
+	str := ""
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	t0, err := strconv.ParseInt(str, 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = UnixMilliTime(time.UnixMilli(t0))
+	return nil
+}
+
+func (t UnixMilliTime) MarshalJSON() ([]byte, error) {
+	str := strconv.FormatInt(t.Time().UnixMilli(), 10)
+	return json.Marshal(str)
+}
+
+func (t UnixMilliTime) MarshalText() ([]byte, error) {
+	return []byte(strconv.FormatInt(t.Time().UnixMilli(), 10)), nil
+}
+
+func (t *UnixMilliTime) UnmarshalText(data []byte) error {
+	t0, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = UnixMilliTime(time.UnixMilli(t0))
+	return nil
+}
+
+func (t UnixMilliTime) Serialize() string {
+	return strconv.FormatInt(t.Time().UnixMilli(), 10)
+}
+
+func (t UnixMilliTime) After(u AnyTime) bool {
+	return t.Time().After(tt(u))
+}
+
+func (t UnixMilliTime) Before(u AnyTime) bool {
+	return t.Time().Before(tt(u))
+}
+
+func (t UnixMilliTime) Equal(u AnyTime) bool {
+	return t.Time().Equal(tt(u))
+}
+
+func (t UnixMilliTime) IsZero() bool {
+	return t.Time().IsZero()
+}
+
+func (t UnixMilliTime) Date() (year int, month time.Month, day int) {
+	return t.Time().Date()
+}
+
+func (t UnixMilliTime) Year() int {
+	return t.Time().Year()
+}
+
+func (t UnixMilliTime) Month() time.Month {
+	return t.Time().Month()
+}
+
+func (t UnixMilliTime) Day() int {
+	return t.Time().Day()
+}
+
+func (t UnixMilliTime) Weekday() time.Weekday {
+	return t.Time().Weekday()
+}
+
+func (t UnixMilliTime) ISOWeek() (year, week int) {
+	return t.Time().ISOWeek()
+}
+
+func (t UnixMilliTime) Clock() (hour, min, sec int) {
+	return t.Time().Clock()
+}
+
+func (t UnixMilliTime) Hour() int {
+	return t.Time().Hour()
+}
+
+func (t UnixMilliTime) Minute() int {
+	return t.Time().Minute()
+}
+
+func (t UnixMilliTime) Second() int {
+	return t.Time().Second()
+}
+
+func (t UnixMilliTime) Nanosecond() int {
+	return t.Time().Nanosecond()
+}
+
+func (t UnixMilliTime) YearDay() int {
+	return t.Time().YearDay()
+}
+
+func (t UnixMilliTime) Add(d time.Duration) UnixMilliTime {
+	return UnixMilliTime(t.Time().Add(d))
+}
+
+func (t UnixMilliTime) Sub(u AnyTime) time.Duration {
+	return t.Time().Sub(tt(u))
+}
+
+func (t UnixMilliTime) AddDate(years int, months int, days int) UnixMilliTime {
+	return UnixMilliTime(t.Time().AddDate(years, months, days))
+}
+
+func (t UnixMilliTime) Unix() int64 {
+	return t.Time().Unix()
+}
+
+func (t UnixMilliTime) UnixMilli() int64 {
+	return t.Time().UnixMilli()
+}
+
+func (t UnixMilliTime) UnixMicro() int64 {
+	return t.Time().UnixMicro()
+}
+
+func (t UnixMilliTime) UnixNano() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t UnixMilliTime) Format(layout string) string {
+	return t.Time().Format(layout)
+}
+
+func (t UnixMilliTime) GoString() string {
+	return t.Time().GoString()
+}
+
+func (t UnixMilliTime) String() string {
+	return t.Time().String()
+}
+
+func (t UnixMilliTime) Location() *time.Location {
+	return t.Time().Location()
+}
+
+func NewUnixMilli(t time.Time) UnixMilliTime {
+	return UnixMilliTime(t)
+}
+
+func NowUnixMilli() UnixMilliTime {
+	return UnixMilliTime(time.Now())
+}

rfctime/unixNano.go

@@ -0,0 +1,180 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+type UnixNanoTime time.Time
+
+func (t UnixNanoTime) Time() time.Time {
+	return time.Time(t)
+}
+
+func (t UnixNanoTime) MarshalBinary() ([]byte, error) {
+	return (time.Time)(t).MarshalBinary()
+}
+
+func (t *UnixNanoTime) UnmarshalBinary(data []byte) error {
+	return (*time.Time)(t).UnmarshalBinary(data)
+}
+
+func (t UnixNanoTime) GobEncode() ([]byte, error) {
+	return (time.Time)(t).GobEncode()
+}
+
+func (t *UnixNanoTime) GobDecode(data []byte) error {
+	return (*time.Time)(t).GobDecode(data)
+}
+
+func (t *UnixNanoTime) UnmarshalJSON(data []byte) error {
+	str := ""
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	t0, err := strconv.ParseInt(str, 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = UnixNanoTime(time.Unix(0, t0))
+	return nil
+}
+
+func (t UnixNanoTime) MarshalJSON() ([]byte, error) {
+	str := strconv.FormatInt(t.Time().UnixNano(), 10)
+	return json.Marshal(str)
+}
+
+func (t UnixNanoTime) MarshalText() ([]byte, error) {
+	return []byte(strconv.FormatInt(t.Time().UnixNano(), 10)), nil
+}
+
+func (t *UnixNanoTime) UnmarshalText(data []byte) error {
+	t0, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = UnixNanoTime(time.Unix(0, t0))
+	return nil
+}
+
+func (t UnixNanoTime) Serialize() string {
+	return strconv.FormatInt(t.Time().UnixNano(), 10)
+}
+
+func (t UnixNanoTime) After(u AnyTime) bool {
+	return t.Time().After(tt(u))
+}
+
+func (t UnixNanoTime) Before(u AnyTime) bool {
+	return t.Time().Before(tt(u))
+}
+
+func (t UnixNanoTime) Equal(u AnyTime) bool {
+	return t.Time().Equal(tt(u))
+}
+
+func (t UnixNanoTime) IsZero() bool {
+	return t.Time().IsZero()
+}
+
+func (t UnixNanoTime) Date() (year int, month time.Month, day int) {
+	return t.Time().Date()
+}
+
+func (t UnixNanoTime) Year() int {
+	return t.Time().Year()
+}
+
+func (t UnixNanoTime) Month() time.Month {
+	return t.Time().Month()
+}
+
+func (t UnixNanoTime) Day() int {
+	return t.Time().Day()
+}
+
+func (t UnixNanoTime) Weekday() time.Weekday {
+	return t.Time().Weekday()
+}
+
+func (t UnixNanoTime) ISOWeek() (year, week int) {
+	return t.Time().ISOWeek()
+}
+
+func (t UnixNanoTime) Clock() (hour, min, sec int) {
+	return t.Time().Clock()
+}
+
+func (t UnixNanoTime) Hour() int {
+	return t.Time().Hour()
+}
+
+func (t UnixNanoTime) Minute() int {
+	return t.Time().Minute()
+}
+
+func (t UnixNanoTime) Second() int {
+	return t.Time().Second()
+}
+
+func (t UnixNanoTime) Nanosecond() int {
+	return t.Time().Nanosecond()
+}
+
+func (t UnixNanoTime) YearDay() int {
+	return t.Time().YearDay()
+}
+
+func (t UnixNanoTime) Add(d time.Duration) UnixNanoTime {
+	return UnixNanoTime(t.Time().Add(d))
+}
+
+func (t UnixNanoTime) Sub(u AnyTime) time.Duration {
+	return t.Time().Sub(tt(u))
+}
+
+func (t UnixNanoTime) AddDate(years int, months int, days int) UnixNanoTime {
+	return UnixNanoTime(t.Time().AddDate(years, months, days))
+}
+
+func (t UnixNanoTime) Unix() int64 {
+	return t.Time().Unix()
+}
+
+func (t UnixNanoTime) UnixMilli() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t UnixNanoTime) UnixMicro() int64 {
+	return t.Time().UnixMicro()
+}
+
+func (t UnixNanoTime) UnixNano() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t UnixNanoTime) Format(layout string) string {
+	return t.Time().Format(layout)
+}
+
+func (t UnixNanoTime) GoString() string {
+	return t.Time().GoString()
+}
+
+func (t UnixNanoTime) String() string {
+	return t.Time().String()
+}
+
+func (t UnixNanoTime) Location() *time.Location {
+	return t.Time().Location()
+}
+
+func NewUnixNano(t time.Time) UnixNanoTime {
+	return UnixNanoTime(t)
+}
+
+func NowUnixNano() UnixNanoTime {
+	return UnixNanoTime(time.Now())
+}

sq/database.go

@@ -12,14 +12,15 @@ type DB interface {
 	Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error)
 	Ping(ctx context.Context) error
 	BeginTransaction(ctx context.Context, iso sql.IsolationLevel) (Tx, error)
-	SetListener(listener Listener)
+	AddListener(listener Listener)
+	Exit() error
 }
 
 type database struct {
 	db    *sqlx.DB
 	txctr uint16
 	lock  sync.Mutex
-	lstr  Listener
+	lstr  []Listener
 }
 
 func NewDB(db *sqlx.DB) DB {
@@ -27,31 +28,50 @@ func NewDB(db *sqlx.DB) DB {
 		db:    db,
 		txctr: 0,
 		lock:  sync.Mutex{},
+		lstr:  make([]Listener, 0),
 	}
 }
 
-func (db *database) SetListener(listener Listener) {
+func (db *database) AddListener(listener Listener) {
-	db.lstr = listener
+	db.lstr = append(db.lstr, listener)
 }
 
-func (db *database) Exec(ctx context.Context, sql string, prep PP) (sql.Result, error) {
+func (db *database) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Result, error) {
-	if db.lstr != nil {
+	origsql := sqlstr
-		db.lstr.OnExec(nil, &sql, &prep)
+	for _, v := range db.lstr {
+		err := v.PreExec(ctx, nil, &sqlstr, &prep)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	res, err := db.db.NamedExecContext(ctx, sqlstr, prep)
+
+	for _, v := range db.lstr {
+		v.PostExec(nil, origsql, sqlstr, prep)
 	}
 
-	res, err := db.db.NamedExecContext(ctx, sql, prep)
 	if err != nil {
 		return nil, err
 	}
 	return res, nil
 }
 
-func (db *database) Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error) {
+func (db *database) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx.Rows, error) {
-	if db.lstr != nil {
+	origsql := sqlstr
-		db.lstr.OnQuery(nil, &sql, &prep)
+	for _, v := range db.lstr {
+		err := v.PreQuery(ctx, nil, &sqlstr, &prep)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	rows, err := sqlx.NamedQueryContext(ctx, db.db, sqlstr, prep)
+
+	for _, v := range db.lstr {
+		v.PostQuery(nil, origsql, sqlstr, prep)
 	}
 
-	rows, err := db.db.NamedQueryContext(ctx, sql, prep)
 	if err != nil {
 		return nil, err
 	}
@@ -59,11 +79,19 @@ func (db *database) Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows,
 }
 
 func (db *database) Ping(ctx context.Context) error {
-	if db.lstr != nil {
+	for _, v := range db.lstr {
-		db.lstr.OnPing()
+		err := v.PrePing(ctx)
+		if err != nil {
+			return err
+		}
 	}
 
 	err := db.db.PingContext(ctx)
+
+	for _, v := range db.lstr {
+		v.PostPing(err)
+	}
+
 	if err != nil {
 		return err
 	}
@@ -76,8 +104,11 @@ func (db *database) BeginTransaction(ctx context.Context, iso sql.IsolationLevel
 	db.txctr += 1 // with overflow !
 	db.lock.Unlock()
 
-	if db.lstr != nil {
+	for _, v := range db.lstr {
-		db.lstr.OnTxBegin(txid)
+		err := v.PreTxBegin(ctx, txid)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	xtx, err := db.db.BeginTxx(ctx, &sql.TxOptions{Isolation: iso})
@@ -85,5 +116,13 @@ func (db *database) BeginTransaction(ctx context.Context, iso sql.IsolationLevel
 		return nil, err
 	}
 
+	for _, v := range db.lstr {
+		v.PostTxBegin(txid, err)
+	}
+
 	return NewTransaction(xtx, txid, db.lstr), nil
 }
+
+func (db *database) Exit() error {
+	return db.db.Close()
+}

sq/listener.go

@@ -1,10 +1,19 @@
 package sq
 
+import "context"
+
 type Listener interface {
-	OnQuery(txID *uint16, sql *string, params *PP)
+	PrePing(ctx context.Context) error
-	OnExec(txID *uint16, sql *string, params *PP)
+	PreTxBegin(ctx context.Context, txid uint16) error
-	OnPing()
+	PreTxCommit(txid uint16) error
-	OnTxBegin(txid uint16)
+	PreTxRollback(txid uint16) error
-	OnTxCommit(txid uint16)
+	PreQuery(ctx context.Context, txID *uint16, sql *string, params *PP) error
-	OnTxRollback(txid uint16)
+	PreExec(ctx context.Context, txID *uint16, sql *string, params *PP) error
+
+	PostPing(result error)
+	PostTxBegin(txid uint16, result error)
+	PostTxCommit(txid uint16, result error)
+	PostTxRollback(txid uint16, result error)
+	PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+	PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 }

sq/params.go

@@ -1,3 +1,13 @@
 package sq
 
 type PP map[string]any
+
+func Join(pps ...PP) PP {
+	r := PP{}
+	for _, add := range pps {
+		for k, v := range add {
+			r[k] = v
+		}
+	}
+	return r
+}

sq/scanner.go

@@ -6,24 +6,78 @@ import (
 	"github.com/jmoiron/sqlx"
 )
 
-func ScanSingle[TData any](rows *sqlx.Rows, close bool) (TData, error) {
+type StructScanMode string
+
+const (
+	SModeFast     StructScanMode = "FAST"
+	SModeExtended StructScanMode = "EXTENDED"
+)
+
+type StructScanSafety string
+
+const (
+	Safe   StructScanSafety = "SAFE"
+	Unsafe StructScanSafety = "UNSAFE"
+)
+
+func ScanSingle[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool) (TData, error) {
 	if rows.Next() {
+		var strscan *StructScanner
+
+		if sec == Safe {
+			strscan = NewStructScanner(rows, false)
 			var data TData
-		err := rows.StructScan(&data)
+			err := strscan.Start(&data)
 			if err != nil {
 				return *new(TData), err
 			}
+		} else if sec == Unsafe {
+			strscan = NewStructScanner(rows, true)
+			var data TData
+			err := strscan.Start(&data)
+			if err != nil {
+				return *new(TData), err
+			}
+		} else {
+			return *new(TData), errors.New("unknown value for <sec>")
+		}
+
+		var data TData
+
+		if mode == SModeFast {
+			err := strscan.StructScanBase(&data)
+			if err != nil {
+				return *new(TData), err
+			}
+		} else if mode == SModeExtended {
+			err := strscan.StructScanExt(&data)
+			if err != nil {
+				return *new(TData), err
+			}
+		} else {
+			return *new(TData), errors.New("unknown value for <mode>")
+		}
+
 		if rows.Next() {
-			_ = rows.Close()
-			return *new(TData), errors.New("sql returned more than onw row")
-		}
 			if close {
-			err = rows.Close()
+				_ = rows.Close()
+			}
+			return *new(TData), errors.New("sql returned more than one row")
+		}
+
+		if close {
+			err := rows.Close()
 			if err != nil {
 				return *new(TData), err
 			}
 		}
+
+		if err := rows.Err(); err != nil {
+			return *new(TData), err
+		}
+
 		return data, nil
+
 	} else {
 		if close {
 			_ = rows.Close()
@@ -32,21 +86,55 @@ func ScanSingle[TData any](rows *sqlx.Rows, close bool) (TData, error) {
 	}
 }
 
-func ScanAll[TData any](rows *sqlx.Rows, close bool) ([]TData, error) {
+func ScanAll[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool) ([]TData, error) {
+	var strscan *StructScanner
+
+	if sec == Safe {
+		strscan = NewStructScanner(rows, false)
+		var data TData
+		err := strscan.Start(&data)
+		if err != nil {
+			return nil, err
+		}
+	} else if sec == Unsafe {
+		strscan = NewStructScanner(rows, true)
+		var data TData
+		err := strscan.Start(&data)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, errors.New("unknown value for <sec>")
+	}
+
 	res := make([]TData, 0)
 	for rows.Next() {
+		if mode == SModeFast {
 			var data TData
-		err := rows.StructScan(&data)
+			err := strscan.StructScanBase(&data)
 			if err != nil {
 				return nil, err
 			}
 			res = append(res, data)
+		} else if mode == SModeExtended {
+			var data TData
+			err := strscan.StructScanExt(&data)
+			if err != nil {
+				return nil, err
+			}
+			res = append(res, data)
+		} else {
+			return nil, errors.New("unknown value for <mode>")
+		}
 	}
 	if close {
-		err := rows.Close()
+		err := strscan.rows.Close()
 		if err != nil {
 			return nil, err
 		}
 	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
 	return res, nil
 }

sq/structscanner.go

@@ -0,0 +1,223 @@
+package sq
+
+import (
+	"errors"
+	"fmt"
+	"github.com/jmoiron/sqlx"
+	"github.com/jmoiron/sqlx/reflectx"
+	"reflect"
+)
+
+// forked from sqlx, but added ability to unmarshal optional-nested structs
+
+type StructScanner struct {
+	rows   *sqlx.Rows
+	Mapper *reflectx.Mapper
+	unsafe bool
+
+	fields  [][]int
+	values  []any
+	columns []string
+}
+
+func NewStructScanner(rows *sqlx.Rows, unsafe bool) *StructScanner {
+	return &StructScanner{
+		rows:   rows,
+		Mapper: reflectx.NewMapper("db"),
+		unsafe: unsafe,
+	}
+}
+
+func (r *StructScanner) Start(dest any) error {
+	v := reflect.ValueOf(dest)
+
+	if v.Kind() != reflect.Ptr {
+		return errors.New("must pass a pointer, not a value, to StructScan destination")
+	}
+
+	columns, err := r.rows.Columns()
+	if err != nil {
+		return err
+	}
+
+	r.columns = columns
+	r.fields = r.Mapper.TraversalsByName(v.Type(), columns)
+	// if we are not unsafe and are missing fields, return an error
+	if f, err := missingFields(r.fields); err != nil && !r.unsafe {
+		return fmt.Errorf("missing destination name %s in %T", columns[f], dest)
+	}
+	r.values = make([]interface{}, len(columns))
+
+	return nil
+}
+
+// StructScanExt forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
+// does also wok with nullabel structs (from LEFT JOIN's)
+func (r *StructScanner) StructScanExt(dest any) error {
+	v := reflect.ValueOf(dest)
+
+	if v.Kind() != reflect.Ptr {
+		return errors.New("must pass a pointer, not a value, to StructScan destination")
+	}
+
+	// ========= STEP 1 ::  =========
+
+	v = v.Elem()
+
+	err := fieldsByTraversalExtended(v, r.fields, r.values)
+	if err != nil {
+		return err
+	}
+	// scan into the struct field pointers and append to our results
+	err = r.rows.Scan(r.values...)
+	if err != nil {
+		return err
+	}
+
+	nullStructs := make(map[string]bool)
+
+	for i, traversal := range r.fields {
+		if len(traversal) == 0 {
+			continue
+		}
+
+		isnsil := reflect.ValueOf(r.values[i]).Elem().IsNil()
+
+		for i := 1; i < len(traversal); i++ {
+
+			canParentNil := reflectx.FieldByIndexes(v, traversal[0:i]).Kind() == reflect.Pointer
+
+			k := fmt.Sprintf("%v", traversal[0:i])
+			if v, ok := nullStructs[k]; ok {
+
+				nullStructs[k] = canParentNil && v && isnsil
+
+			} else {
+				nullStructs[k] = canParentNil && isnsil
+			}
+		}
+
+	}
+
+	forcenulled := make(map[string]bool)
+
+	for i, traversal := range r.fields {
+		if len(traversal) == 0 {
+			continue
+		}
+
+		anyparentnull := false
+		for i := 1; i < len(traversal); i++ {
+			k := fmt.Sprintf("%v", traversal[0:i])
+			if nv, ok := nullStructs[k]; ok && nv {
+
+				if _, ok := forcenulled[k]; !ok {
+					f := reflectx.FieldByIndexes(v, traversal[0:i])
+					f.Set(reflect.Zero(f.Type())) // set to nil
+					forcenulled[k] = true
+				}
+
+				anyparentnull = true
+				break
+
+			}
+		}
+
+		if anyparentnull {
+			continue
+		}
+
+		f := reflectx.FieldByIndexes(v, traversal)
+
+		val1 := reflect.ValueOf(r.values[i])
+		val2 := val1.Elem()
+		val3 := val2.Elem()
+
+		if val2.IsNil() {
+			if f.Kind() != reflect.Pointer {
+				return errors.New(fmt.Sprintf("Cannot set field %v to NULL value from column '%s' (type: %s)", traversal, r.columns[i], f.Type().String()))
+			}
+
+			f.Set(reflect.Zero(f.Type())) // set to nil
+		} else {
+			f.Set(val3)
+		}
+
+	}
+
+	return r.rows.Err()
+}
+
+// StructScanBase forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
+// without (relevant) changes
+func (r *StructScanner) StructScanBase(dest any) error {
+	v := reflect.ValueOf(dest)
+
+	if v.Kind() != reflect.Ptr {
+		return errors.New("must pass a pointer, not a value, to StructScan destination")
+	}
+
+	v = v.Elem()
+
+	err := fieldsByTraversalBase(v, r.fields, r.values, true)
+	if err != nil {
+		return err
+	}
+	// scan into the struct field pointers and append to our results
+	err = r.rows.Scan(r.values...)
+	if err != nil {
+		return err
+	}
+	return r.rows.Err()
+}
+
+// fieldsByTraversal forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
+func fieldsByTraversalExtended(v reflect.Value, traversals [][]int, values []interface{}) error {
+	v = reflect.Indirect(v)
+	if v.Kind() != reflect.Struct {
+		return errors.New("argument not a struct")
+	}
+
+	for i, traversal := range traversals {
+		if len(traversal) == 0 {
+			values[i] = new(interface{})
+			continue
+		}
+		f := reflectx.FieldByIndexes(v, traversal)
+
+		values[i] = reflect.New(reflect.PointerTo(f.Type())).Interface()
+	}
+	return nil
+}
+
+// fieldsByTraversal forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
+func fieldsByTraversalBase(v reflect.Value, traversals [][]int, values []interface{}, ptrs bool) error {
+	v = reflect.Indirect(v)
+	if v.Kind() != reflect.Struct {
+		return errors.New("argument not a struct")
+	}
+
+	for i, traversal := range traversals {
+		if len(traversal) == 0 {
+			values[i] = new(interface{})
+			continue
+		}
+		f := reflectx.FieldByIndexes(v, traversal)
+		if ptrs {
+			values[i] = f.Addr().Interface()
+		} else {
+			values[i] = f.Interface()
+		}
+	}
+	return nil
+}
+
+// missingFields forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
+func missingFields(transversals [][]int) (field int, err error) {
+	for i, t := range transversals {
+		if len(t) == 0 {
+			return i, errors.New("missing field")
+		}
+	}
+	return 0, nil
+}

sq/transaction.go

@@ -17,10 +17,10 @@ type Tx interface {
 type transaction struct {
 	tx   *sqlx.Tx
 	id   uint16
-	lstr Listener
+	lstr []Listener
 }
 
-func NewTransaction(xtx *sqlx.Tx, txid uint16, lstr Listener) Tx {
+func NewTransaction(xtx *sqlx.Tx, txid uint16, lstr []Listener) Tx {
 	return &transaction{
 		tx:   xtx,
 		id:   txid,
@@ -29,39 +29,75 @@ func NewTransaction(xtx *sqlx.Tx, txid uint16, lstr Listener) Tx {
 }
 
 func (tx *transaction) Rollback() error {
-	if tx.lstr != nil {
+	for _, v := range tx.lstr {
-		tx.lstr.OnTxRollback(tx.id)
+		err := v.PreTxRollback(tx.id)
+		if err != nil {
+			return err
+		}
 	}
 
-	return tx.tx.Rollback()
+	result := tx.tx.Rollback()
+
+	for _, v := range tx.lstr {
+		v.PostTxRollback(tx.id, result)
+	}
+
+	return result
 }
 
 func (tx *transaction) Commit() error {
-	if tx.lstr != nil {
+	for _, v := range tx.lstr {
-		tx.lstr.OnTxCommit(tx.id)
+		err := v.PreTxCommit(tx.id)
+		if err != nil {
+			return err
+		}
 	}
 
-	return tx.tx.Commit()
+	result := tx.tx.Commit()
+
+	for _, v := range tx.lstr {
+		v.PostTxRollback(tx.id, result)
+	}
+
+	return result
 }
 
-func (tx *transaction) Exec(ctx context.Context, sql string, prep PP) (sql.Result, error) {
+func (tx *transaction) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Result, error) {
-	if tx.lstr != nil {
+	origsql := sqlstr
-		tx.lstr.OnExec(langext.Ptr(tx.id), &sql, &prep)
+	for _, v := range tx.lstr {
+		err := v.PreExec(ctx, langext.Ptr(tx.id), &sqlstr, &prep)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	res, err := tx.tx.NamedExecContext(ctx, sqlstr, prep)
+
+	for _, v := range tx.lstr {
+		v.PostExec(langext.Ptr(tx.id), origsql, sqlstr, prep)
 	}
 
-	res, err := tx.tx.NamedExecContext(ctx, sql, prep)
 	if err != nil {
 		return nil, err
 	}
 	return res, nil
 }
 
-func (tx *transaction) Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error) {
+func (tx *transaction) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx.Rows, error) {
-	if tx.lstr != nil {
+	origsql := sqlstr
-		tx.lstr.OnQuery(langext.Ptr(tx.id), &sql, &prep)
+	for _, v := range tx.lstr {
+		err := v.PreQuery(ctx, langext.Ptr(tx.id), &sqlstr, &prep)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	rows, err := sqlx.NamedQueryContext(ctx, tx.tx, sqlstr, prep)
+
+	for _, v := range tx.lstr {
+		v.PostQuery(langext.Ptr(tx.id), origsql, sqlstr, prep)
 	}
 
-	rows, err := sqlx.NamedQueryContext(ctx, tx.tx, sql, prep)
 	if err != nil {
 		return nil, err
 	}

syncext/atomic.go

@@ -2,58 +2,50 @@ package syncext
 
 import (
 	"context"
-	"sync/atomic"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"sync"
 	"time"
 )
 
 type AtomicBool struct {
-	v      int32
+	v        bool
-	waiter chan bool // unbuffered
+	listener map[string]chan bool
+	lock     sync.Mutex
 }
 
 func NewAtomicBool(value bool) *AtomicBool {
-	if value {
+	return &AtomicBool{
-		return &AtomicBool{v: 1, waiter: make(chan bool)}
+		v:        value,
-	} else {
+		listener: make(map[string]chan bool),
-		return &AtomicBool{v: 0, waiter: make(chan bool)}
+		lock:     sync.Mutex{},
 	}
 }
 
 func (a *AtomicBool) Get() bool {
-	return atomic.LoadInt32(&a.v) == 1
+	a.lock.Lock()
+	defer a.lock.Unlock()
+	return a.v
 }
 
 func (a *AtomicBool) Set(value bool) {
-	if value {
+	a.lock.Lock()
-		atomic.StoreInt32(&a.v, 1)
+	defer a.lock.Unlock()
-	} else {
-		atomic.StoreInt32(&a.v, 0)
-	}
 
+	a.v = value
+
+	for k, v := range a.listener {
 		select {
-	case a.waiter <- value:
+		case v <- value:
 			// message sent
 		default:
 			// no receiver on channel
+			delete(a.listener, k)
+		}
 	}
 }
 
 func (a *AtomicBool) Wait(waitFor bool) {
-	if a.Get() == waitFor {
+	_ = a.WaitWithContext(context.Background(), waitFor)
-		return
-	}
-
-	for {
-		if v, ok := ReadChannelWithTimeout(a.waiter, 128*time.Millisecond); ok {
-			if v == waitFor {
-				return
-			}
-		} else {
-			if a.Get() == waitFor {
-				return
-			}
-		}
-	}
 }
 
 func (a *AtomicBool) WaitWithTimeout(timeout time.Duration, waitFor bool) error {
@@ -71,12 +63,25 @@ func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
 		return nil
 	}
 
+	uuid, _ := langext.NewHexUUID()
+
+	waitchan := make(chan bool)
+
+	a.lock.Lock()
+	a.listener[uuid] = waitchan
+	a.lock.Unlock()
+	defer func() {
+		a.lock.Lock()
+		delete(a.listener, uuid)
+		a.lock.Unlock()
+	}()
+
 	for {
 		if err := ctx.Err(); err != nil {
 			return err
 		}
 
-		timeOut := 128 * time.Millisecond
+		timeOut := 1024 * time.Millisecond
 
 		if dl, ok := ctx.Deadline(); ok {
 			timeOutMax := dl.Sub(time.Now())
@@ -87,7 +92,7 @@ func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
 			}
 		}
 
-		if v, ok := ReadChannelWithTimeout(a.waiter, timeOut); ok {
+		if v, ok := ReadChannelWithTimeout(waitchan, timeOut); ok {
 			if v == waitFor {
 				return nil
 			}