v0.0.88

_data/version.sh

@@ -7,6 +7,20 @@ set -o pipefail  # Return value of a pipeline is the value of the last (rightmos
 IFS=$'\n\t'      # Set $IFS to only newline and tab.
 
 
+function black() { echo -e "\x1B[30m $1 \x1B[0m"; }
+function red()   { echo -e "\x1B[31m $1 \x1B[0m"; }
+function green() { echo -e "\x1B[32m $1 \x1B[0m"; }
+function yellow(){ echo -e "\x1B[33m $1 \x1B[0m"; }
+function blue()  { echo -e "\x1B[34m $1 \x1B[0m"; }
+function purple(){ echo -e "\x1B[35m $1 \x1B[0m"; }
+function cyan()  { echo -e "\x1B[36m $1 \x1B[0m"; }
+function white() { echo -e "\x1B[37m $1 \x1B[0m"; }
+
+if [ "$( git rev-parse --abbrev-ref HEAD )" != "master" ]; then
+  >&2 red "[ERROR] Can only create versions of <master>"
+  exit 1
+fi
+
 curr_vers=$(git describe --tags --abbrev=0 | sed 's/v//g')
 
 next_ver=$(echo "$curr_vers" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{if(length($NF+1)>length($NF))$(NF-1)++; $NF=sprintf("%0*d", length($NF), ($NF+1)%(10^length($NF))); print}')

cmdext/builder.go

@@ -0,0 +1,93 @@
+package cmdext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"time"
+)
+
+type CommandRunner struct {
+	program          string
+	args             []string
+	timeout          *time.Duration
+	env              []string
+	listener         []CommandListener
+	enforceExitCodes *[]int
+	enforceNoTimeout bool
+}
+
+func Runner(program string) *CommandRunner {
+	return &CommandRunner{
+		program:          program,
+		args:             make([]string, 0),
+		timeout:          nil,
+		env:              make([]string, 0),
+		listener:         make([]CommandListener, 0),
+		enforceExitCodes: nil,
+		enforceNoTimeout: false,
+	}
+}
+
+func (r *CommandRunner) Arg(arg string) *CommandRunner {
+	r.args = append(r.args, arg)
+	return r
+}
+
+func (r *CommandRunner) Args(arg []string) *CommandRunner {
+	r.args = append(r.args, arg...)
+	return r
+}
+
+func (r *CommandRunner) Timeout(timeout time.Duration) *CommandRunner {
+	r.timeout = &timeout
+	return r
+}
+
+func (r *CommandRunner) Env(key, value string) *CommandRunner {
+	r.env = append(r.env, fmt.Sprintf("%s=%s", key, value))
+	return r
+}
+
+func (r *CommandRunner) RawEnv(env string) *CommandRunner {
+	r.env = append(r.env, env)
+	return r
+}
+
+func (r *CommandRunner) Envs(env []string) *CommandRunner {
+	r.env = append(r.env, env...)
+	return r
+}
+
+func (r *CommandRunner) EnsureExitcode(arg ...int) *CommandRunner {
+	r.enforceExitCodes = langext.Ptr(langext.ForceArray(arg))
+	return r
+}
+
+func (r *CommandRunner) FailOnExitCode() *CommandRunner {
+	r.enforceExitCodes = langext.Ptr([]int{0})
+	return r
+}
+
+func (r *CommandRunner) FailOnTimeout() *CommandRunner {
+	r.enforceNoTimeout = true
+	return r
+}
+
+func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
+	r.listener = append(r.listener, lstr)
+	return r
+}
+
+func (r *CommandRunner) ListenStdout(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStdoutLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) ListenStderr(lstr func(string)) *CommandRunner {
+	r.listener = append(r.listener, genericCommandListener{_readStderrLine: &lstr})
+	return r
+}
+
+func (r *CommandRunner) Run() (CommandResult, error) {
+	return run(*r)
+}

cmdext/cmdrunner.go

@@ -1,11 +1,17 @@
 package cmdext
 
 import (
-	"bufio"
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
 	"os/exec"
 	"time"
 )
 
+var ErrExitCode = errors.New("process exited with an unexpected exitcode")
+var ErrTimeout = errors.New("process did not exit after the specified timeout")
+
 type CommandResult struct {
 	StdOut          string
 	StdErr          string
@@ -14,9 +20,10 @@ type CommandResult struct {
 	CommandTimedOut bool
 }
 
-func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+func run(opt CommandRunner) (CommandResult, error) {
+	cmd := exec.Command(opt.program, opt.args...)
 
-	cmd := exec.Command(program, args...)
+	cmd.Env = append(cmd.Env, opt.env...)
 
 	stdoutPipe, err := cmd.StdoutPipe()
 	if err != nil {
@@ -28,115 +35,120 @@ func RunCommand(program string, args []string, timeout *time.Duration) (CommandR
 		return CommandResult{}, err
 	}
 
+	preader := pipeReader{
+		lineBufferSize: langext.Ptr(128 * 1024 * 1024), // 128MB max size of a single line, is hopefully enough....
+		stdout:         stdoutPipe,
+		stderr:         stderrPipe,
+	}
+
 	err = cmd.Start()
 	if err != nil {
 		return CommandResult{}, err
 	}
 
-	errch := make(chan error, 1)
+	type resultObj struct {
-	go func() { errch <- cmd.Wait() }()
+		stdout      string
+		stderr      string
+		stdcombined string
+		err         error
+	}
 
-	combch := make(chan string, 32)
+	outputChan := make(chan resultObj)
-	stopCombch := make(chan bool)
-
-	stdout := ""
 	go func() {
-		scanner := bufio.NewScanner(stdoutPipe)
+		// we need to first fully read the pipes and then call Wait
-		for scanner.Scan() {
+		// see https://pkg.go.dev/os/exec#Cmd.StdoutPipe
-			txt := scanner.Text()
-			stdout += txt
-			combch <- txt
-		}
-	}()
 
-	stderr := ""
+		stdout, stderr, stdcombined, err := preader.Read(opt.listener)
-	go func() {
+		if err != nil {
-		scanner := bufio.NewScanner(stderrPipe)
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
-		for scanner.Scan() {
-			txt := scanner.Text()
-			stderr += txt
-			combch <- txt
-		}
-	}()
-
-	defer func() {
-		stopCombch <- true
-	}()
-
-	stdcombined := ""
-	go func() {
-		for {
-			select {
-			case txt := <-combch:
-				stdcombined += txt
-			case <-stopCombch:
-				return
-			}
-		}
-	}()
-
-	if timeout != nil {
-
-		select {
-
-		case <-time.After(*timeout):
 			_ = cmd.Process.Kill()
-			return CommandResult{
+			return
-				StdOut:          stdout,
+		}
-				StdErr:          stderr,
+
-				StdCombined:     stdcombined,
+		err = cmd.Wait()
+		if err != nil {
+			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+		} else {
+			outputChan <- resultObj{stdout, stderr, stdcombined, nil}
+		}
+
+	}()
+
+	var timeoutChan <-chan time.Time = make(chan time.Time, 1)
+	if opt.timeout != nil {
+		timeoutChan = time.After(*opt.timeout)
+	}
+
+	select {
+
+	case <-timeoutChan:
+		_ = cmd.Process.Kill()
+		for _, lstr := range opt.listener {
+			lstr.Timeout()
+		}
+
+		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
+			// most of the time the cmd.Process.Kill() should also ahve finished the pipereader
+			// and we can at least return the already collected stdout, stderr, etc
+			res := CommandResult{
+				StdOut:          fallback.stdout,
+				StdErr:          fallback.stderr,
+				StdCombined:     fallback.stdcombined,
 				ExitCode:        -1,
 				CommandTimedOut: true,
-			}, nil
-
-		case err := <-errch:
-			if exiterr, ok := err.(*exec.ExitError); ok {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        exiterr.ExitCode(),
-					CommandTimedOut: false,
-				}, nil
-			} else if err != nil {
-				return CommandResult{}, err
-			} else {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        0,
-					CommandTimedOut: false,
-				}, nil
 			}
-
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
+		} else {
+			res := CommandResult{
+				StdOut:          "",
+				StdErr:          "",
+				StdCombined:     "",
+				ExitCode:        -1,
+				CommandTimedOut: true,
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
 		}
 
-	} else {
+	case outobj := <-outputChan:
-
+		if exiterr, ok := outobj.err.(*exec.ExitError); ok {
-		select {
+			excode := exiterr.ExitCode()
-
+			for _, lstr := range opt.listener {
-		case err := <-errch:
+				lstr.Finished(excode)
-			if exiterr, ok := err.(*exec.ExitError); ok {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        exiterr.ExitCode(),
-					CommandTimedOut: false,
-				}, nil
-			} else if err != nil {
-				return CommandResult{}, err
-			} else {
-				return CommandResult{
-					StdOut:          stdout,
-					StdErr:          stderr,
-					StdCombined:     stdcombined,
-					ExitCode:        0,
-					CommandTimedOut: false,
-				}, nil
 			}
-
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        excode,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(excode, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
+		} else if err != nil {
+			return CommandResult{}, err
+		} else {
+			for _, lstr := range opt.listener {
+				lstr.Finished(0)
+			}
+			res := CommandResult{
+				StdOut:          outobj.stdout,
+				StdErr:          outobj.stderr,
+				StdCombined:     outobj.stdcombined,
+				ExitCode:        0,
+				CommandTimedOut: false,
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(0, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
 		}
 	}
 }

cmdext/cmdrunner_test.go

@@ -0,0 +1,323 @@
+package cmdext
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestStdout(t *testing.T) {
+
+	res1, err := Runner("printf").Arg("hello").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "hello" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "hello\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStderr(t *testing.T) {
+
+	res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "error" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "error\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestStdcombined(t *testing.T) {
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"1\", file=sys.stderr, flush=True); time.sleep(0.1); print(\"2\", file=sys.stdout, flush=True); time.sleep(0.1); print(\"3\", file=sys.stderr, flush=True)").
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "1\n3\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "2\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "1\n2\n3\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialRead(t *testing.T) {
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", flush=True); time.sleep(5); print(\"cant see me\", flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message\n" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadStderr(t *testing.T) {
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, flush=True); time.sleep(5); print(\"cant see me\", file=sys.stderr, flush=True);").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message\n" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStdout(t *testing.T) {
+
+	res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stdout, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "message101" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestReadUnflushedStderr(t *testing.T) {
+
+	res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stderr, end='')").Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "message101" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "message101\n" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushed(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", end=''); time.sleep(5); print(\"cant see me\", end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "first message" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestPartialReadUnflushedStderr(t *testing.T) {
+	t.SkipNow()
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"first message\", file=sys.stderr, end=''); time.sleep(5); print(\"cant see me\", file=sys.stderr, end='');").
+		Timeout(100 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if !res1.CommandTimedOut {
+		t.Errorf("!CommandTimedOut")
+	}
+	if res1.StdErr != "first message" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if res1.StdOut != "" {
+		t.Errorf("res1.StdOut == '%v'", res1.StdOut)
+	}
+	if res1.StdCombined != "first message" {
+		t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
+	}
+
+}
+
+func TestListener(t *testing.T) {
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys;" +
+			"import time;" +
+			"print(\"message 1\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 2\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 3\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 4\", file=sys.stderr, flush=True);" +
+			"time.sleep(1);" +
+			"print(\"message 5\", flush=True);" +
+			"time.sleep(1);" +
+			"print(\"final\");").
+		ListenStdout(func(s string) { fmt.Printf("@@STDOUT <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		ListenStderr(func(s string) { fmt.Printf("@@STDERR <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
+		Timeout(10 * time.Second).
+		Run()
+
+	if err != nil {
+		panic(err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+}
+
+func TestLongStdout(t *testing.T) {
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"X\" * 125001 + \"\\n\"); print(\"Y\" * 125001 + \"\\n\"); print(\"Z\" * 125001 + \"\\n\");").
+		Timeout(5000 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if len(res1.StdOut) != 375009 {
+		t.Errorf("len(res1.StdOut) == '%v'", len(res1.StdOut))
+	}
+
+}
+
+func TestFailOnTimeout(t *testing.T) {
+
+	_, err := Runner("sleep").Arg("2").Timeout(200 * time.Millisecond).FailOnTimeout().Run()
+	if err != ErrTimeout {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestFailOnExitcode(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).FailOnExitCode().Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode1(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).EnsureExitcode(1).Run()
+	if err != nil {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode2(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200*time.Millisecond).EnsureExitcode(0, 2, 3).Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}

cmdext/helper.go

@@ -0,0 +1,12 @@
+package cmdext
+
+import "time"
+
+func RunCommand(program string, args []string, timeout *time.Duration) (CommandResult, error) {
+	b := Runner(program)
+	b = b.Args(args)
+	if timeout != nil {
+		b = b.Timeout(*timeout)
+	}
+	return b.Run()
+}

cmdext/listener.go

@@ -0,0 +1,57 @@
+package cmdext
+
+type CommandListener interface {
+	ReadRawStdout([]byte)
+	ReadRawStderr([]byte)
+
+	ReadStdoutLine(string)
+	ReadStderrLine(string)
+
+	Finished(int)
+	Timeout()
+}
+
+type genericCommandListener struct {
+	_readRawStdout  *func([]byte)
+	_readRawStderr  *func([]byte)
+	_readStdoutLine *func(string)
+	_readStderrLine *func(string)
+	_finished       *func(int)
+	_timeout        *func()
+}
+
+func (g genericCommandListener) ReadRawStdout(v []byte) {
+	if g._readRawStdout != nil {
+		(*g._readRawStdout)(v)
+	}
+}
+
+func (g genericCommandListener) ReadRawStderr(v []byte) {
+	if g._readRawStderr != nil {
+		(*g._readRawStderr)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStdoutLine(v string) {
+	if g._readStdoutLine != nil {
+		(*g._readStdoutLine)(v)
+	}
+}
+
+func (g genericCommandListener) ReadStderrLine(v string) {
+	if g._readStderrLine != nil {
+		(*g._readStderrLine)(v)
+	}
+}
+
+func (g genericCommandListener) Finished(v int) {
+	if g._finished != nil {
+		(*g._finished)(v)
+	}
+}
+
+func (g genericCommandListener) Timeout() {
+	if g._timeout != nil {
+		(*g._timeout)()
+	}
+}

cmdext/pipereader.go

@@ -0,0 +1,158 @@
+package cmdext
+
+import (
+	"bufio"
+	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
+	"io"
+	"sync"
+)
+
+type pipeReader struct {
+	lineBufferSize *int
+	stdout         io.ReadCloser
+	stderr         io.ReadCloser
+}
+
+// Read ready stdout and stdin until finished
+// also splits both pipes into lines and calld the listener
+func (pr *pipeReader) Read(listener []CommandListener) (string, string, string, error) {
+	type combevt struct {
+		line string
+		stop bool
+	}
+
+	errch := make(chan error, 8)
+
+	wg := sync.WaitGroup{}
+
+	// [1] read raw stdout
+
+	wg.Add(1)
+	stdoutBufferReader, stdoutBufferWriter := io.Pipe()
+	stdout := ""
+	go func() {
+		buf := make([]byte, 128)
+		for true {
+			n, out := pr.stdout.Read(buf)
+			if n > 0 {
+				txt := string(buf[:n])
+				stdout += txt
+				_, _ = stdoutBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStdout(buf[:n])
+				}
+			}
+			if out == io.EOF {
+				break
+			}
+			if out != nil {
+				errch <- out
+				break
+			}
+		}
+		_ = stdoutBufferWriter.Close()
+		wg.Done()
+	}()
+
+	// [2] read raw stderr
+
+	wg.Add(1)
+	stderrBufferReader, stderrBufferWriter := io.Pipe()
+	stderr := ""
+	go func() {
+		buf := make([]byte, 128)
+		for true {
+			n, err := pr.stderr.Read(buf)
+
+			if n > 0 {
+				txt := string(buf[:n])
+				stderr += txt
+				_, _ = stderrBufferWriter.Write(buf[:n])
+				for _, lstr := range listener {
+					lstr.ReadRawStderr(buf[:n])
+				}
+			}
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				errch <- err
+				break
+			}
+		}
+		_ = stderrBufferWriter.Close()
+		wg.Done()
+	}()
+
+	combch := make(chan combevt, 32)
+
+	// [3] collect stdout line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stdoutBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStdoutLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [4] collect stderr line-by-line
+
+	wg.Add(1)
+	go func() {
+		scanner := bufio.NewScanner(stderrBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			for _, lstr := range listener {
+				lstr.ReadStderrLine(txt)
+			}
+			combch <- combevt{txt, false}
+		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
+		combch <- combevt{"", true}
+		wg.Done()
+	}()
+
+	// [5] combine stdcombined
+
+	wg.Add(1)
+	stdcombined := ""
+	go func() {
+		stopctr := 0
+		for stopctr < 2 {
+			vvv := <-combch
+			if vvv.stop {
+				stopctr++
+			} else {
+				stdcombined += vvv.line + "\n" // this comes from bufio.Scanner and has no newlines...
+			}
+		}
+		wg.Done()
+	}()
+
+	// wait for all (5) goroutines to finish
+	wg.Wait()
+
+	if err, ok := syncext.ReadNonBlocking(errch); ok {
+		return "", "", "", err
+	}
+
+	return stdout, stderr, stdcombined, nil
+}

cryptext/aes.go

@@ -1,10 +1,13 @@
 package cryptext
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
-	"encoding/base64"
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
 	"errors"
 	"golang.org/x/crypto/scrypt"
 	"io"
@@ -12,35 +15,90 @@ import (
 
 // https://stackoverflow.com/a/18819040/1761622
 
-func EncryptAESSimple(password, text []byte) ([]byte, error) {
+type aesPayload struct {
-
+	Salt    []byte `json:"s"`
-	key, err := scrypt.Key(password, nil, 32768, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
+	IV      []byte `json:"i"`
-	if err != nil {
+	Data    []byte `json:"d"`
-		return nil, err
+	Rounds  int    `json:"r"`
-	}
+	Version uint   `json:"v"`
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return nil, err
-	}
-
-	b := base64.StdEncoding.EncodeToString(text)
-	ciphertext := make([]byte, aes.BlockSize+len(b))
-
-	iv := ciphertext[:aes.BlockSize]
-	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
-		return nil, err
-	}
-
-	cfb := cipher.NewCFBEncrypter(block, iv)
-	cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
-
-	return ciphertext, nil
 }
 
-func DecryptAESSimple(password, text []byte) ([]byte, error) {
+func EncryptAESSimple(password []byte, data []byte, rounds int) (string, error) {
 
-	key, err := scrypt.Key(password, nil, 32768, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
+	salt := make([]byte, 8)
+	_, err := io.ReadFull(rand.Reader, salt)
+	if err != nil {
+		return "", err
+	}
+
+	key, err := scrypt.Key(password, salt, rounds, 8, 1, 32)
+	if err != nil {
+		return "", err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	h := sha256.New()
+	h.Write(data)
+	checksum := h.Sum(nil)
+	if len(checksum) != 32 {
+		return "", errors.New("wrong cs size")
+	}
+
+	ciphertext := make([]byte, 32+len(data))
+
+	iv := make([]byte, aes.BlockSize)
+	_, err = io.ReadFull(rand.Reader, iv)
+	if err != nil {
+		return "", err
+	}
+
+	combinedData := make([]byte, 0, 32+len(data))
+	combinedData = append(combinedData, checksum...)
+	combinedData = append(combinedData, data...)
+
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext, combinedData)
+
+	pl := aesPayload{
+		Salt:    salt,
+		IV:      iv,
+		Data:    ciphertext,
+		Version: 1,
+		Rounds:  rounds,
+	}
+
+	jbin, err := json.Marshal(pl)
+	if err != nil {
+		return "", err
+	}
+
+	res := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(jbin)
+
+	return res, nil
+}
+
+func DecryptAESSimple(password []byte, encText string) ([]byte, error) {
+
+	jbin, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(encText)
+	if err != nil {
+		return nil, err
+	}
+
+	var pl aesPayload
+	err = json.Unmarshal(jbin, &pl)
+	if err != nil {
+		return nil, err
+	}
+
+	if pl.Version != 1 {
+		return nil, errors.New("unsupported version")
+	}
+
+	key, err := scrypt.Key(password, pl.Salt, pl.Rounds, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
 	if err != nil {
 		return nil, err
 	}
@@ -50,18 +108,24 @@ func DecryptAESSimple(password, text []byte) ([]byte, error) {
 		return nil, err
 	}
 
-	if len(text) < aes.BlockSize {
+	dest := make([]byte, len(pl.Data))
-		return nil, errors.New("ciphertext too short")
+
+	cfb := cipher.NewCFBDecrypter(block, pl.IV)
+	cfb.XORKeyStream(dest, pl.Data)
+
+	if len(dest) < 32 {
+		return nil, errors.New("payload too small")
 	}
 
-	iv := text[:aes.BlockSize]
+	chck := dest[:32]
-	text = text[aes.BlockSize:]
+	data := dest[32:]
-	cfb := cipher.NewCFBDecrypter(block, iv)
-	cfb.XORKeyStream(text, text)
 
-	data, err := base64.StdEncoding.DecodeString(string(text))
+	h := sha256.New()
-	if err != nil {
+	h.Write(data)
-		return nil, err
+	chck2 := h.Sum(nil)
+
+	if !bytes.Equal(chck, chck2) {
+		return nil, errors.New("checksum mismatch")
 	}
 
 	return data, nil

cryptext/aes_test.go

@@ -1,6 +1,9 @@
 package cryptext
 
-import "testing"
+import (
+	"fmt"
+	"testing"
+)
 
 func TestEncryptAESSimple(t *testing.T) {
 
@@ -8,15 +11,25 @@ func TestEncryptAESSimple(t *testing.T) {
 
 	str1 := []byte("Hello World")
 
-	str2, err := EncryptAESSimple(pw, str1)
+	str2, err := EncryptAESSimple(pw, str1, 512)
 	if err != nil {
 		panic(err)
 	}
 
+	fmt.Printf("%s\n", str2)
+
 	str3, err := DecryptAESSimple(pw, str2)
 	if err != nil {
 		panic(err)
 	}
 
 	assertEqual(t, string(str1), string(str3))
+
+	str4, err := EncryptAESSimple(pw, str3, 512)
+	if err != nil {
+		panic(err)
+	}
+
+	assertNotEqual(t, string(str2), string(str4))
+
 }

cryptext/hash_test.go

@@ -23,3 +23,9 @@ func assertEqual(t *testing.T, actual string, expected string) {
 		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
 	}
 }
+
+func assertNotEqual(t *testing.T, actual string, expected string) {
+	if actual == expected {
+		t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

dataext/lruMap_test.go

@@ -12,7 +12,7 @@ func init() {
 }
 
 func TestResultCache1(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key := randomKey()
@@ -50,7 +50,7 @@ func TestResultCache1(t *testing.T) {
 }
 
 func TestResultCache2(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -150,7 +150,7 @@ func TestResultCache2(t *testing.T) {
 }
 
 func TestResultCache3(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -173,7 +173,7 @@ func TestResultCache3(t *testing.T) {
 }
 
 // does a basic consistency check over the internal cache representation
-func verifyLRUList[TData any](cache *LRUMap[TData], t *testing.T) {
+func verifyLRUList[TKey comparable, TData any](cache *LRUMap[TKey, TData], t *testing.T) {
 	size := 0
 
 	tailFound := false

go.mod

@@ -9,5 +9,6 @@ require (
 
 require (
 	github.com/jmoiron/sqlx v1.3.5 // indirect
+	go.mongodb.org/mongo-driver v1.11.1 // indirect
 	golang.org/x/crypto v0.4.0 // indirect
 )

go.sum

@@ -1,15 +1,50 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
+go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
 golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
 golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

langext/array.go

@@ -265,6 +265,36 @@ func ArrMap[T1 any, T2 any](arr []T1, conv func(v T1) T2) []T2 {
 	return r
 }
 
+func ArrMapExt[T1 any, T2 any](arr []T1, conv func(idx int, v T1) T2) []T2 {
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i] = conv(i, v)
+	}
+	return r
+}
+
+func ArrMapErr[T1 any, T2 any](arr []T1, conv func(v T1) (T2, error)) ([]T2, error) {
+	var err error
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i], err = conv(v)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return r, nil
+}
+
+func ArrFilterMap[T1 any, T2 any](arr []T1, filter func(v T1) bool, conv func(v T1) T2) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v := range arr {
+		if filter(v) {
+			r = append(r, conv(v))
+		}
+	}
+	return r
+}
+
 func ArrSum[T NumberConstraint](arr []T) T {
 	var r T = 0
 	for _, v := range arr {
@@ -272,3 +302,19 @@ func ArrSum[T NumberConstraint](arr []T) T {
 	}
 	return r
 }
+
+func ArrFlatten[T1 any, T2 any](arr []T1, conv func(v T1) []T2) []T2 {
+	r := make([]T2, 0, len(arr))
+	for _, v1 := range arr {
+		r = append(r, conv(v1)...)
+	}
+	return r
+}
+
+func ArrFlattenDirect[T1 any](arr [][]T1) []T1 {
+	r := make([]T1, 0, len(arr))
+	for _, v1 := range arr {
+		r = append(r, v1...)
+	}
+	return r
+}

langext/maps.go

@@ -7,3 +7,11 @@ func MapKeyArr[T comparable, V any](v map[T]V) []T {
 	}
 	return result
 }
+
+func ArrToMap[T comparable, V any](a []V, keyfunc func(V) T) map[T]V {
+	result := make(map[T]V, len(a))
+	for _, v := range a {
+		result[keyfunc(v)] = v
+	}
+	return result
+}

rfctime/interface.go

@@ -43,3 +43,45 @@ type RFCTime interface {
 	GoString() string
 	String() string
 }
+
+type RFCDuration interface {
+	Time() time.Time
+	Serialize() string
+
+	UnmarshalJSON(bytes []byte) error
+	MarshalJSON() ([]byte, error)
+
+	MarshalBinary() ([]byte, error)
+	UnmarshalBinary(data []byte) error
+
+	GobEncode() ([]byte, error)
+	GobDecode(data []byte) error
+
+	MarshalText() ([]byte, error)
+	UnmarshalText(data []byte) error
+
+	After(u RFCTime) bool
+	Before(u RFCTime) bool
+	Equal(u RFCTime) bool
+	IsZero() bool
+	Date() (year int, month time.Month, day int)
+	Year() int
+	Month() time.Month
+	Day() int
+	Weekday() time.Weekday
+	ISOWeek() (year, week int)
+	Clock() (hour, min, sec int)
+	Hour() int
+	Minute() int
+	Second() int
+	Nanosecond() int
+	YearDay() int
+	Sub(u RFCTime) time.Duration
+	Unix() int64
+	UnixMilli() int64
+	UnixMicro() int64
+	UnixNano() int64
+	Format(layout string) string
+	GoString() string
+	String() string
+}

rfctime/rfc3339.go

@@ -0,0 +1,203 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"time"
+)
+
+type RFC3339Time time.Time
+
+func (t RFC3339Time) Time() time.Time {
+	return time.Time(t)
+}
+
+func (t RFC3339Time) MarshalBinary() ([]byte, error) {
+	return (time.Time)(t).MarshalBinary()
+}
+
+func (t *RFC3339Time) UnmarshalBinary(data []byte) error {
+	return (*time.Time)(t).UnmarshalBinary(data)
+}
+
+func (t RFC3339Time) GobEncode() ([]byte, error) {
+	return (time.Time)(t).GobEncode()
+}
+
+func (t *RFC3339Time) GobDecode(data []byte) error {
+	return (*time.Time)(t).GobDecode(data)
+}
+
+func (t *RFC3339Time) UnmarshalJSON(data []byte) error {
+	str := ""
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+	t0, err := time.Parse(t.FormatStr(), str)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339Time(t0)
+	return nil
+}
+
+func (t RFC3339Time) MarshalJSON() ([]byte, error) {
+	str := t.Time().Format(t.FormatStr())
+	return json.Marshal(str)
+}
+
+func (t RFC3339Time) MarshalText() ([]byte, error) {
+	b := make([]byte, 0, len(t.FormatStr()))
+	return t.Time().AppendFormat(b, t.FormatStr()), nil
+}
+
+func (t *RFC3339Time) UnmarshalText(data []byte) error {
+	var err error
+	v, err := time.Parse(t.FormatStr(), string(data))
+	if err != nil {
+		return err
+	}
+	tt := RFC3339Time(v)
+	*t = tt
+	return nil
+}
+
+func (t *RFC3339Time) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt != bsontype.DateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339Time", bt))
+	}
+	var tt time.Time
+	err := bson.Unmarshal(data, &tt)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339Time(tt)
+	return nil
+}
+
+func (t RFC3339Time) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
+func (t RFC3339Time) Serialize() string {
+	return t.Time().Format(t.FormatStr())
+}
+
+func (t RFC3339Time) FormatStr() string {
+	return time.RFC3339
+}
+
+func (t RFC3339Time) After(u RFCTime) bool {
+	return t.Time().After(u.Time())
+}
+
+func (t RFC3339Time) Before(u RFCTime) bool {
+	return t.Time().Before(u.Time())
+}
+
+func (t RFC3339Time) Equal(u RFCTime) bool {
+	return t.Time().Equal(u.Time())
+}
+
+func (t RFC3339Time) IsZero() bool {
+	return t.Time().IsZero()
+}
+
+func (t RFC3339Time) Date() (year int, month time.Month, day int) {
+	return t.Time().Date()
+}
+
+func (t RFC3339Time) Year() int {
+	return t.Time().Year()
+}
+
+func (t RFC3339Time) Month() time.Month {
+	return t.Time().Month()
+}
+
+func (t RFC3339Time) Day() int {
+	return t.Time().Day()
+}
+
+func (t RFC3339Time) Weekday() time.Weekday {
+	return t.Time().Weekday()
+}
+
+func (t RFC3339Time) ISOWeek() (year, week int) {
+	return t.Time().ISOWeek()
+}
+
+func (t RFC3339Time) Clock() (hour, min, sec int) {
+	return t.Time().Clock()
+}
+
+func (t RFC3339Time) Hour() int {
+	return t.Time().Hour()
+}
+
+func (t RFC3339Time) Minute() int {
+	return t.Time().Minute()
+}
+
+func (t RFC3339Time) Second() int {
+	return t.Time().Second()
+}
+
+func (t RFC3339Time) Nanosecond() int {
+	return t.Time().Nanosecond()
+}
+
+func (t RFC3339Time) YearDay() int {
+	return t.Time().YearDay()
+}
+
+func (t RFC3339Time) Add(d time.Duration) RFC3339Time {
+	return RFC3339Time(t.Time().Add(d))
+}
+
+func (t RFC3339Time) Sub(u RFCTime) time.Duration {
+	return t.Time().Sub(u.Time())
+}
+
+func (t RFC3339Time) AddDate(years int, months int, days int) RFC3339Time {
+	return RFC3339Time(t.Time().AddDate(years, months, days))
+}
+
+func (t RFC3339Time) Unix() int64 {
+	return t.Time().Unix()
+}
+
+func (t RFC3339Time) UnixMilli() int64 {
+	return t.Time().UnixMilli()
+}
+
+func (t RFC3339Time) UnixMicro() int64 {
+	return t.Time().UnixMicro()
+}
+
+func (t RFC3339Time) UnixNano() int64 {
+	return t.Time().UnixNano()
+}
+
+func (t RFC3339Time) Format(layout string) string {
+	return t.Time().Format(layout)
+}
+
+func (t RFC3339Time) GoString() string {
+	return t.Time().GoString()
+}
+
+func (t RFC3339Time) String() string {
+	return t.Time().String()
+}
+
+func NewRFC3339(t time.Time) RFC3339Time {
+	return RFC3339Time(t)
+}
+
+func NowRFC3339() RFC3339Time {
+	return RFC3339Time(time.Now())
+}

rfctime/rfc3339Nano.go

@@ -2,6 +2,10 @@ package rfctime
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
 	"time"
 )
 
@@ -61,6 +65,23 @@ func (t *RFC3339NanoTime) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (t *RFC3339NanoTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
+	if bt != bsontype.DateTime {
+		return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339NanoTime", bt))
+	}
+	var tt time.Time
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
+	if err != nil {
+		return err
+	}
+	*t = RFC3339NanoTime(tt)
+	return nil
+}
+
+func (t RFC3339NanoTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
+	return bson.MarshalValue(time.Time(t))
+}
+
 func (t RFC3339NanoTime) Serialize() string {
 	return t.Time().Format(t.FormatStr())
 }

rfctime/rfc3339Nano_test.go

@@ -12,7 +12,7 @@ func TestRoundtrip(t *testing.T) {
 		Value RFC3339NanoTime `json:"v"`
 	}
 
-	val1 := NewRFC3339Nano(time.Now())
+	val1 := NewRFC3339Nano(time.Unix(0, 1675951556820915171))
 	w1 := Wrap{val1}
 
 	jstr1, err := json.Marshal(w1)
@@ -20,7 +20,8 @@ func TestRoundtrip(t *testing.T) {
 		panic(err)
 	}
 
-	if string(jstr1) != "{\"v\":\"2023-01-29T20:32:36.149692117+01:00\"}" {
+	if string(jstr1) != "{\"v\":\"2023-02-09T15:05:56.820915171+01:00\"}" {
+		t.Errorf(string(jstr1))
 		t.Errorf("repr differs")
 	}
 

rfctime/seconds.go

@@ -0,0 +1,59 @@
+package rfctime
+
+import (
+	"encoding/json"
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"time"
+)
+
+type SecondsF64 time.Duration
+
+func (d SecondsF64) Duration() time.Duration {
+	return time.Duration(d)
+}
+
+func (d SecondsF64) String() string {
+	return d.Duration().String()
+}
+
+func (d SecondsF64) Nanoseconds() int64 {
+	return d.Duration().Nanoseconds()
+}
+
+func (d SecondsF64) Microseconds() int64 {
+	return d.Duration().Microseconds()
+}
+
+func (d SecondsF64) Milliseconds() int64 {
+	return d.Duration().Milliseconds()
+}
+
+func (d SecondsF64) Seconds() float64 {
+	return d.Duration().Seconds()
+}
+
+func (d SecondsF64) Minutes() float64 {
+	return d.Duration().Minutes()
+}
+
+func (d SecondsF64) Hours() float64 {
+	return d.Duration().Hours()
+}
+
+func (d *SecondsF64) UnmarshalJSON(data []byte) error {
+	var secs float64 = 0
+	if err := json.Unmarshal(data, &secs); err != nil {
+		return err
+	}
+	*d = SecondsF64(timeext.FromSeconds(secs))
+	return nil
+}
+
+func (d SecondsF64) MarshalJSON() ([]byte, error) {
+	secs := d.Seconds()
+	return json.Marshal(secs)
+}
+
+func NewSecondsF64(t time.Duration) SecondsF64 {
+	return SecondsF64(t)
+}

syncext/atomic.go

@@ -2,58 +2,50 @@ package syncext
 
 import (
 	"context"
-	"sync/atomic"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"sync"
 	"time"
 )
 
 type AtomicBool struct {
-	v      int32
+	v        bool
-	waiter chan bool // unbuffered
+	listener map[string]chan bool
+	lock     sync.Mutex
 }
 
 func NewAtomicBool(value bool) *AtomicBool {
-	if value {
+	return &AtomicBool{
-		return &AtomicBool{v: 1, waiter: make(chan bool)}
+		v:        value,
-	} else {
+		listener: make(map[string]chan bool),
-		return &AtomicBool{v: 0, waiter: make(chan bool)}
+		lock:     sync.Mutex{},
 	}
 }
 
 func (a *AtomicBool) Get() bool {
-	return atomic.LoadInt32(&a.v) == 1
+	a.lock.Lock()
+	defer a.lock.Unlock()
+	return a.v
 }
 
 func (a *AtomicBool) Set(value bool) {
-	if value {
+	a.lock.Lock()
-		atomic.StoreInt32(&a.v, 1)
+	defer a.lock.Unlock()
-	} else {
-		atomic.StoreInt32(&a.v, 0)
-	}
 
-	select {
+	a.v = value
-	case a.waiter <- value:
+
-		// message sent
+	for k, v := range a.listener {
-	default:
+		select {
-		// no receiver on channel
+		case v <- value:
+			// message sent
+		default:
+			// no receiver on channel
+			delete(a.listener, k)
+		}
 	}
 }
 
 func (a *AtomicBool) Wait(waitFor bool) {
-	if a.Get() == waitFor {
+	_ = a.WaitWithContext(context.Background(), waitFor)
-		return
-	}
-
-	for {
-		if v, ok := ReadChannelWithTimeout(a.waiter, 128*time.Millisecond); ok {
-			if v == waitFor {
-				return
-			}
-		} else {
-			if a.Get() == waitFor {
-				return
-			}
-		}
-	}
 }
 
 func (a *AtomicBool) WaitWithTimeout(timeout time.Duration, waitFor bool) error {
@@ -71,12 +63,25 @@ func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
 		return nil
 	}
 
+	uuid, _ := langext.NewHexUUID()
+
+	waitchan := make(chan bool)
+
+	a.lock.Lock()
+	a.listener[uuid] = waitchan
+	a.lock.Unlock()
+	defer func() {
+		a.lock.Lock()
+		delete(a.listener, uuid)
+		a.lock.Unlock()
+	}()
+
 	for {
 		if err := ctx.Err(); err != nil {
 			return err
 		}
 
-		timeOut := 128 * time.Millisecond
+		timeOut := 1024 * time.Millisecond
 
 		if dl, ok := ctx.Deadline(); ok {
 			timeOutMax := dl.Sub(time.Now())
@@ -87,7 +92,7 @@ func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
 			}
 		}
 
-		if v, ok := ReadChannelWithTimeout(a.waiter, timeOut); ok {
+		if v, ok := ReadChannelWithTimeout(waitchan, timeOut); ok {
 			if v == waitFor {
 				return nil
 			}