v0.0.85

cmdext/builder.go

@@ -2,6 +2,7 @@ package cmdext
 
 import (
 	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"time"
 )
 
@@ -11,6 +12,8 @@ type CommandRunner struct {
 	timeout          *time.Duration
 	env              []string
 	listener         []CommandListener
+	enforceExitCodes *[]int
+	enforceNoTimeout bool
 }
 
 func Runner(program string) *CommandRunner {
@@ -20,6 +23,8 @@ func Runner(program string) *CommandRunner {
 		timeout:          nil,
 		env:              make([]string, 0),
 		listener:         make([]CommandListener, 0),
+		enforceExitCodes: nil,
+		enforceNoTimeout: false,
 	}
 }
 
@@ -53,6 +58,21 @@ func (r *CommandRunner) Envs(env []string) *CommandRunner {
 	return r
 }
 
+func (r *CommandRunner) EnsureExitcode(arg ...int) *CommandRunner {
+	r.enforceExitCodes = langext.Ptr(langext.ForceArray(arg))
+	return r
+}
+
+func (r *CommandRunner) FailOnExitCode() *CommandRunner {
+	r.enforceExitCodes = langext.Ptr([]int{0})
+	return r
+}
+
+func (r *CommandRunner) FailOnTimeout() *CommandRunner {
+	r.enforceNoTimeout = true
+	return r
+}
+
 func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
 	r.listener = append(r.listener, lstr)
 	return r

cmdext/cmdrunner.go

@@ -1,12 +1,17 @@
 package cmdext
 
 import (
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/mathext"
 	"gogs.mikescher.com/BlackForestBytes/goext/syncext"
 	"os/exec"
 	"time"
 )
 
+var ErrExitCode = errors.New("process exited with an unexpected exitcode")
+var ErrTimeout = errors.New("process did not exit after the specified timeout")
+
 type CommandResult struct {
 	StdOut          string
 	StdErr          string
@@ -31,6 +36,7 @@ func run(opt CommandRunner) (CommandResult, error) {
 	}
 
 	preader := pipeReader{
+		lineBufferSize: langext.Ptr(128 * 1024 * 1024), // 128MB max size of a single line, is hopefully enough....
 		stdout:         stdoutPipe,
 		stderr:         stderrPipe,
 	}
@@ -55,14 +61,17 @@ func run(opt CommandRunner) (CommandResult, error) {
 		stdout, stderr, stdcombined, err := preader.Read(opt.listener)
 		if err != nil {
 			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+			_ = cmd.Process.Kill()
+			return
 		}
 
 		err = cmd.Wait()
 		if err != nil {
 			outputChan <- resultObj{stdout, stderr, stdcombined, err}
+		} else {
+			outputChan <- resultObj{stdout, stderr, stdcombined, nil}
 		}
 
-		outputChan <- resultObj{stdout, stderr, stdcombined, nil}
 	}()
 
 	var timeoutChan <-chan time.Time = make(chan time.Time, 1)
@@ -81,21 +90,29 @@ func run(opt CommandRunner) (CommandResult, error) {
 		if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
 			// most of the time the cmd.Process.Kill() should also ahve finished the pipereader
 			// and we can at least return the already collected stdout, stderr, etc
-			return CommandResult{
+			res := CommandResult{
 				StdOut:          fallback.stdout,
 				StdErr:          fallback.stderr,
 				StdCombined:     fallback.stdcombined,
 				ExitCode:        -1,
 				CommandTimedOut: true,
-			}, nil
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
 		} else {
-			return CommandResult{
+			res := CommandResult{
 				StdOut:          "",
 				StdErr:          "",
 				StdCombined:     "",
 				ExitCode:        -1,
 				CommandTimedOut: true,
-			}, nil
+			}
+			if opt.enforceNoTimeout {
+				return res, ErrTimeout
+			}
+			return res, nil
 		}
 
 	case outobj := <-outputChan:
@@ -104,26 +121,34 @@ func run(opt CommandRunner) (CommandResult, error) {
 			for _, lstr := range opt.listener {
 				lstr.Finished(excode)
 			}
-			return CommandResult{
+			res := CommandResult{
 				StdOut:          outobj.stdout,
 				StdErr:          outobj.stderr,
 				StdCombined:     outobj.stdcombined,
 				ExitCode:        excode,
 				CommandTimedOut: false,
-			}, nil
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(excode, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
 		} else if err != nil {
 			return CommandResult{}, err
 		} else {
 			for _, lstr := range opt.listener {
 				lstr.Finished(0)
 			}
-			return CommandResult{
+			res := CommandResult{
 				StdOut:          outobj.stdout,
 				StdErr:          outobj.stderr,
 				StdCombined:     outobj.stdcombined,
 				ExitCode:        0,
 				CommandTimedOut: false,
-			}, nil
+			}
+			if opt.enforceExitCodes != nil && !langext.InArray(0, *opt.enforceExitCodes) {
+				return res, ErrExitCode
+			}
+			return res, nil
 		}
 	}
 }

cmdext/cmdrunner_test.go

@@ -12,6 +12,12 @@ func TestStdout(t *testing.T) {
 	if err != nil {
 		t.Errorf("%v", err)
 	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
 	if res1.StdErr != "" {
 		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
 	}
@@ -30,6 +36,12 @@ func TestStderr(t *testing.T) {
 	if err != nil {
 		t.Errorf("%v", err)
 	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
 	if res1.StdErr != "error" {
 		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
 	}
@@ -50,6 +62,12 @@ func TestStdcombined(t *testing.T) {
 	if err != nil {
 		t.Errorf("%v", err)
 	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
 	if res1.StdErr != "1\n3\n" {
 		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
 	}
@@ -116,6 +134,12 @@ func TestReadUnflushedStdout(t *testing.T) {
 	if err != nil {
 		t.Errorf("%v", err)
 	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
 	if res1.StdErr != "" {
 		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
 	}
@@ -134,6 +158,12 @@ func TestReadUnflushedStderr(t *testing.T) {
 	if err != nil {
 		t.Errorf("%v", err)
 	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
 	if res1.StdErr != "message101" {
 		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
 	}
@@ -200,7 +230,7 @@ func TestPartialReadUnflushedStderr(t *testing.T) {
 
 func TestListener(t *testing.T) {
 
-	_, err := Runner("python").
+	res1, err := Runner("python").
 		Arg("-c").
 		Arg("import sys;" +
 			"import time;" +
@@ -223,4 +253,71 @@ func TestListener(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+}
+
+func TestLongStdout(t *testing.T) {
+
+	res1, err := Runner("python").
+		Arg("-c").
+		Arg("import sys; import time; print(\"X\" * 125001 + \"\\n\"); print(\"Y\" * 125001 + \"\\n\"); print(\"Z\" * 125001 + \"\\n\");").
+		Timeout(5000 * time.Millisecond).
+		Run()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	if res1.CommandTimedOut {
+		t.Errorf("Timeout")
+	}
+	if res1.ExitCode != 0 {
+		t.Errorf("res1.ExitCode == %v", res1.ExitCode)
+	}
+	if res1.StdErr != "" {
+		t.Errorf("res1.StdErr == '%v'", res1.StdErr)
+	}
+	if len(res1.StdOut) != 375009 {
+		t.Errorf("len(res1.StdOut) == '%v'", len(res1.StdOut))
+	}
+
+}
+
+func TestFailOnTimeout(t *testing.T) {
+
+	_, err := Runner("sleep").Arg("2").Timeout(200 * time.Millisecond).FailOnTimeout().Run()
+	if err != ErrTimeout {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestFailOnExitcode(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).FailOnExitCode().Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode1(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200 * time.Millisecond).EnsureExitcode(1).Run()
+	if err != nil {
+		t.Errorf("wrong err := %v", err)
+	}
+
+}
+
+func TestEnsureExitcode2(t *testing.T) {
+
+	_, err := Runner("false").Timeout(200*time.Millisecond).EnsureExitcode(0, 2, 3).Run()
+	if err != ErrExitCode {
+		t.Errorf("wrong err := %v", err)
+	}
+
 }

cmdext/pipereader.go

@@ -8,6 +8,7 @@ import (
 )
 
 type pipeReader struct {
+	lineBufferSize *int
 	stdout         io.ReadCloser
 	stderr         io.ReadCloser
 }
@@ -33,7 +34,6 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 		buf := make([]byte, 128)
 		for true {
 			n, out := pr.stdout.Read(buf)
-
 			if n > 0 {
 				txt := string(buf[:n])
 				stdout += txt
@@ -91,6 +91,9 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	wg.Add(1)
 	go func() {
 		scanner := bufio.NewScanner(stdoutBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
 		for scanner.Scan() {
 			txt := scanner.Text()
 			for _, lstr := range listener {
@@ -98,6 +101,9 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 			}
 			combch <- combevt{txt, false}
 		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
 		combch <- combevt{"", true}
 		wg.Done()
 	}()
@@ -107,6 +113,9 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 	wg.Add(1)
 	go func() {
 		scanner := bufio.NewScanner(stderrBufferReader)
+		if pr.lineBufferSize != nil {
+			scanner.Buffer([]byte{}, *pr.lineBufferSize)
+		}
 		for scanner.Scan() {
 			txt := scanner.Text()
 			for _, lstr := range listener {
@@ -114,6 +123,9 @@ func (pr *pipeReader) Read(listener []CommandListener) (string, string, string,
 			}
 			combch <- combevt{txt, false}
 		}
+		if err := scanner.Err(); err != nil {
+			errch <- err
+		}
 		combch <- combevt{"", true}
 		wg.Done()
 	}()

dataext/lruMap_test.go

@@ -12,7 +12,7 @@ func init() {
 }
 
 func TestResultCache1(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key := randomKey()
@@ -50,7 +50,7 @@ func TestResultCache1(t *testing.T) {
 }
 
 func TestResultCache2(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -150,7 +150,7 @@ func TestResultCache2(t *testing.T) {
 }
 
 func TestResultCache3(t *testing.T) {
-	cache := NewLRUMap[string](8)
+	cache := NewLRUMap[string, string](8)
 	verifyLRUList(cache, t)
 
 	key1 := "key1"
@@ -173,7 +173,7 @@ func TestResultCache3(t *testing.T) {
 }
 
 // does a basic consistency check over the internal cache representation
-func verifyLRUList[TData any](cache *LRUMap[TData], t *testing.T) {
+func verifyLRUList[TKey comparable, TData any](cache *LRUMap[TKey, TData], t *testing.T) {
 	size := 0
 
 	tailFound := false

langext/array.go

@@ -265,6 +265,14 @@ func ArrMap[T1 any, T2 any](arr []T1, conv func(v T1) T2) []T2 {
 	return r
 }
 
+func ArrMapExt[T1 any, T2 any](arr []T1, conv func(idx int, v T1) T2) []T2 {
+	r := make([]T2, len(arr))
+	for i, v := range arr {
+		r[i] = conv(i, v)
+	}
+	return r
+}
+
 func ArrFilterMap[T1 any, T2 any](arr []T1, filter func(v T1) bool, conv func(v T1) T2) []T2 {
 	r := make([]T2, 0, len(arr))
 	for _, v := range arr {

rfctime/rfc3339Nano.go

@@ -70,7 +70,7 @@ func (t *RFC3339NanoTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) erro
 		return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339NanoTime", bt))
 	}
 	var tt time.Time
-	err := bson.Unmarshal(data, &tt)
+	err := bson.RawValue{Type: bt, Value: data}.Unmarshal(&tt)
 	if err != nil {
 		return err
 	}

rfctime/rfc3339Nano_test.go

@@ -12,7 +12,7 @@ func TestRoundtrip(t *testing.T) {
 		Value RFC3339NanoTime `json:"v"`
 	}
 
-	val1 := NewRFC3339Nano(time.Now())
+	val1 := NewRFC3339Nano(time.Unix(0, 1675951556820915171))
 	w1 := Wrap{val1}
 
 	jstr1, err := json.Marshal(w1)
@@ -20,7 +20,8 @@ func TestRoundtrip(t *testing.T) {
 		panic(err)
 	}
 
-	if string(jstr1) != "{\"v\":\"2023-01-29T20:32:36.149692117+01:00\"}" {
+	if string(jstr1) != "{\"v\":\"2023-02-09T15:05:56.820915171+01:00\"}" {
+		t.Errorf(string(jstr1))
 		t.Errorf("repr differs")
 	}