Compare commits

...

12 Commits

Author SHA1 Message Date
94f72e4ddf v0.0.80 2023-02-09 11:16:23 +01:00
df4388e6dc v0.0.79 2023-02-08 18:55:51 +01:00
fd33b43f31 v0.0.78 2023-02-03 01:05:36 +01:00
be4de07eb8 v0.0.77 2023-02-03 00:59:54 +01:00
36ed474bfe v0.0.76 2023-01-31 23:46:35 +01:00
fdc590c8c3 v0.0.75 2023-01-31 22:41:12 +01:00
1990e5d32d v0.0.74 2023-01-31 11:01:45 +01:00
72883cf6bd v0.0.73 2023-01-31 10:56:30 +01:00
ff08d5f180 v0.0.72 2023-01-30 19:55:55 +01:00
72d6b538f7 v0.0.71 2023-01-29 22:28:08 +01:00
48dd30fb94 v0.0.70 2023-01-29 22:07:28 +01:00
b7c5756f11 v0.0.69 2023-01-29 22:00:40 +01:00
17 changed files with 1059 additions and 133 deletions

View File

@@ -6,18 +6,20 @@ import (
) )
type CommandRunner struct { type CommandRunner struct {
program string program string
args []string args []string
timeout *time.Duration timeout *time.Duration
env []string env []string
listener []CommandListener
} }
func Runner(program string) *CommandRunner { func Runner(program string) *CommandRunner {
return &CommandRunner{ return &CommandRunner{
program: program, program: program,
args: make([]string, 0), args: make([]string, 0),
timeout: nil, timeout: nil,
env: make([]string, 0), env: make([]string, 0),
listener: make([]CommandListener, 0),
} }
} }
@@ -51,6 +53,21 @@ func (r *CommandRunner) Envs(env []string) *CommandRunner {
return r return r
} }
func (r *CommandRunner) Listen(lstr CommandListener) *CommandRunner {
r.listener = append(r.listener, lstr)
return r
}
func (r *CommandRunner) ListenStdout(lstr func(string)) *CommandRunner {
r.listener = append(r.listener, genericCommandListener{_readStdoutLine: &lstr})
return r
}
func (r *CommandRunner) ListenStderr(lstr func(string)) *CommandRunner {
r.listener = append(r.listener, genericCommandListener{_readStderrLine: &lstr})
return r
}
func (r *CommandRunner) Run() (CommandResult, error) { func (r *CommandRunner) Run() (CommandResult, error) {
return run(*r) return run(*r)
} }

View File

@@ -1,7 +1,8 @@
package cmdext package cmdext
import ( import (
"bufio" "gogs.mikescher.com/BlackForestBytes/goext/mathext"
"gogs.mikescher.com/BlackForestBytes/goext/syncext"
"os/exec" "os/exec"
"time" "time"
) )
@@ -16,7 +17,8 @@ type CommandResult struct {
func run(opt CommandRunner) (CommandResult, error) { func run(opt CommandRunner) (CommandResult, error) {
cmd := exec.Command(opt.program, opt.args...) cmd := exec.Command(opt.program, opt.args...)
cmd.Env = append(cmd.Env, opt.env)
cmd.Env = append(cmd.Env, opt.env...)
stdoutPipe, err := cmd.StdoutPipe() stdoutPipe, err := cmd.StdoutPipe()
if err != nil { if err != nil {
@@ -28,51 +30,39 @@ func run(opt CommandRunner) (CommandResult, error) {
return CommandResult{}, err return CommandResult{}, err
} }
preader := pipeReader{
stdout: stdoutPipe,
stderr: stderrPipe,
}
err = cmd.Start() err = cmd.Start()
if err != nil { if err != nil {
return CommandResult{}, err return CommandResult{}, err
} }
errch := make(chan error, 1) type resultObj struct {
go func() { errch <- cmd.Wait() }() stdout string
stderr string
stdcombined string
err error
}
combch := make(chan string, 32) outputChan := make(chan resultObj)
stopCombch := make(chan bool)
stdout := ""
go func() { go func() {
scanner := bufio.NewScanner(stdoutPipe) // we need to first fully read the pipes and then call Wait
for scanner.Scan() { // see https://pkg.go.dev/os/exec#Cmd.StdoutPipe
txt := scanner.Text()
stdout += txt
combch <- txt
}
}()
stderr := "" stdout, stderr, stdcombined, err := preader.Read(opt.listener)
go func() { if err != nil {
scanner := bufio.NewScanner(stderrPipe) outputChan <- resultObj{stdout, stderr, stdcombined, err}
for scanner.Scan() {
txt := scanner.Text()
stderr += txt
combch <- txt
} }
}()
defer func() { err = cmd.Wait()
stopCombch <- true if err != nil {
}() outputChan <- resultObj{stdout, stderr, stdcombined, err}
stdcombined := ""
go func() {
for {
select {
case txt := <-combch:
stdcombined += txt
case <-stopCombch:
return
}
} }
outputChan <- resultObj{stdout, stderr, stdcombined, nil}
}() }()
var timeoutChan <-chan time.Time = make(chan time.Time, 1) var timeoutChan <-chan time.Time = make(chan time.Time, 1)
@@ -84,30 +74,53 @@ func run(opt CommandRunner) (CommandResult, error) {
case <-timeoutChan: case <-timeoutChan:
_ = cmd.Process.Kill() _ = cmd.Process.Kill()
return CommandResult{ for _, lstr := range opt.listener {
StdOut: stdout, lstr.Timeout()
StdErr: stderr, }
StdCombined: stdcombined,
ExitCode: -1,
CommandTimedOut: true,
}, nil
case err := <-errch: if fallback, ok := syncext.ReadChannelWithTimeout(outputChan, mathext.Min(32*time.Millisecond, *opt.timeout)); ok {
if exiterr, ok := err.(*exec.ExitError); ok { // most of the time the cmd.Process.Kill() should also ahve finished the pipereader
// and we can at least return the already collected stdout, stderr, etc
return CommandResult{ return CommandResult{
StdOut: stdout, StdOut: fallback.stdout,
StdErr: stderr, StdErr: fallback.stderr,
StdCombined: stdcombined, StdCombined: fallback.stdcombined,
ExitCode: exiterr.ExitCode(), ExitCode: -1,
CommandTimedOut: true,
}, nil
} else {
return CommandResult{
StdOut: "",
StdErr: "",
StdCombined: "",
ExitCode: -1,
CommandTimedOut: true,
}, nil
}
case outobj := <-outputChan:
if exiterr, ok := outobj.err.(*exec.ExitError); ok {
excode := exiterr.ExitCode()
for _, lstr := range opt.listener {
lstr.Finished(excode)
}
return CommandResult{
StdOut: outobj.stdout,
StdErr: outobj.stderr,
StdCombined: outobj.stdcombined,
ExitCode: excode,
CommandTimedOut: false, CommandTimedOut: false,
}, nil }, nil
} else if err != nil { } else if err != nil {
return CommandResult{}, err return CommandResult{}, err
} else { } else {
for _, lstr := range opt.listener {
lstr.Finished(0)
}
return CommandResult{ return CommandResult{
StdOut: stdout, StdOut: outobj.stdout,
StdErr: stderr, StdErr: outobj.stderr,
StdCombined: stdcombined, StdCombined: outobj.stdcombined,
ExitCode: 0, ExitCode: 0,
CommandTimedOut: false, CommandTimedOut: false,
}, nil }, nil

226
cmdext/cmdrunner_test.go Normal file
View File

@@ -0,0 +1,226 @@
package cmdext
import (
"fmt"
"testing"
"time"
)
func TestStdout(t *testing.T) {
res1, err := Runner("printf").Arg("hello").Run()
if err != nil {
t.Errorf("%v", err)
}
if res1.StdErr != "" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "hello" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "hello\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestStderr(t *testing.T) {
res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"error\", file=sys.stderr, end='')").Run()
if err != nil {
t.Errorf("%v", err)
}
if res1.StdErr != "error" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "error\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestStdcombined(t *testing.T) {
res1, err := Runner("python").
Arg("-c").
Arg("import sys; import time; print(\"1\", file=sys.stderr, flush=True); time.sleep(0.1); print(\"2\", file=sys.stdout, flush=True); time.sleep(0.1); print(\"3\", file=sys.stderr, flush=True)").
Run()
if err != nil {
t.Errorf("%v", err)
}
if res1.StdErr != "1\n3\n" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "2\n" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "1\n2\n3\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestPartialRead(t *testing.T) {
res1, err := Runner("python").
Arg("-c").
Arg("import sys; import time; print(\"first message\", flush=True); time.sleep(5); print(\"cant see me\", flush=True);").
Timeout(100 * time.Millisecond).
Run()
if err != nil {
t.Errorf("%v", err)
}
if !res1.CommandTimedOut {
t.Errorf("!CommandTimedOut")
}
if res1.StdErr != "" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "first message\n" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "first message\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestPartialReadStderr(t *testing.T) {
res1, err := Runner("python").
Arg("-c").
Arg("import sys; import time; print(\"first message\", file=sys.stderr, flush=True); time.sleep(5); print(\"cant see me\", file=sys.stderr, flush=True);").
Timeout(100 * time.Millisecond).
Run()
if err != nil {
t.Errorf("%v", err)
}
if !res1.CommandTimedOut {
t.Errorf("!CommandTimedOut")
}
if res1.StdErr != "first message\n" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "first message\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestReadUnflushedStdout(t *testing.T) {
res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stdout, end='')").Run()
if err != nil {
t.Errorf("%v", err)
}
if res1.StdErr != "" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "message101" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "message101\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestReadUnflushedStderr(t *testing.T) {
res1, err := Runner("python").Arg("-c").Arg("import sys; print(\"message101\", file=sys.stderr, end='')").Run()
if err != nil {
t.Errorf("%v", err)
}
if res1.StdErr != "message101" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "message101\n" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestPartialReadUnflushed(t *testing.T) {
t.SkipNow()
res1, err := Runner("python").
Arg("-c").
Arg("import sys; import time; print(\"first message\", end=''); time.sleep(5); print(\"cant see me\", end='');").
Timeout(100 * time.Millisecond).
Run()
if err != nil {
t.Errorf("%v", err)
}
if !res1.CommandTimedOut {
t.Errorf("!CommandTimedOut")
}
if res1.StdErr != "" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "first message" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "first message" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestPartialReadUnflushedStderr(t *testing.T) {
t.SkipNow()
res1, err := Runner("python").
Arg("-c").
Arg("import sys; import time; print(\"first message\", file=sys.stderr, end=''); time.sleep(5); print(\"cant see me\", file=sys.stderr, end='');").
Timeout(100 * time.Millisecond).
Run()
if err != nil {
t.Errorf("%v", err)
}
if !res1.CommandTimedOut {
t.Errorf("!CommandTimedOut")
}
if res1.StdErr != "first message" {
t.Errorf("res1.StdErr == '%v'", res1.StdErr)
}
if res1.StdOut != "" {
t.Errorf("res1.StdOut == '%v'", res1.StdOut)
}
if res1.StdCombined != "first message" {
t.Errorf("res1.StdCombined == '%v'", res1.StdCombined)
}
}
func TestListener(t *testing.T) {
_, err := Runner("python").
Arg("-c").
Arg("import sys;" +
"import time;" +
"print(\"message 1\", flush=True);" +
"time.sleep(1);" +
"print(\"message 2\", flush=True);" +
"time.sleep(1);" +
"print(\"message 3\", flush=True);" +
"time.sleep(1);" +
"print(\"message 4\", file=sys.stderr, flush=True);" +
"time.sleep(1);" +
"print(\"message 5\", flush=True);" +
"time.sleep(1);" +
"print(\"final\");").
ListenStdout(func(s string) { fmt.Printf("@@STDOUT <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
ListenStderr(func(s string) { fmt.Printf("@@STDERR <<- %v (%v)\n", s, time.Now().Format(time.RFC3339Nano)) }).
Timeout(10 * time.Second).
Run()
if err != nil {
panic(err)
}
}

57
cmdext/listener.go Normal file
View File

@@ -0,0 +1,57 @@
package cmdext
type CommandListener interface {
ReadRawStdout([]byte)
ReadRawStderr([]byte)
ReadStdoutLine(string)
ReadStderrLine(string)
Finished(int)
Timeout()
}
type genericCommandListener struct {
_readRawStdout *func([]byte)
_readRawStderr *func([]byte)
_readStdoutLine *func(string)
_readStderrLine *func(string)
_finished *func(int)
_timeout *func()
}
func (g genericCommandListener) ReadRawStdout(v []byte) {
if g._readRawStdout != nil {
(*g._readRawStdout)(v)
}
}
func (g genericCommandListener) ReadRawStderr(v []byte) {
if g._readRawStderr != nil {
(*g._readRawStderr)(v)
}
}
func (g genericCommandListener) ReadStdoutLine(v string) {
if g._readStdoutLine != nil {
(*g._readStdoutLine)(v)
}
}
func (g genericCommandListener) ReadStderrLine(v string) {
if g._readStderrLine != nil {
(*g._readStderrLine)(v)
}
}
func (g genericCommandListener) Finished(v int) {
if g._finished != nil {
(*g._finished)(v)
}
}
func (g genericCommandListener) Timeout() {
if g._timeout != nil {
(*g._timeout)()
}
}

146
cmdext/pipereader.go Normal file
View File

@@ -0,0 +1,146 @@
package cmdext
import (
"bufio"
"gogs.mikescher.com/BlackForestBytes/goext/syncext"
"io"
"sync"
)
type pipeReader struct {
stdout io.ReadCloser
stderr io.ReadCloser
}
// Read ready stdout and stdin until finished
// also splits both pipes into lines and calld the listener
func (pr *pipeReader) Read(listener []CommandListener) (string, string, string, error) {
type combevt struct {
line string
stop bool
}
errch := make(chan error, 8)
wg := sync.WaitGroup{}
// [1] read raw stdout
wg.Add(1)
stdoutBufferReader, stdoutBufferWriter := io.Pipe()
stdout := ""
go func() {
buf := make([]byte, 128)
for true {
n, out := pr.stdout.Read(buf)
if n > 0 {
txt := string(buf[:n])
stdout += txt
_, _ = stdoutBufferWriter.Write(buf[:n])
for _, lstr := range listener {
lstr.ReadRawStdout(buf[:n])
}
}
if out == io.EOF {
break
}
if out != nil {
errch <- out
break
}
}
_ = stdoutBufferWriter.Close()
wg.Done()
}()
// [2] read raw stderr
wg.Add(1)
stderrBufferReader, stderrBufferWriter := io.Pipe()
stderr := ""
go func() {
buf := make([]byte, 128)
for true {
n, err := pr.stderr.Read(buf)
if n > 0 {
txt := string(buf[:n])
stderr += txt
_, _ = stderrBufferWriter.Write(buf[:n])
for _, lstr := range listener {
lstr.ReadRawStderr(buf[:n])
}
}
if err == io.EOF {
break
}
if err != nil {
errch <- err
break
}
}
_ = stderrBufferWriter.Close()
wg.Done()
}()
combch := make(chan combevt, 32)
// [3] collect stdout line-by-line
wg.Add(1)
go func() {
scanner := bufio.NewScanner(stdoutBufferReader)
for scanner.Scan() {
txt := scanner.Text()
for _, lstr := range listener {
lstr.ReadStdoutLine(txt)
}
combch <- combevt{txt, false}
}
combch <- combevt{"", true}
wg.Done()
}()
// [4] collect stderr line-by-line
wg.Add(1)
go func() {
scanner := bufio.NewScanner(stderrBufferReader)
for scanner.Scan() {
txt := scanner.Text()
for _, lstr := range listener {
lstr.ReadStderrLine(txt)
}
combch <- combevt{txt, false}
}
combch <- combevt{"", true}
wg.Done()
}()
// [5] combine stdcombined
wg.Add(1)
stdcombined := ""
go func() {
stopctr := 0
for stopctr < 2 {
vvv := <-combch
if vvv.stop {
stopctr++
} else {
stdcombined += vvv.line + "\n" // this comes from bufio.Scanner and has no newlines...
}
}
wg.Done()
}()
// wait for all (5) goroutines to finish
wg.Wait()
if err, ok := syncext.ReadNonBlocking(errch); ok {
return "", "", "", err
}
return stdout, stderr, stdcombined, nil
}

View File

@@ -1,10 +1,13 @@
package cryptext package cryptext
import ( import (
"bytes"
"crypto/aes" "crypto/aes"
"crypto/cipher" "crypto/cipher"
"crypto/rand" "crypto/rand"
"encoding/base64" "crypto/sha256"
"encoding/base32"
"encoding/json"
"errors" "errors"
"golang.org/x/crypto/scrypt" "golang.org/x/crypto/scrypt"
"io" "io"
@@ -12,35 +15,90 @@ import (
// https://stackoverflow.com/a/18819040/1761622 // https://stackoverflow.com/a/18819040/1761622
func EncryptAESSimple(password, text []byte) ([]byte, error) { type aesPayload struct {
Salt []byte `json:"s"`
key, err := scrypt.Key(password, nil, 32768, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing IV []byte `json:"i"`
if err != nil { Data []byte `json:"d"`
return nil, err Rounds int `json:"r"`
} Version uint `json:"v"`
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
b := base64.StdEncoding.EncodeToString(text)
ciphertext := make([]byte, aes.BlockSize+len(b))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
cfb := cipher.NewCFBEncrypter(block, iv)
cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
return ciphertext, nil
} }
func DecryptAESSimple(password, text []byte) ([]byte, error) { func EncryptAESSimple(password []byte, data []byte, rounds int) (string, error) {
key, err := scrypt.Key(password, nil, 32768, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing salt := make([]byte, 8)
_, err := io.ReadFull(rand.Reader, salt)
if err != nil {
return "", err
}
key, err := scrypt.Key(password, salt, rounds, 8, 1, 32)
if err != nil {
return "", err
}
block, err := aes.NewCipher(key)
if err != nil {
return "", err
}
h := sha256.New()
h.Write(data)
checksum := h.Sum(nil)
if len(checksum) != 32 {
return "", errors.New("wrong cs size")
}
ciphertext := make([]byte, 32+len(data))
iv := make([]byte, aes.BlockSize)
_, err = io.ReadFull(rand.Reader, iv)
if err != nil {
return "", err
}
combinedData := make([]byte, 0, 32+len(data))
combinedData = append(combinedData, checksum...)
combinedData = append(combinedData, data...)
cfb := cipher.NewCFBEncrypter(block, iv)
cfb.XORKeyStream(ciphertext, combinedData)
pl := aesPayload{
Salt: salt,
IV: iv,
Data: ciphertext,
Version: 1,
Rounds: rounds,
}
jbin, err := json.Marshal(pl)
if err != nil {
return "", err
}
res := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(jbin)
return res, nil
}
func DecryptAESSimple(password []byte, encText string) ([]byte, error) {
jbin, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(encText)
if err != nil {
return nil, err
}
var pl aesPayload
err = json.Unmarshal(jbin, &pl)
if err != nil {
return nil, err
}
if pl.Version != 1 {
return nil, errors.New("unsupported version")
}
key, err := scrypt.Key(password, pl.Salt, pl.Rounds, 8, 1, 32) // this is not 100% correct, rounds too low and salt is missing
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -50,18 +108,24 @@ func DecryptAESSimple(password, text []byte) ([]byte, error) {
return nil, err return nil, err
} }
if len(text) < aes.BlockSize { dest := make([]byte, len(pl.Data))
return nil, errors.New("ciphertext too short")
cfb := cipher.NewCFBDecrypter(block, pl.IV)
cfb.XORKeyStream(dest, pl.Data)
if len(dest) < 32 {
return nil, errors.New("payload too small")
} }
iv := text[:aes.BlockSize] chck := dest[:32]
text = text[aes.BlockSize:] data := dest[32:]
cfb := cipher.NewCFBDecrypter(block, iv)
cfb.XORKeyStream(text, text)
data, err := base64.StdEncoding.DecodeString(string(text)) h := sha256.New()
if err != nil { h.Write(data)
return nil, err chck2 := h.Sum(nil)
if !bytes.Equal(chck, chck2) {
return nil, errors.New("checksum mismatch")
} }
return data, nil return data, nil

View File

@@ -1,6 +1,9 @@
package cryptext package cryptext
import "testing" import (
"fmt"
"testing"
)
func TestEncryptAESSimple(t *testing.T) { func TestEncryptAESSimple(t *testing.T) {
@@ -8,15 +11,25 @@ func TestEncryptAESSimple(t *testing.T) {
str1 := []byte("Hello World") str1 := []byte("Hello World")
str2, err := EncryptAESSimple(pw, str1) str2, err := EncryptAESSimple(pw, str1, 512)
if err != nil { if err != nil {
panic(err) panic(err)
} }
fmt.Printf("%s\n", str2)
str3, err := DecryptAESSimple(pw, str2) str3, err := DecryptAESSimple(pw, str2)
if err != nil { if err != nil {
panic(err) panic(err)
} }
assertEqual(t, string(str1), string(str3)) assertEqual(t, string(str1), string(str3))
str4, err := EncryptAESSimple(pw, str3, 512)
if err != nil {
panic(err)
}
assertNotEqual(t, string(str2), string(str4))
} }

View File

@@ -23,3 +23,9 @@ func assertEqual(t *testing.T, actual string, expected string) {
t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected) t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
} }
} }
func assertNotEqual(t *testing.T, actual string, expected string) {
if actual == expected {
t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
}
}

1
go.mod
View File

@@ -9,5 +9,6 @@ require (
require ( require (
github.com/jmoiron/sqlx v1.3.5 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect
go.mongodb.org/mongo-driver v1.11.1 // indirect
golang.org/x/crypto v0.4.0 // indirect golang.org/x/crypto v0.4.0 // indirect
) )

35
go.sum
View File

@@ -1,15 +1,50 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -265,6 +265,16 @@ func ArrMap[T1 any, T2 any](arr []T1, conv func(v T1) T2) []T2 {
return r return r
} }
func ArrFilterMap[T1 any, T2 any](arr []T1, filter func(v T1) bool, conv func(v T1) T2) []T2 {
r := make([]T2, 0, len(arr))
for _, v := range arr {
if filter(v) {
r = append(r, conv(v))
}
}
return r
}
func ArrSum[T NumberConstraint](arr []T) T { func ArrSum[T NumberConstraint](arr []T) T {
var r T = 0 var r T = 0
for _, v := range arr { for _, v := range arr {

View File

@@ -7,3 +7,11 @@ func MapKeyArr[T comparable, V any](v map[T]V) []T {
} }
return result return result
} }
func ArrToMap[T comparable, V any](a []V, keyfunc func(V) T) map[T]V {
result := make(map[T]V, len(a))
for _, v := range a {
result[keyfunc(v)] = v
}
return result
}

View File

@@ -43,3 +43,45 @@ type RFCTime interface {
GoString() string GoString() string
String() string String() string
} }
type RFCDuration interface {
Time() time.Time
Serialize() string
UnmarshalJSON(bytes []byte) error
MarshalJSON() ([]byte, error)
MarshalBinary() ([]byte, error)
UnmarshalBinary(data []byte) error
GobEncode() ([]byte, error)
GobDecode(data []byte) error
MarshalText() ([]byte, error)
UnmarshalText(data []byte) error
After(u RFCTime) bool
Before(u RFCTime) bool
Equal(u RFCTime) bool
IsZero() bool
Date() (year int, month time.Month, day int)
Year() int
Month() time.Month
Day() int
Weekday() time.Weekday
ISOWeek() (year, week int)
Clock() (hour, min, sec int)
Hour() int
Minute() int
Second() int
Nanosecond() int
YearDay() int
Sub(u RFCTime) time.Duration
Unix() int64
UnixMilli() int64
UnixMicro() int64
UnixNano() int64
Format(layout string) string
GoString() string
String() string
}

203
rfctime/rfc3339.go Normal file
View File

@@ -0,0 +1,203 @@
package rfctime
import (
"encoding/json"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsontype"
"time"
)
type RFC3339Time time.Time
func (t RFC3339Time) Time() time.Time {
return time.Time(t)
}
func (t RFC3339Time) MarshalBinary() ([]byte, error) {
return (time.Time)(t).MarshalBinary()
}
func (t *RFC3339Time) UnmarshalBinary(data []byte) error {
return (*time.Time)(t).UnmarshalBinary(data)
}
func (t RFC3339Time) GobEncode() ([]byte, error) {
return (time.Time)(t).GobEncode()
}
func (t *RFC3339Time) GobDecode(data []byte) error {
return (*time.Time)(t).GobDecode(data)
}
func (t *RFC3339Time) UnmarshalJSON(data []byte) error {
str := ""
if err := json.Unmarshal(data, &str); err != nil {
return err
}
t0, err := time.Parse(t.FormatStr(), str)
if err != nil {
return err
}
*t = RFC3339Time(t0)
return nil
}
func (t RFC3339Time) MarshalJSON() ([]byte, error) {
str := t.Time().Format(t.FormatStr())
return json.Marshal(str)
}
func (t RFC3339Time) MarshalText() ([]byte, error) {
b := make([]byte, 0, len(t.FormatStr()))
return t.Time().AppendFormat(b, t.FormatStr()), nil
}
func (t *RFC3339Time) UnmarshalText(data []byte) error {
var err error
v, err := time.Parse(t.FormatStr(), string(data))
if err != nil {
return err
}
tt := RFC3339Time(v)
*t = tt
return nil
}
func (t *RFC3339Time) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
if bt != bsontype.DateTime {
return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339Time", bt))
}
var tt time.Time
err := bson.Unmarshal(data, &tt)
if err != nil {
return err
}
*t = RFC3339Time(tt)
return nil
}
func (t RFC3339Time) MarshalBSONValue() (bsontype.Type, []byte, error) {
return bson.MarshalValue(time.Time(t))
}
func (t RFC3339Time) Serialize() string {
return t.Time().Format(t.FormatStr())
}
func (t RFC3339Time) FormatStr() string {
return time.RFC3339
}
func (t RFC3339Time) After(u RFCTime) bool {
return t.Time().After(u.Time())
}
func (t RFC3339Time) Before(u RFCTime) bool {
return t.Time().Before(u.Time())
}
func (t RFC3339Time) Equal(u RFCTime) bool {
return t.Time().Equal(u.Time())
}
func (t RFC3339Time) IsZero() bool {
return t.Time().IsZero()
}
func (t RFC3339Time) Date() (year int, month time.Month, day int) {
return t.Time().Date()
}
func (t RFC3339Time) Year() int {
return t.Time().Year()
}
func (t RFC3339Time) Month() time.Month {
return t.Time().Month()
}
func (t RFC3339Time) Day() int {
return t.Time().Day()
}
func (t RFC3339Time) Weekday() time.Weekday {
return t.Time().Weekday()
}
func (t RFC3339Time) ISOWeek() (year, week int) {
return t.Time().ISOWeek()
}
func (t RFC3339Time) Clock() (hour, min, sec int) {
return t.Time().Clock()
}
func (t RFC3339Time) Hour() int {
return t.Time().Hour()
}
func (t RFC3339Time) Minute() int {
return t.Time().Minute()
}
func (t RFC3339Time) Second() int {
return t.Time().Second()
}
func (t RFC3339Time) Nanosecond() int {
return t.Time().Nanosecond()
}
func (t RFC3339Time) YearDay() int {
return t.Time().YearDay()
}
func (t RFC3339Time) Add(d time.Duration) RFC3339Time {
return RFC3339Time(t.Time().Add(d))
}
func (t RFC3339Time) Sub(u RFCTime) time.Duration {
return t.Time().Sub(u.Time())
}
func (t RFC3339Time) AddDate(years int, months int, days int) RFC3339Time {
return RFC3339Time(t.Time().AddDate(years, months, days))
}
func (t RFC3339Time) Unix() int64 {
return t.Time().Unix()
}
func (t RFC3339Time) UnixMilli() int64 {
return t.Time().UnixMilli()
}
func (t RFC3339Time) UnixMicro() int64 {
return t.Time().UnixMicro()
}
func (t RFC3339Time) UnixNano() int64 {
return t.Time().UnixNano()
}
func (t RFC3339Time) Format(layout string) string {
return t.Time().Format(layout)
}
func (t RFC3339Time) GoString() string {
return t.Time().GoString()
}
func (t RFC3339Time) String() string {
return t.Time().String()
}
func NewRFC3339(t time.Time) RFC3339Time {
return RFC3339Time(t)
}
func NowRFC3339() RFC3339Time {
return RFC3339Time(time.Now())
}

View File

@@ -2,6 +2,10 @@ package rfctime
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsontype"
"time" "time"
) )
@@ -61,6 +65,23 @@ func (t *RFC3339NanoTime) UnmarshalText(data []byte) error {
return nil return nil
} }
func (t *RFC3339NanoTime) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
if bt != bsontype.DateTime {
return errors.New(fmt.Sprintf("cannot unmarshal %v into RFC3339NanoTime", bt))
}
var tt time.Time
err := bson.Unmarshal(data, &tt)
if err != nil {
return err
}
*t = RFC3339NanoTime(tt)
return nil
}
func (t RFC3339NanoTime) MarshalBSONValue() (bsontype.Type, []byte, error) {
return bson.MarshalValue(time.Time(t))
}
func (t RFC3339NanoTime) Serialize() string { func (t RFC3339NanoTime) Serialize() string {
return t.Time().Format(t.FormatStr()) return t.Time().Format(t.FormatStr())
} }

59
rfctime/seconds.go Normal file
View File

@@ -0,0 +1,59 @@
package rfctime
import (
"encoding/json"
"gogs.mikescher.com/BlackForestBytes/goext/timeext"
"time"
)
type SecondsF64 time.Duration
func (d SecondsF64) Duration() time.Duration {
return time.Duration(d)
}
func (d SecondsF64) String() string {
return d.Duration().String()
}
func (d SecondsF64) Nanoseconds() int64 {
return d.Duration().Nanoseconds()
}
func (d SecondsF64) Microseconds() int64 {
return d.Duration().Microseconds()
}
func (d SecondsF64) Milliseconds() int64 {
return d.Duration().Milliseconds()
}
func (d SecondsF64) Seconds() float64 {
return d.Duration().Seconds()
}
func (d SecondsF64) Minutes() float64 {
return d.Duration().Minutes()
}
func (d SecondsF64) Hours() float64 {
return d.Duration().Hours()
}
func (d *SecondsF64) UnmarshalJSON(data []byte) error {
var secs float64 = 0
if err := json.Unmarshal(data, &secs); err != nil {
return err
}
*d = SecondsF64(timeext.FromSeconds(secs))
return nil
}
func (d SecondsF64) MarshalJSON() ([]byte, error) {
secs := d.Seconds()
return json.Marshal(secs)
}
func NewSecondsF64(t time.Duration) SecondsF64 {
return SecondsF64(t)
}

View File

@@ -2,58 +2,50 @@ package syncext
import ( import (
"context" "context"
"sync/atomic" "gogs.mikescher.com/BlackForestBytes/goext/langext"
"sync"
"time" "time"
) )
type AtomicBool struct { type AtomicBool struct {
v int32 v bool
waiter chan bool // unbuffered listener map[string]chan bool
lock sync.Mutex
} }
func NewAtomicBool(value bool) *AtomicBool { func NewAtomicBool(value bool) *AtomicBool {
if value { return &AtomicBool{
return &AtomicBool{v: 1, waiter: make(chan bool)} v: value,
} else { listener: make(map[string]chan bool),
return &AtomicBool{v: 0, waiter: make(chan bool)} lock: sync.Mutex{},
} }
} }
func (a *AtomicBool) Get() bool { func (a *AtomicBool) Get() bool {
return atomic.LoadInt32(&a.v) == 1 a.lock.Lock()
defer a.lock.Unlock()
return a.v
} }
func (a *AtomicBool) Set(value bool) { func (a *AtomicBool) Set(value bool) {
if value { a.lock.Lock()
atomic.StoreInt32(&a.v, 1) defer a.lock.Unlock()
} else {
atomic.StoreInt32(&a.v, 0)
}
select { a.v = value
case a.waiter <- value:
// message sent for k, v := range a.listener {
default: select {
// no receiver on channel case v <- value:
// message sent
default:
// no receiver on channel
delete(a.listener, k)
}
} }
} }
func (a *AtomicBool) Wait(waitFor bool) { func (a *AtomicBool) Wait(waitFor bool) {
if a.Get() == waitFor { _ = a.WaitWithContext(context.Background(), waitFor)
return
}
for {
if v, ok := ReadChannelWithTimeout(a.waiter, 128*time.Millisecond); ok {
if v == waitFor {
return
}
} else {
if a.Get() == waitFor {
return
}
}
}
} }
func (a *AtomicBool) WaitWithTimeout(timeout time.Duration, waitFor bool) error { func (a *AtomicBool) WaitWithTimeout(timeout time.Duration, waitFor bool) error {
@@ -71,12 +63,25 @@ func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
return nil return nil
} }
uuid, _ := langext.NewHexUUID()
waitchan := make(chan bool)
a.lock.Lock()
a.listener[uuid] = waitchan
a.lock.Unlock()
defer func() {
a.lock.Lock()
delete(a.listener, uuid)
a.lock.Unlock()
}()
for { for {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return err return err
} }
timeOut := 128 * time.Millisecond timeOut := 1024 * time.Millisecond
if dl, ok := ctx.Deadline(); ok { if dl, ok := ctx.Deadline(); ok {
timeOutMax := dl.Sub(time.Now()) timeOutMax := dl.Sub(time.Now())
@@ -87,7 +92,7 @@ func (a *AtomicBool) WaitWithContext(ctx context.Context, waitFor bool) error {
} }
} }
if v, ok := ReadChannelWithTimeout(a.waiter, timeOut); ok { if v, ok := ReadChannelWithTimeout(waitchan, timeOut); ok {
if v == waitFor { if v == waitFor {
return nil return nil
} }