v0.0.367

.idea/sqldialects.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/sq/sq_test.go" dialect="SQLite" />
+  </component>
+</project>

README.md

@@ -40,4 +40,6 @@ Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
 |             |            |                                                                                                               |
 | rext        | Mike       | Regex Wrapper, wraps regexp with a better interface                                                           |
 | wmo         | Mike       | Mongo Wrapper, wraps mongodb with a better interface                                                          |
+|             |            |                                                                                                               |
+| scn         | Mike       | SimpleCloudNotifier                                                                                           |
 |             |            |                                                                                                               |

bfcodegen/csid-generate.go

@@ -70,9 +70,9 @@ func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
 	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
 
 	if newChecksum != oldChecksum {
-		fmt.Printf("[IDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+		fmt.Printf("[CSIDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
 	} else {
-		fmt.Printf("[IDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		fmt.Printf("[CSIDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
 		return nil
 	}
 

bfcodegen/enum-generate.go

@@ -61,11 +61,15 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		}
 	}
 
-	gocode, _, err := _generateEnumSpecs(sourceDir, oldChecksum, destFile, true)
+	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, true)
 	if err != nil {
 		return err
 	}
 
+	if !changed {
+		return nil
+	}
+
 	err = os.WriteFile(destFile, []byte(gocode), 0o755)
 	if err != nil {
 		return err
@@ -74,11 +78,11 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool) (string, string, error) {
+func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool) (string, string, bool, error) {
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
-		return "", "", err
+		return "", "", false, err
 	}
 
 	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
@@ -90,7 +94,7 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 	for _, f := range files {
 		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
 		if err != nil {
-			return "", "", err
+			return "", "", false, err
 		}
 		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
 	}
@@ -101,7 +105,7 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 		fmt.Printf("[EnumGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
 	} else {
 		fmt.Printf("[EnumGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
-		return "", oldChecksum, nil
+		return "", oldChecksum, false, nil
 	}
 
 	allEnums := make([]EnumDef, 0)
@@ -112,7 +116,7 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 		fmt.Printf("========= %s =========\n\n", f.Name())
 		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()))
 		if err != nil {
-			return "", "", err
+			return "", "", false, err
 		}
 
 		fmt.Printf("\n")
@@ -125,21 +129,21 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 	}
 
 	if pkgname == "" {
-		return "", "", errors.New("no package name found in any file")
+		return "", "", false, errors.New("no package name found in any file")
 	}
 
 	rdata := fmtEnumOutput(newChecksum, allEnums, pkgname)
 
 	if !gofmt {
-		return rdata, newChecksum, nil
+		return rdata, newChecksum, true, nil
 	}
 
 	fdata, err := format.Source([]byte(rdata))
 	if err != nil {
-		return "", "", err
+		return "", "", false, err
 	}
 
-	return string(fdata), newChecksum, nil
+	return string(fdata), newChecksum, true, nil
 }
 
 func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {

bfcodegen/enum-generate_test.go

@@ -37,10 +37,10 @@ func TestGenerateEnumSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	s1, cs1, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
 	tst.AssertNoErr(t, err)
 
-	s2, cs2, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
 	tst.AssertNoErr(t, err)
 
 	tst.AssertEqual(t, cs1, cs2)
@@ -76,7 +76,7 @@ func TestGenerateEnumSpecsData(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	s1, _, err := _generateEnumSpecs(tmpDir, "", "", true)
+	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true)
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

dataext/syncMap.go

@@ -0,0 +1,113 @@
+package dataext
+
+import "sync"
+
+type SyncMap[TKey comparable, TData any] struct {
+	data map[TKey]TData
+	lock sync.Mutex
+}
+
+func (s *SyncMap[TKey, TData]) Set(key TKey, data TData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	s.data[key] = data
+}
+
+func (s *SyncMap[TKey, TData]) SetIfNotContains(key TKey, data TData) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if _, existsInPreState := s.data[key]; existsInPreState {
+		return false
+	}
+
+	s.data[key] = data
+
+	return true
+}
+
+func (s *SyncMap[TKey, TData]) Get(key TKey) (TData, bool) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	if v, ok := s.data[key]; ok {
+		return v, true
+	} else {
+		return *new(TData), false
+	}
+}
+
+func (s *SyncMap[TKey, TData]) Delete(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	delete(s.data, key)
+
+	return ok
+}
+
+func (s *SyncMap[TKey, TData]) Contains(key TKey) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	_, ok := s.data[key]
+
+	return ok
+}
+
+func (s *SyncMap[TKey, TData]) GetAllKeys() []TKey {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TKey, 0, len(s.data))
+
+	for k := range s.data {
+		r = append(r, k)
+	}
+
+	return r
+}
+
+func (s *SyncMap[TKey, TData]) GetAllValues() []TData {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[TKey]TData)
+	}
+
+	r := make([]TData, 0, len(s.data))
+
+	for _, v := range s.data {
+		r = append(r, v)
+	}
+
+	return r
+}

exerr/builder.go

@@ -420,7 +420,7 @@ func (b *Builder) Build() error {
 		b.errorData.ShortLog(stackSkipLogger.Error())
 	}
 
-	b.CallListener(MethodBuild)
+	b.errorData.CallListener(MethodBuild)
 
 	return b.errorData
 }
@@ -442,7 +442,7 @@ func (b *Builder) Output(ctx context.Context, g *gin.Context) {
 		b.errorData.Log(stackSkipLogger.Warn())
 	}
 
-	b.CallListener(MethodOutput)
+	b.errorData.CallListener(MethodOutput)
 }
 
 // Print prints the error
@@ -454,7 +454,7 @@ func (b *Builder) Print() {
 		b.errorData.ShortLog(stackSkipLogger.Warn())
 	}
 
-	b.CallListener(MethodPrint)
+	b.errorData.CallListener(MethodPrint)
 }
 
 func (b *Builder) Format(level LogPrintLevel) string {
@@ -467,7 +467,7 @@ func (b *Builder) Fatal() {
 	b.errorData.Severity = SevFatal
 	b.errorData.Log(stackSkipLogger.WithLevel(zerolog.FatalLevel))
 
-	b.CallListener(MethodFatal)
+	b.errorData.CallListener(MethodFatal)
 
 	os.Exit(1)
 }

exerr/data.go

@@ -48,6 +48,9 @@ var (
 	TypeMongoReflection   = NewType("MONGO_REFLECTION", langext.Ptr(500))
 	TypeMongoInvalidOpt   = NewType("MONGO_INVALIDOPT", langext.Ptr(500))
 
+	TypeSQLQuery = NewType("SQL_QUERY", langext.Ptr(500))
+	TypeSQLBuild = NewType("SQL_BUILD", langext.Ptr(500))
+
 	TypeWrap = NewType("Wrap", nil)
 
 	TypeBindFailURI      = NewType("BINDFAIL_URI", langext.Ptr(400))
@@ -65,18 +68,21 @@ var (
 	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
 	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))
 
-	// other values come the used package
+	// other values come from the downstream application that uses goext
 )
 
-var registeredTypes = dataext.SyncSet[string]{}
+var registeredTypes = dataext.SyncMap[string, ErrorType]{}
 
 func NewType(key string, defStatusCode *int) ErrorType {
-	insertOkay := registeredTypes.Add(key)
-	if !insertOkay {
-		panic("Cannot register same ErrType ('" + key + "') more than once")
-	}
+	et := ErrorType{key, defStatusCode}
 
-	return ErrorType{key, defStatusCode}
+	registeredTypes.Set(key, et)
+
+	return et
+}
+
+func ListRegisteredTypes() []ErrorType {
+	return registeredTypes.GetAllValues()
 }
 
 type LogPrintLevel string

exerr/exerr.go

@@ -258,6 +258,73 @@ func (ee *ExErr) Depth() int {
 	}
 }
 
+// GetMeta returns the meta value with the specified key
+// this method recurses through all wrapped errors and returns the first matching meta value
+func (ee *ExErr) GetMeta(key string) (any, bool) {
+	for curr := ee; curr != nil; curr = curr.OriginalError {
+		if v, ok := curr.Meta[key]; ok {
+			return v.Value, true
+		}
+	}
+
+	return nil, false
+}
+
+// GetMetaString functions the same as GetMeta, but returns false if the type does not match
+func (ee *ExErr) GetMetaString(key string) (string, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(string); ok {
+			return v2, true
+		}
+	}
+	return "", false
+}
+
+func (ee *ExErr) GetMetaBool(key string) (bool, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(bool); ok {
+			return v2, true
+		}
+	}
+	return false, false
+}
+
+func (ee *ExErr) GetMetaInt(key string) (int, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(int); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaFloat32(key string) (float32, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(float32); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaFloat64(key string) (float64, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(float64); ok {
+			return v2, true
+		}
+	}
+	return 0, false
+}
+
+func (ee *ExErr) GetMetaTime(key string) (time.Time, bool) {
+	if v1, ok := ee.GetMeta(key); ok {
+		if v2, ok := v1.(time.Time); ok {
+			return v2, true
+		}
+	}
+	return time.Time{}, false
+}
+
 // contains test if the supplied error is contained in this error (anywhere in the chain)
 func (ee *ExErr) contains(original *ExErr) (*ExErr, bool) {
 	if original == nil {

exerr/gin.go

@@ -57,6 +57,19 @@ func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) la
 	return ginJson
 }
 
+func (ee *ExErr) ToDefaultAPIJson() (string, error) {
+
+	gjr := json.GoJsonRender{Data: ee.ToAPIJson(true, pkgconfig.ExtendedGinOutput, pkgconfig.IncludeMetaInGinOutput), NilSafeSlices: true, NilSafeMaps: true}
+
+	r, err := gjr.RenderString()
+
+	if err != nil {
+		return "", err
+	}
+
+	return r, nil
+}
+
 // ToAPIJson converts the ExError to a json object
 // (the same object as used in the Output(gin) method)
 //

exerr/listener.go

@@ -25,13 +25,11 @@ func RegisterListener(l Listener) {
 	listener = append(listener, l)
 }
 
-func (b *Builder) CallListener(m Method) {
-	valErr := b.errorData
-
+func (ee *ExErr) CallListener(m Method) {
 	listenerLock.Lock()
 	defer listenerLock.Unlock()
 
 	for _, v := range listener {
-		v(m, valErr)
+		v(m, ee)
 	}
 }

ginext/engine.go

@@ -9,6 +9,7 @@ import (
 	"gogs.mikescher.com/BlackForestBytes/goext/rext"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"regexp"
 	"strings"
 	"time"
@@ -16,12 +17,14 @@ import (
 
 type GinWrapper struct {
 	engine          *gin.Engine
-	SuppressGinLogs bool
+	suppressGinLogs bool
 
-	allowCors      bool
-	ginDebug       bool
-	bufferBody     bool
-	requestTimeout time.Duration
+	allowCors             bool
+	ginDebug              bool
+	bufferBody            bool
+	requestTimeout        time.Duration
+	listenerBeforeRequest []func(g *gin.Context)
+	listenerAfterRequest  []func(g *gin.Context, resp HTTPResponse)
 
 	routeSpecs []ginRouteSpec
 }
@@ -33,40 +36,46 @@ type ginRouteSpec struct {
 	Handler     string
 }
 
+type Options struct {
+	AllowCors             *bool                                     // Add cors handler to allow all CORS requests on the default http methods
+	GinDebug              *bool                                     // Set gin.debug to true (adds more logs)
+	BufferBody            *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
+	Timeout               *time.Duration                            // The default handler timeout
+	ListenerBeforeRequest []func(g *gin.Context)                    // Register listener that are called before the handler method
+	ListenerAfterRequest  []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+}
+
 // NewEngine creates a new (wrapped) ginEngine
-// Parameters are:
-// - [allowCors]    Add cors handler to allow all CORS requests on the default http methods
-// - [ginDebug]     Set gin.debug to true (adds more logs)
-// - [bufferBody]   Buffers the input body stream, this way the ginext error handler can later include the whole request body
-// - [timeout]      The default handler timeout
-func NewEngine(allowCors bool, ginDebug bool, bufferBody bool, timeout time.Duration) *GinWrapper {
+func NewEngine(opt Options) *GinWrapper {
 	engine := gin.New()
 
 	wrapper := &GinWrapper{
-		engine:          engine,
-		SuppressGinLogs: false,
-		allowCors:       allowCors,
-		ginDebug:        ginDebug,
-		bufferBody:      bufferBody,
-		requestTimeout:  timeout,
+		engine:                engine,
+		suppressGinLogs:       false,
+		allowCors:             langext.Coalesce(opt.AllowCors, false),
+		ginDebug:              langext.Coalesce(opt.GinDebug, true),
+		bufferBody:            langext.Coalesce(opt.BufferBody, false),
+		requestTimeout:        langext.Coalesce(opt.Timeout, 24*time.Hour),
+		listenerBeforeRequest: opt.ListenerBeforeRequest,
+		listenerAfterRequest:  opt.ListenerAfterRequest,
 	}
 
 	engine.RedirectFixedPath = false
 	engine.RedirectTrailingSlash = false
 
-	if allowCors {
+	if wrapper.allowCors {
 		engine.Use(CorsMiddleware())
 	}
 
 	// do not debug-print routes
 	gin.DebugPrintRouteFunc = func(_, _, _ string, _ int) {}
 
-	if !ginDebug {
+	if !wrapper.ginDebug {
 		gin.SetMode(gin.ReleaseMode)
 
 		ginlogger := gin.Logger()
 		engine.Use(func(context *gin.Context) {
-			if !wrapper.SuppressGinLogs {
+			if !wrapper.suppressGinLogs {
 				ginlogger(context)
 			}
 		})
@@ -177,3 +186,10 @@ func (w *GinWrapper) cleanMiddlewareName(fname string) string {
 
 	return fname
 }
+
+// ServeHTTP only used for unit tests
+func (w *GinWrapper) ServeHTTP(req *http.Request) *httptest.ResponseRecorder {
+	respRec := httptest.NewRecorder()
+	w.engine.ServeHTTP(respRec, req)
+	return respRec
+}

ginext/funcWrapper.go

@@ -20,6 +20,10 @@ func Wrap(w *GinWrapper, fn WHandlerFunc) gin.HandlerFunc {
 			persistantData: &preContextData{},
 		}
 
+		for _, lstr := range w.listenerBeforeRequest {
+			lstr(g)
+		}
+
 		wrap, stackTrace, panicObj := callPanicSafe(fn, pctx)
 		if panicObj != nil {
 
@@ -45,6 +49,10 @@ func Wrap(w *GinWrapper, fn WHandlerFunc) gin.HandlerFunc {
 			}
 		}
 
+		for _, lstr := range w.listenerAfterRequest {
+			lstr(g, wrap)
+		}
+
 		if reqctx.Err() == nil {
 			wrap.Write(g)
 		}

ginext/response.go

@@ -5,8 +5,20 @@ import (
 	"github.com/gin-gonic/gin"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"os"
 )
 
+type cookieval struct {
+	name     string
+	value    string
+	maxAge   int
+	path     string
+	domain   string
+	secure   bool
+	httpOnly bool
+}
+
 type headerval struct {
 	Key string
 	Val string
@@ -15,24 +27,42 @@ type headerval struct {
 type HTTPResponse interface {
 	Write(g *gin.Context)
 	WithHeader(k string, v string) HTTPResponse
+	WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse
 	IsSuccess() bool
 }
 
+type InspectableHTTPResponse interface {
+	HTTPResponse
+
+	Statuscode() int
+	BodyString(g *gin.Context) *string
+	ContentType() string
+	Headers() []string
+}
+
 type jsonHTTPResponse struct {
 	statusCode int
 	data       any
 	headers    []headerval
+	cookies    []cookieval
+}
+
+func (j jsonHTTPResponse) jsonRenderer(g *gin.Context) json.GoJsonRender {
+	var f *string
+	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
+		f = &jsonfilter
+	}
+	return json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f}
 }
 
 func (j jsonHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
-	var f *string
-	if jsonfilter := g.GetString("goext.jsonfilter"); jsonfilter != "" {
-		f = &jsonfilter
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
 	}
-	g.Render(j.statusCode, json.GoJsonRender{Data: j.data, NilSafeSlices: true, NilSafeMaps: true, Filter: f})
+	g.Render(j.statusCode, j.jsonRenderer(g))
 }
 
 func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {
@@ -40,19 +70,48 @@ func (j jsonHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j jsonHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j jsonHTTPResponse) IsSuccess() bool {
 	return j.statusCode >= 200 && j.statusCode <= 399
 }
 
+func (j jsonHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j jsonHTTPResponse) BodyString(g *gin.Context) *string {
+	if str, err := j.jsonRenderer(g).RenderString(); err == nil {
+		return &str
+	} else {
+		return nil
+	}
+}
+
+func (j jsonHTTPResponse) ContentType() string {
+	return "application/json"
+}
+
+func (j jsonHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type emptyHTTPResponse struct {
 	statusCode int
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j emptyHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Status(j.statusCode)
 }
 
@@ -61,20 +120,45 @@ func (j emptyHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j emptyHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j emptyHTTPResponse) IsSuccess() bool {
 	return j.statusCode >= 200 && j.statusCode <= 399
 }
 
+func (j emptyHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j emptyHTTPResponse) BodyString(*gin.Context) *string {
+	return nil
+}
+
+func (j emptyHTTPResponse) ContentType() string {
+	return ""
+}
+
+func (j emptyHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type textHTTPResponse struct {
 	statusCode int
 	data       string
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j textHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.String(j.statusCode, "%s", j.data)
 }
 
@@ -83,21 +167,46 @@ func (j textHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j textHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j textHTTPResponse) IsSuccess() bool {
 	return j.statusCode >= 200 && j.statusCode <= 399
 }
 
+func (j textHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j textHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(j.data)
+}
+
+func (j textHTTPResponse) ContentType() string {
+	return "text/plain"
+}
+
+func (j textHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type dataHTTPResponse struct {
 	statusCode  int
 	data        []byte
 	contentType string
 	headers     []headerval
+	cookies     []cookieval
 }
 
 func (j dataHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Data(j.statusCode, j.contentType, j.data)
 }
 
@@ -106,15 +215,37 @@ func (j dataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j dataHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j dataHTTPResponse) IsSuccess() bool {
 	return j.statusCode >= 200 && j.statusCode <= 399
 }
 
+func (j dataHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j dataHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(string(j.data))
+}
+
+func (j dataHTTPResponse) ContentType() string {
+	return j.contentType
+}
+
+func (j dataHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type fileHTTPResponse struct {
 	mimetype string
 	filepath string
 	filename *string
 	headers  []headerval
+	cookies  []cookieval
 }
 
 func (j fileHTTPResponse) Write(g *gin.Context) {
@@ -126,6 +257,9 @@ func (j fileHTTPResponse) Write(g *gin.Context) {
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.File(j.filepath)
 }
 
@@ -134,27 +268,55 @@ func (j fileHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j fileHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j fileHTTPResponse) IsSuccess() bool {
 	return true
 }
 
+func (j fileHTTPResponse) Statuscode() int {
+	return 200
+}
+
+func (j fileHTTPResponse) BodyString(*gin.Context) *string {
+	data, err := os.ReadFile(j.filepath)
+	if err != nil {
+		return nil
+	}
+	return langext.Ptr(string(data))
+}
+
+func (j fileHTTPResponse) ContentType() string {
+	return j.mimetype
+}
+
+func (j fileHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type downloadDataHTTPResponse struct {
 	statusCode int
 	mimetype   string
 	data       []byte
 	filename   *string
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j downloadDataHTTPResponse) Write(g *gin.Context) {
 	g.Header("Content-Type", j.mimetype) // if we don't set it here gin does weird file-sniffing later...
 	if j.filename != nil {
 		g.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", *j.filename))
-
 	}
 	for _, v := range j.headers {
 		g.Header(v.Key, v.Val)
 	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Data(j.statusCode, j.mimetype, j.data)
 }
 
@@ -163,17 +325,45 @@ func (j downloadDataHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j downloadDataHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j downloadDataHTTPResponse) IsSuccess() bool {
 	return j.statusCode >= 200 && j.statusCode <= 399
 }
 
+func (j downloadDataHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j downloadDataHTTPResponse) BodyString(*gin.Context) *string {
+	return langext.Ptr(string(j.data))
+}
+
+func (j downloadDataHTTPResponse) ContentType() string {
+	return j.mimetype
+}
+
+func (j downloadDataHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type redirectHTTPResponse struct {
 	statusCode int
 	url        string
 	headers    []headerval
+	cookies    []cookieval
 }
 
 func (j redirectHTTPResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	g.Redirect(j.statusCode, j.url)
 }
 
@@ -182,17 +372,47 @@ func (j redirectHTTPResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j redirectHTTPResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j redirectHTTPResponse) IsSuccess() bool {
 	return j.statusCode >= 200 && j.statusCode <= 399
 }
 
+func (j redirectHTTPResponse) Statuscode() int {
+	return j.statusCode
+}
+
+func (j redirectHTTPResponse) BodyString(*gin.Context) *string {
+	return nil
+}
+
+func (j redirectHTTPResponse) ContentType() string {
+	return ""
+}
+
+func (j redirectHTTPResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
 type jsonAPIErrResponse struct {
 	err     *exerr.ExErr
 	headers []headerval
+	cookies []cookieval
 }
 
 func (j jsonAPIErrResponse) Write(g *gin.Context) {
+	for _, v := range j.headers {
+		g.Header(v.Key, v.Val)
+	}
+	for _, v := range j.cookies {
+		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
+	}
 	j.err.Output(g)
+
+	j.err.CallListener(exerr.MethodOutput)
 }
 
 func (j jsonAPIErrResponse) WithHeader(k string, v string) HTTPResponse {
@@ -200,10 +420,39 @@ func (j jsonAPIErrResponse) WithHeader(k string, v string) HTTPResponse {
 	return j
 }
 
+func (j jsonAPIErrResponse) WithCookie(name string, value string, maxAge int, path string, domain string, secure bool, httpOnly bool) HTTPResponse {
+	j.cookies = append(j.cookies, cookieval{name, value, maxAge, path, domain, secure, httpOnly})
+	return j
+}
+
 func (j jsonAPIErrResponse) IsSuccess() bool {
 	return false
 }
 
+func (j jsonAPIErrResponse) Statuscode() int {
+	return langext.Coalesce(j.err.RecursiveStatuscode(), 0)
+}
+
+func (j jsonAPIErrResponse) BodyString(*gin.Context) *string {
+	if str, err := j.err.ToDefaultAPIJson(); err == nil {
+		return &str
+	} else {
+		return nil
+	}
+}
+
+func (j jsonAPIErrResponse) ContentType() string {
+	return "application/json"
+}
+
+func (j jsonAPIErrResponse) Headers() []string {
+	return langext.ArrMap(j.headers, func(v headerval) string { return v.Key + "=" + v.Val })
+}
+
+func (j jsonAPIErrResponse) Unwrap() error {
+	return j.err
+}
+
 func Status(sc int) HTTPResponse {
 	return &emptyHTTPResponse{statusCode: sc}
 }

go.mod

@@ -4,19 +4,21 @@ go 1.21
 
 require (
 	github.com/gin-gonic/gin v1.9.1
+	github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.-
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/rs/xid v1.5.0
 	github.com/rs/zerolog v1.31.0
 	go.mongodb.org/mongo-driver v1.13.1
-	golang.org/x/crypto v0.16.0
-	golang.org/x/sys v0.15.0
-	golang.org/x/term v0.15.0
+	golang.org/x/crypto v0.18.0
+	golang.org/x/sys v0.16.0
+	golang.org/x/term v0.16.0
 )
 
 require (
 	github.com/bytedance/sonic v1.10.2 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
@@ -24,6 +26,7 @@ require (
 	github.com/go-playground/validator/v10 v10.16.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.5.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
@@ -33,17 +36,22 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	golang.org/x/arch v0.6.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.37.6 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/sqlite v1.28.0 // indirect
 )

go.sum

@@ -6,7 +6,6 @@ github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA=
-github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo=
 github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
@@ -14,21 +13,22 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
+github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
-github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
 github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
@@ -44,24 +44,18 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
-github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
-github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
 github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
-github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
@@ -72,12 +66,12 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -86,11 +80,13 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
-github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
-github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
+github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
@@ -108,8 +104,6 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -122,47 +116,33 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
-go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
-go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY=
-go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
 go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
 go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y=
-golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc=
-golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
+golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
+golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
-golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
+golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -174,43 +154,41 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw=
+modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
+modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.342"
+const GoextVersion = "0.0.367"
 
-const GoextVersionTimestamp = "2023-12-07T17:57:06+0100"
+const GoextVersionTimestamp = "2024-01-12T18:40:29+0100"

gojson/gionic.go

@@ -37,6 +37,14 @@ func (r GoJsonRender) Render(w http.ResponseWriter) error {
 	return nil
 }
 
+func (r GoJsonRender) RenderString() (string, error) {
+	jsonBytes, err := MarshalSafeCollections(r.Data, r.NilSafeSlices, r.NilSafeMaps, r.Indent, r.Filter)
+	if err != nil {
+		panic(err)
+	}
+	return string(jsonBytes), nil
+}
+
 func (r GoJsonRender) WriteContentType(w http.ResponseWriter) {
 	header := w.Header()
 	if val := header["Content-Type"]; len(val) == 0 {

googleapi/mimeMessage_test.go

@@ -1,7 +1,6 @@
 package googleapi
 
 import (
-	"fmt"
 	"gogs.mikescher.com/BlackForestBytes/goext/tst"
 	"os"
 	"testing"
@@ -18,7 +17,7 @@ func TestEncodeMimeMail(t *testing.T) {
 		MailBody{Plain: "Plain Text"},
 		nil)
 
-	fmt.Printf("%s\n\n", mail)
+	verifyMime(mail)
 }
 
 func TestEncodeMimeMail2(t *testing.T) {
@@ -35,7 +34,7 @@ func TestEncodeMimeMail2(t *testing.T) {
 		},
 		nil)
 
-	fmt.Printf("%s\n\n", mail)
+	verifyMime(mail)
 }
 
 func TestEncodeMimeMail3(t *testing.T) {
@@ -53,12 +52,12 @@ func TestEncodeMimeMail3(t *testing.T) {
 			{Data: []byte("HelloWorld"), Filename: "test.txt", IsInline: false, ContentType: "text/plain"},
 		})
 
-	fmt.Printf("%s\n\n", mail)
+	verifyMime(mail)
 }
 
 func TestEncodeMimeMail4(t *testing.T) {
 
-	b := tst.Must(os.ReadFile("/home/mike/Pictures/Screenshot_20220706_190205.png"))(t)
+	b := tst.Must(os.ReadFile("test_placeholder.png"))(t)
 
 	mail := encodeMimeMail(
 		"noreply@heydyno.de",
@@ -73,5 +72,9 @@ func TestEncodeMimeMail4(t *testing.T) {
 			{Data: b, Filename: "img.png", IsInline: true, ContentType: "image/png"},
 		})
 
-	fmt.Printf("%s\n\n", mail)
+	verifyMime(mail)
+}
+
+func verifyMime(mail string) {
+	//fmt.Printf("%s\n\n", mail)
 }

googleapi/sendMail_test.go

@@ -127,7 +127,7 @@ func TestSendMail4(t *testing.T) {
 
 	gclient := NewGoogleClient(auth)
 
-	b := tst.Must(os.ReadFile("/home/mike/Pictures/Screenshot_20220706_190205.png"))(t)
+	b := tst.Must(os.ReadFile("test_placeholder.png"))(t)
 
 	mail, err := gclient.SendMail(
 		ctx,

langext/pointer.go

@@ -10,10 +10,23 @@ var PTrue = Ptr(true)
 // PFalse := &false
 var PFalse = Ptr(false)
 
+// PNil := &nil
+var PNil = Ptr[any](nil)
+
 func Ptr[T any](v T) *T {
 	return &v
 }
 
+func DblPtr[T any](v T) **T {
+	v_ := &v
+	return &v_
+}
+
+func DblPtrNil[T any]() **T {
+	var v *T = nil
+	return &v
+}
+
 func PtrInt32(v int32) *int32 {
 	return &v
 }

pagination/pagination.go

@@ -1,16 +1,19 @@
 package pagination
 
 type Pagination struct {
-	Page             int `json:"page"`            // page (first page == 1)
-	Limit            int `json:"limit"`           // max-page-size
-	TotalPages       int `json:"totalPages"`      // total page-count
-	TotalItems       int `json:"totalItems"`      // total items-count
-	CurrentPageCount int `json:"currntPageCount"` // item-count in current page ( == len(data) )
+	Page             int `json:"page"`             // page (first page == 1)
+	Limit            int `json:"limit"`            // max-page-size
+	TotalPages       int `json:"totalPages"`       // total page-count
+	TotalItems       int `json:"totalItems"`       // total items-count
+	CurrentPageCount int `json:"currentPageCount"` // item-count in current page ( == len(data) )
 }
 
 func CalcPaginationTotalPages(totalItems int, limit int) int {
 	if totalItems == 0 {
 		return 0
 	}
+	if limit == 0 {
+		return 0
+	}
 	return 1 + (totalItems-1)/limit
 }

reflectext/mapAccess.go

@@ -0,0 +1,98 @@
+package reflectext
+
+import (
+	"reflect"
+	"strings"
+)
+
+// GetMapPath returns the value deep inside a hierahically nested map structure
+// eg:
+// x := langext.H{"K1": langext.H{"K2": 665}}
+// GetMapPath[int](x, "K1.K2") == 665
+func GetMapPath[TData any](mapval any, path string) (TData, bool) {
+	var ok bool
+
+	split := strings.Split(path, ".")
+
+	for i, key := range split {
+
+		if i < len(split)-1 {
+			mapval, ok = GetMapField[any](mapval, key)
+			if !ok {
+				return *new(TData), false
+			}
+		} else {
+			return GetMapField[TData](mapval, key)
+		}
+	}
+
+	return *new(TData), false
+}
+
+// GetMapField gets the value of a map, without knowing the actual types (mapval is any)
+// eg:
+// x := langext.H{"K1": 665}
+// GetMapPath[int](x, "K1") == 665
+//
+// works with aliased types and autom. dereferences pointes
+func GetMapField[TData any, TKey comparable](mapval any, key TKey) (TData, bool) {
+
+	rval := reflect.ValueOf(mapval)
+
+	for rval.Kind() == reflect.Ptr && !rval.IsNil() {
+		rval = rval.Elem()
+	}
+
+	if rval.Kind() != reflect.Map {
+		return *new(TData), false // mapval is not a map
+	}
+
+	kval := reflect.ValueOf(key)
+
+	if !kval.Type().AssignableTo(rval.Type().Key()) {
+		return *new(TData), false // key cannot index mapval
+	}
+
+	eval := rval.MapIndex(kval)
+	if !eval.IsValid() {
+		return *new(TData), false // key does not exist in mapval
+	}
+
+	destType := reflect.TypeOf(new(TData)).Elem()
+
+	if eval.Type() == destType {
+		return eval.Interface().(TData), true
+	}
+
+	if eval.CanConvert(destType) && !preventConvert(eval.Type(), destType) {
+		return eval.Convert(destType).Interface().(TData), true
+	}
+
+	if (eval.Kind() == reflect.Ptr || eval.Kind() == reflect.Interface) && eval.IsNil() && destType.Kind() == reflect.Ptr {
+		return *new(TData), false // special case: mapval[key] is nil
+	}
+
+	for (eval.Kind() == reflect.Ptr || eval.Kind() == reflect.Interface) && !eval.IsNil() {
+		eval = eval.Elem()
+
+		if eval.Type() == destType {
+			return eval.Interface().(TData), true
+		}
+
+		if eval.CanConvert(destType) && !preventConvert(eval.Type(), destType) {
+			return eval.Convert(destType).Interface().(TData), true
+		}
+	}
+
+	return *new(TData), false // mapval[key] is not of type TData
+}
+
+func preventConvert(t1 reflect.Type, t2 reflect.Type) bool {
+	if t1.Kind() == reflect.String && t1.Kind() != reflect.String {
+		return true
+	}
+	if t2.Kind() == reflect.String && t1.Kind() != reflect.String {
+		return true
+	}
+	return false
+}

reflectext/mapAccess_test.go

@@ -0,0 +1,55 @@
+package reflectext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestGetMapPath(t *testing.T) {
+	type PseudoInt = int64
+
+	mymap2 := map[string]map[string]any{"Test": {"Second": 3}}
+
+	var maany2 any = mymap2
+
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[int](maany2, "Test.Second")), "3 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[int](maany2, "Test2.Second")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[int](maany2, "Test.Second2")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[string](maany2, "Test.Second")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[string](maany2, "Test2.Second")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[string](maany2, "Test.Second2")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[PseudoInt](maany2, "Test.Second")), "3 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[PseudoInt](maany2, "Test2.Second")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapPath[PseudoInt](maany2, "Test.Second2")), "0 false")
+}
+
+func TestGetMapField(t *testing.T) {
+	type PseudoInt = int64
+
+	mymap1 := map[string]any{"Test": 12}
+	mymap2 := map[string]int{"Test": 12}
+
+	var maany1 any = mymap1
+	var maany2 any = mymap2
+
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany1, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany1, "Test2")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany1, "Test")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany1, "Test2")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany1, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany1, "Test2")), "0 false")
+
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany2, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[int](maany2, "Test2")), "0 false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany2, "Test")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[string](maany2, "Test2")), "false")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany2, "Test")), "12 true")
+	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany2, "Test2")), "0 false")
+}
+
+func main2() {
+}
+
+func main() {
+}

scn/scn.go

@@ -0,0 +1,13 @@
+package scn
+
+type Connection struct {
+	uid   string
+	token string
+}
+
+func New(userid string, token string) *Connection {
+	return &Connection{
+		uid:   userid,
+		token: token,
+	}
+}

scn/send.go

@@ -0,0 +1,171 @@
+package scn
+
+import (
+	"bytes"
+	"context"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	json "gogs.mikescher.com/BlackForestBytes/goext/gojson"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"io"
+	"net/http"
+	"time"
+)
+
+var (
+	ErrAuthFailed        = exerr.NewType("GOEXT_SCN_AUTHFAILED", nil)
+	ErrQuota             = exerr.NewType("GOEXT_SCN_QUOTAEXCEEDED", nil)
+	ErrBadRequest        = exerr.NewType("GOEXT_SCN_BADREQUEST", nil)
+	ErrInternalServerErr = exerr.NewType("GOEXT_SCN_INTERNALSERVER", nil)
+	ErrOther             = exerr.NewType("GOEXT_SCN_OTHER", nil)
+)
+
+type MessageResponse struct {
+	ErrHighlight int    `json:"errhighlight"`
+	Error        int    `json:"error"`
+	IsPro        bool   `json:"is_pro"`
+	Message      string `json:"message"`
+	Messagecount int    `json:"messagecount"`
+	Quota        int    `json:"quota"`
+	QuotaMax     int    `json:"quota_max"`
+	SCNMessageID string `json:"scn_msg_id"`
+	Success      bool   `json:"success"`
+	SuppressSend bool   `json:"suppress_send"`
+}
+
+type MessageErrResponse struct {
+	Errhighlight int    `json:"errhighlight"`
+	Error        int    `json:"error"`
+	Message      string `json:"message"`
+	Success      bool   `json:"success"`
+}
+
+type MessageBuilder struct {
+	conn       *Connection
+	title      string
+	content    *string
+	channel    *string
+	time       *time.Time
+	sendername *string
+	priority   *int
+}
+
+func (c *Connection) Message(title string) *MessageBuilder {
+	return &MessageBuilder{conn: c, title: title}
+}
+
+func (c *MessageBuilder) Channel(channel string) *MessageBuilder {
+	c.channel = &channel
+	return c
+}
+
+func (c *MessageBuilder) Content(content string) *MessageBuilder {
+	c.content = &content
+	return c
+}
+
+func (c *MessageBuilder) Time(t time.Time) *MessageBuilder {
+	c.time = &t
+	return c
+}
+
+func (c *MessageBuilder) SenderName(sn string) *MessageBuilder {
+	c.sendername = &sn
+	return c
+}
+
+func (c *MessageBuilder) Priority(p int) *MessageBuilder {
+	c.priority = &p
+	return c
+}
+
+func (c *MessageBuilder) Send(ctx context.Context) (MessageResponse, error) {
+	client := http.Client{Timeout: 5 * time.Second}
+
+	body := langext.H{}
+
+	body["user_id"] = c.conn.uid
+	body["key"] = c.conn.token
+
+	if c.channel != nil {
+		body["channel"] = *c.channel
+	}
+
+	body["title"] = c.title
+
+	if c.content != nil {
+		body["content"] = *c.content
+	}
+
+	if c.sendername != nil {
+		body["content"] = *c.sendername
+	}
+
+	if c.time != nil {
+		body["timestamp"] = c.time.Unix()
+	}
+
+	if c.priority != nil {
+		body["priority"] = *c.priority
+	}
+
+	body["msg_id"] = langext.MustHexUUID()
+
+	rawbody, err := json.Marshal(body)
+	if err != nil {
+		return MessageResponse{}, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "POST", "https://simplecloudnotifier.de/", bytes.NewBuffer(rawbody))
+	if err != nil {
+		return MessageResponse{}, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return MessageResponse{}, err
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+
+		raw, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return MessageResponse{}, err
+		}
+
+		var mr MessageResponse
+		err = json.Unmarshal(raw, &mr)
+		if err != nil {
+			return MessageResponse{}, err
+		}
+
+		return mr, nil
+	} else {
+		errMessage := resp.Status
+
+		if raw, err := io.ReadAll(resp.Body); err == nil {
+			var mr MessageErrResponse
+			if err = json.Unmarshal(raw, &mr); err == nil {
+				errMessage = mr.Message
+			}
+		}
+
+		if resp.StatusCode == 400 {
+			return MessageResponse{}, exerr.New(ErrBadRequest, errMessage).Build()
+		}
+		if resp.StatusCode == 401 {
+			return MessageResponse{}, exerr.New(ErrAuthFailed, errMessage).Build()
+		}
+		if resp.StatusCode == 403 {
+			return MessageResponse{}, exerr.New(ErrQuota, errMessage).Build()
+		}
+		if resp.StatusCode == 500 {
+			return MessageResponse{}, exerr.New(ErrInternalServerErr, errMessage).Build()
+		}
+
+		return MessageResponse{}, exerr.New(ErrOther, errMessage).Build()
+	}
+
+}

scn/send_test.go

@@ -0,0 +1,24 @@
+package scn
+
+import (
+	"context"
+	"fmt"
+	"testing"
+)
+
+func TestSendSCN(t *testing.T) {
+	t.Skip()
+	return
+
+	mr, err := New("TODO", "TODO").
+		Message("Hello Test").
+		Content("This is a test / goext").
+		Send(context.Background())
+
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%+v\n", mr)
+
+}

sq/builder.go

@@ -0,0 +1,71 @@
+package sq
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"reflect"
+	"strings"
+)
+
+func BuildUpdateStatement(q Queryable, tableName string, obj any, idColumn string) (string, PP, error) {
+	rval := reflect.ValueOf(obj)
+	rtyp := rval.Type()
+
+	params := PP{}
+
+	setClauses := make([]string, 0)
+
+	matchClause := ""
+
+	for i := 0; i < rtyp.NumField(); i++ {
+
+		rsfield := rtyp.Field(i)
+		rvfield := rval.Field(i)
+
+		if !rsfield.IsExported() {
+			continue
+		}
+
+		columnName := rsfield.Tag.Get("db")
+		if columnName == "" || columnName == "-" {
+			continue
+		}
+
+		if idColumn == columnName {
+			idValue, err := convertValueToDB(q, rvfield.Interface())
+			if err != nil {
+				return "", nil, err
+			}
+
+			matchClause = fmt.Sprintf("(%s = :%s)", columnName, params.Add(idValue))
+
+			continue
+		}
+
+		if rsfield.Type.Kind() == reflect.Ptr && rvfield.IsNil() {
+
+			setClauses = append(setClauses, fmt.Sprintf("%s = NULL", columnName))
+
+		} else {
+
+			val, err := convertValueToDB(q, rvfield.Interface())
+			if err != nil {
+				return "", nil, err
+			}
+
+			setClauses = append(setClauses, fmt.Sprintf("(%s = :%s)", columnName, params.Add(val)))
+
+		}
+	}
+
+	if len(setClauses) == 0 {
+		return "", nil, exerr.New(exerr.TypeSQLBuild, "no updates clauses found in object").Build()
+	}
+
+	if matchClause == "" {
+		return "", nil, exerr.New(exerr.TypeSQLBuild, "id column not found in object").Build()
+	}
+
+	//goland:noinspection SqlNoDataSourceInspection
+	return fmt.Sprintf("UPDATE %s SET %s WHERE %s", tableName, strings.Join(setClauses, ", "), matchClause), params, nil
+}

sq/builder_test.go

@@ -0,0 +1,87 @@
+package sq
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"github.com/glebarez/go-sqlite"
+	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"path/filepath"
+	"testing"
+)
+
+type dummyQueryable struct {
+}
+
+func (d dummyQueryable) Exec(ctx context.Context, sql string, prep PP) (sql.Result, error) {
+	return nil, errors.New("err")
+}
+
+func (d dummyQueryable) Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error) {
+	return nil, errors.New("err")
+}
+
+func (d dummyQueryable) ListConverter() []DBTypeConverter {
+	return nil
+}
+
+func TestCreateUpdateStatement(t *testing.T) {
+
+	type request struct {
+		ID        string  `json:"id"        db:"id"`
+		Timestamp int     `json:"timestamp" db:"timestamp"`
+		StrVal    string  `json:"strVal"    db:"str_val"`
+		FloatVal  float64 `json:"floatVal"    db:"float_val"`
+		Dummy     bool    `json:"dummyBool" db:"dummy_bool"`
+		JsonVal   JsonObj `json:"jsonVal" db:"json_val"`
+	}
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb)
+	db.RegisterDefaultConverter()
+
+	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
+	tst.AssertNoErr(t, err)
+
+	sqlStr, pp, err := BuildUpdateStatement(db, "requests", request{
+		ID:        "9927",
+		Timestamp: 12321,
+		StrVal:    "hello world",
+		Dummy:     true,
+		FloatVal:  3.14159,
+		JsonVal: JsonObj{
+			"firs":   1,
+			"second": true,
+		},
+	}, "id")
+
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("\n\n")
+
+	fmt.Printf("######## PP:\n")
+	for k, v := range pp {
+		fmt.Printf("['%s']       =>       %+v\n", k, v)
+	}
+	fmt.Printf("\n\n")
+
+	fmt.Printf("######## SQL:\n%+v\n\n", sqlStr)
+
+	fmt.Printf("\n\n")
+
+}

sq/converter.go

@@ -1,20 +1,16 @@
 package sq
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
+	"reflect"
 	"time"
 )
 
-//TODO UNFINISHED
-// this is not finished
-// idea was that we can register converter in the database struct
-// they get inherited from the transactions
-// and when marshallingunmarshaling (sq.Query | sq.QueryAll)
-// or marshaling (sq.InsertSingle)
-// the types get converter automatically...
-
 type DBTypeConverter interface {
 	ModelTypeString() string
 	DBTypeString() string
@@ -40,16 +36,108 @@ var ConverterTimeToUnixMillis = NewDBTypeConverter[time.Time, int64](func(v time
 	return time.UnixMilli(v), nil
 })
 
-var ConverterOptTimeToUnixMillis = NewDBTypeConverter[*time.Time, *int64](func(v *time.Time) (*int64, error) {
-	if v == nil {
-		return nil, nil
+var ConverterRFCUnixMilliTimeToUnixMillis = NewDBTypeConverter[rfctime.UnixMilliTime, int64](func(v rfctime.UnixMilliTime) (int64, error) {
+	return v.UnixMilli(), nil
+}, func(v int64) (rfctime.UnixMilliTime, error) {
+	return rfctime.NewUnixMilli(time.UnixMilli(v)), nil
+})
+
+var ConverterRFCUnixNanoTimeToUnixNanos = NewDBTypeConverter[rfctime.UnixNanoTime, int64](func(v rfctime.UnixNanoTime) (int64, error) {
+	return v.UnixNano(), nil
+}, func(v int64) (rfctime.UnixNanoTime, error) {
+	return rfctime.NewUnixNano(time.Unix(0, v)), nil
+})
+
+var ConverterRFCUnixTimeToUnixSeconds = NewDBTypeConverter[rfctime.UnixTime, int64](func(v rfctime.UnixTime) (int64, error) {
+	return v.Unix(), nil
+}, func(v int64) (rfctime.UnixTime, error) {
+	return rfctime.NewUnix(time.Unix(v, 0)), nil
+})
+
+// ConverterRFC339TimeToString
+// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
+var ConverterRFC339TimeToString = NewDBTypeConverter[rfctime.RFC3339Time, string](func(v rfctime.RFC3339Time) (string, error) {
+	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05"), nil
+}, func(v string) (rfctime.RFC3339Time, error) {
+	t, err := time.Parse("2006-01-02 15:04:05", v)
+	if err != nil {
+		return rfctime.RFC3339Time{}, err
 	}
-	return langext.Ptr(v.UnixMilli()), nil
-}, func(v *int64) (*time.Time, error) {
-	if v == nil {
-		return nil, nil
+	return rfctime.NewRFC3339(t), nil
+})
+
+// ConverterRFC339NanoTimeToString
+// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
+var ConverterRFC339NanoTimeToString = NewDBTypeConverter[rfctime.RFC3339NanoTime, string](func(v rfctime.RFC3339NanoTime) (string, error) {
+	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05.999999999"), nil
+}, func(v string) (rfctime.RFC3339NanoTime, error) {
+	t, err := time.ParseInLocation("2006-01-02 15:04:05.999999999", v, time.UTC)
+	if err != nil {
+		return rfctime.RFC3339NanoTime{}, err
 	}
-	return langext.Ptr(time.UnixMilli(*v)), nil
+	return rfctime.NewRFC3339Nano(t), nil
+})
+
+var ConverterJsonObjToString = NewDBTypeConverter[JsonObj, string](func(v JsonObj) (string, error) {
+	mrsh, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}, func(v string) (JsonObj, error) {
+	var mrsh JsonObj
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return JsonObj{}, err
+	}
+	return mrsh, nil
+})
+
+var ConverterJsonArrToString = NewDBTypeConverter[JsonArr, string](func(v JsonArr) (string, error) {
+	mrsh, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}, func(v string) (JsonArr, error) {
+	var mrsh JsonArr
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return JsonArr{}, err
+	}
+	return mrsh, nil
+})
+
+var ConverterExErrCategoryToString = NewDBTypeConverter[exerr.ErrorCategory, string](func(v exerr.ErrorCategory) (string, error) {
+	return v.Category, nil
+}, func(v string) (exerr.ErrorCategory, error) {
+	for _, cat := range exerr.AllCategories {
+		if cat.Category == v {
+			return cat, nil
+		}
+	}
+	return exerr.CatUser, errors.New("failed to convert '" + v + "' to exerr.ErrorCategory")
+})
+
+var ConverterExErrSeverityToString = NewDBTypeConverter[exerr.ErrorSeverity, string](func(v exerr.ErrorSeverity) (string, error) {
+	return v.Severity, nil
+}, func(v string) (exerr.ErrorSeverity, error) {
+	for _, sev := range exerr.AllSeverities {
+		if sev.Severity == v {
+			return sev, nil
+		}
+	}
+	return exerr.SevErr, errors.New("failed to convert '" + v + "' to exerr.ErrorSeverity")
+})
+
+var ConverterExErrTypeToString = NewDBTypeConverter[exerr.ErrorType, string](func(v exerr.ErrorType) (string, error) {
+	return v.Key, nil
+}, func(v string) (exerr.ErrorType, error) {
+	for _, etp := range exerr.ListRegisteredTypes() {
+		if etp.Key == v {
+			return etp, nil
+		}
+	}
+
+	return exerr.NewType(v, nil), nil
 })
 
 type dbTypeConverterImpl[TModelData any, TDBData any] struct {
@@ -89,3 +177,36 @@ func NewDBTypeConverter[TModelData any, TDBData any](todb func(v TModelData) (TD
 		tomodel:         tomodel,
 	}
 }
+
+func convertValueToDB(q Queryable, value any) (any, error) {
+	modelTypeStr := fmt.Sprintf("%T", value)
+
+	for _, conv := range q.ListConverter() {
+		if conv.ModelTypeString() == modelTypeStr {
+			return conv.ModelToDB(value)
+		}
+	}
+
+	if value != nil && reflect.TypeOf(value).Kind() == reflect.Ptr {
+		vof := reflect.ValueOf(value)
+		if vof.IsNil() {
+			return nil, nil
+		} else {
+			return convertValueToDB(q, vof.Elem().Interface())
+		}
+	}
+
+	return value, nil
+}
+
+func convertValueToModel(q Queryable, value any, destinationType string) (any, error) {
+	dbTypeString := fmt.Sprintf("%T", value)
+
+	for _, conv := range q.ListConverter() {
+		if conv.ModelTypeString() == destinationType && conv.DBTypeString() == dbTypeString {
+			return conv.DBToModel(value)
+		}
+	}
+
+	return value, nil
+}

sq/database.go

@@ -4,16 +4,19 @@ import (
 	"context"
 	"database/sql"
 	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"sync"
 )
 
 type DB interface {
-	Exec(ctx context.Context, sql string, prep PP) (sql.Result, error)
-	Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error)
+	Queryable
+
 	Ping(ctx context.Context) error
 	BeginTransaction(ctx context.Context, iso sql.IsolationLevel) (Tx, error)
 	AddListener(listener Listener)
 	Exit() error
+	RegisterConverter(DBTypeConverter)
+	RegisterDefaultConverter()
 }
 
 type database struct {
@@ -21,6 +24,7 @@ type database struct {
 	txctr uint16
 	lock  sync.Mutex
 	lstr  []Listener
+	conv  []DBTypeConverter
 }
 
 func NewDB(db *sqlx.DB) DB {
@@ -120,9 +124,33 @@ func (db *database) BeginTransaction(ctx context.Context, iso sql.IsolationLevel
 		v.PostTxBegin(txid, err)
 	}
 
-	return NewTransaction(xtx, txid, db.lstr), nil
+	return NewTransaction(xtx, txid, db), nil
 }
 
 func (db *database) Exit() error {
 	return db.db.Close()
 }
+
+func (db *database) ListConverter() []DBTypeConverter {
+	return db.conv
+}
+
+func (db *database) RegisterConverter(conv DBTypeConverter) {
+	db.conv = langext.ArrFilter(db.conv, func(v DBTypeConverter) bool { return v.ModelTypeString() != conv.ModelTypeString() })
+	db.conv = append(db.conv, conv)
+}
+
+func (db *database) RegisterDefaultConverter() {
+	db.RegisterConverter(ConverterBoolToBit)
+	db.RegisterConverter(ConverterTimeToUnixMillis)
+	db.RegisterConverter(ConverterRFCUnixMilliTimeToUnixMillis)
+	db.RegisterConverter(ConverterRFCUnixNanoTimeToUnixNanos)
+	db.RegisterConverter(ConverterRFCUnixTimeToUnixSeconds)
+	db.RegisterConverter(ConverterRFC339TimeToString)
+	db.RegisterConverter(ConverterRFC339NanoTimeToString)
+	db.RegisterConverter(ConverterJsonObjToString)
+	db.RegisterConverter(ConverterJsonArrToString)
+	db.RegisterConverter(ConverterExErrCategoryToString)
+	db.RegisterConverter(ConverterExErrSeverityToString)
+	db.RegisterConverter(ConverterExErrTypeToString)
+}

sq/hasher.go

@@ -13,7 +13,9 @@ import (
 	"strings"
 )
 
-func HashSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
+// HashMattnSqliteSchema
+// use if github.com/glebarez/go-sqlite
+func HashMattnSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
 	dbdir := os.TempDir()
 	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
 
@@ -39,6 +41,34 @@ func HashSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
 	return HashSqliteDatabase(ctx, db)
 }
 
+// HashGoSqliteSchema
+// use if mattn/go-sqlite3
+func HashGoSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
+	dbdir := os.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	err := os.MkdirAll(dbdir, os.ModePerm)
+	if err != nil {
+		return "", err
+	}
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb, err := sqlx.Open("sqlite", url)
+	if err != nil {
+		return "", err
+	}
+
+	db := NewDB(xdb)
+
+	_, err = db.Exec(ctx, schemaStr, PP{})
+	if err != nil {
+		return "", err
+	}
+
+	return HashSqliteDatabase(ctx, db)
+}
+
 func HashSqliteDatabase(ctx context.Context, db Queryable) (string, error) {
 	ss, err := CreateSqliteDatabaseSchemaString(ctx, db)
 	if err != nil {
@@ -91,7 +121,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 	if err != nil {
 		return "", err
 	}
-	tableList, err := ScanAll[tabInfo](rowsTableList, SModeFast, Unsafe, true)
+	tableList, err := ScanAll[tabInfo](ctx, db, rowsTableList, SModeFast, Unsafe, true)
 	if err != nil {
 		return "", err
 	}
@@ -113,7 +143,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 				return "", err
 			}
 
-			columnList, err := ScanAll[colInfo](rowsColumnList, SModeFast, Unsafe, true)
+			columnList, err := ScanAll[colInfo](ctx, db, rowsColumnList, SModeFast, Unsafe, true)
 			if err != nil {
 				return "", err
 			}
@@ -128,7 +158,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 			if err != nil {
 				return "", err
 			}
-			idxList, err := ScanAll[idxInfo](rowsIdxList, SModeFast, Unsafe, true)
+			idxList, err := ScanAll[idxInfo](ctx, db, rowsIdxList, SModeFast, Unsafe, true)
 			if err != nil {
 				return "", err
 			}
@@ -143,7 +173,7 @@ func CreateSqliteDatabaseSchemaString(ctx context.Context, db Queryable) (string
 			if err != nil {
 				return "", err
 			}
-			fkyList, err := ScanAll[fkyInfo](rowsIdxList, SModeFast, Unsafe, true)
+			fkyList, err := ScanAll[fkyInfo](ctx, db, rowsIdxList, SModeFast, Unsafe, true)
 			if err != nil {
 				return "", err
 			}

sq/json.go

@@ -0,0 +1,5 @@
+package sq
+
+type JsonObj map[string]any
+
+type JsonArr []any

sq/params.go

@@ -1,5 +1,7 @@
 package sq
 
+import "gogs.mikescher.com/BlackForestBytes/goext/langext"
+
 type PP map[string]any
 
 func Join(pps ...PP) PP {
@@ -11,3 +13,13 @@ func Join(pps ...PP) PP {
 	}
 	return r
 }
+
+func (pp *PP) Add(v any) string {
+	id := PPID()
+	(*pp)[id] = v
+	return id
+}
+
+func PPID() string {
+	return "p_" + langext.RandBase62(8)
+}

sq/queryable.go

@@ -9,4 +9,5 @@ import (
 type Queryable interface {
 	Exec(ctx context.Context, sql string, prep PP) (sql.Result, error)
 	Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error)
+	ListConverter() []DBTypeConverter
 }

sq/scanner.go

@@ -13,8 +13,8 @@ import (
 type StructScanMode string
 
 const (
-	SModeFast     StructScanMode = "FAST"
-	SModeExtended StructScanMode = "EXTENDED"
+	SModeFast     StructScanMode = "FAST"     // Use default sq.Scan, does not work with joined/resolved types and/or custom value converter
+	SModeExtended StructScanMode = "EXTENDED" // Fully featured perhaps (?) a tiny bit slower - default
 )
 
 type StructScanSafety string
@@ -51,7 +51,13 @@ func InsertSingle[TData any](ctx context.Context, q Queryable, tableName string,
 
 		columns = append(columns, "\""+columnName+"\"")
 		params = append(params, ":"+paramkey)
-		pp[paramkey] = rvfield.Interface()
+
+		val, err := convertValueToDB(q, rvfield.Interface())
+		if err != nil {
+			return nil, err
+		}
+
+		pp[paramkey] = val
 
 	}
 
@@ -71,7 +77,7 @@ func QuerySingle[TData any](ctx context.Context, q Queryable, sql string, pp PP,
 		return *new(TData), err
 	}
 
-	data, err := ScanSingle[TData](rows, mode, sec, true)
+	data, err := ScanSingle[TData](ctx, q, rows, mode, sec, true)
 	if err != nil {
 		return *new(TData), err
 	}
@@ -85,7 +91,7 @@ func QueryAll[TData any](ctx context.Context, q Queryable, sql string, pp PP, mo
 		return nil, err
 	}
 
-	data, err := ScanAll[TData](rows, mode, sec, true)
+	data, err := ScanAll[TData](ctx, q, rows, mode, sec, true)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +99,7 @@ func QueryAll[TData any](ctx context.Context, q Queryable, sql string, pp PP, mo
 	return data, nil
 }
 
-func ScanSingle[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool) (TData, error) {
+func ScanSingle[TData any](ctx context.Context, q Queryable, rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool) (TData, error) {
 	if rows.Next() {
 		var strscan *StructScanner
 
@@ -123,7 +129,7 @@ func ScanSingle[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanS
 				return *new(TData), err
 			}
 		} else if mode == SModeExtended {
-			err := strscan.StructScanExt(&data)
+			err := strscan.StructScanExt(q, &data)
 			if err != nil {
 				return *new(TData), err
 			}
@@ -149,6 +155,10 @@ func ScanSingle[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanS
 			return *new(TData), err
 		}
 
+		if err := ctx.Err(); err != nil {
+			return *new(TData), err
+		}
+
 		return data, nil
 
 	} else {
@@ -159,7 +169,7 @@ func ScanSingle[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanS
 	}
 }
 
-func ScanAll[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool) ([]TData, error) {
+func ScanAll[TData any](ctx context.Context, q Queryable, rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool) ([]TData, error) {
 	var strscan *StructScanner
 
 	if sec == Safe {
@@ -182,6 +192,11 @@ func ScanAll[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanSafe
 
 	res := make([]TData, 0)
 	for rows.Next() {
+
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+
 		if mode == SModeFast {
 			var data TData
 			err := strscan.StructScanBase(&data)
@@ -191,7 +206,7 @@ func ScanAll[TData any](rows *sqlx.Rows, mode StructScanMode, sec StructScanSafe
 			res = append(res, data)
 		} else if mode == SModeExtended {
 			var data TData
-			err := strscan.StructScanExt(&data)
+			err := strscan.StructScanExt(q, &data)
 			if err != nil {
 				return nil, err
 			}

sq/sq_test.go

@@ -0,0 +1,95 @@
+package sq
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/glebarez/go-sqlite"
+	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestTypeConverter1(t *testing.T) {
+	type RequestData struct {
+		ID        string    `db:"id"`
+		Timestamp time.Time `db:"timestamp"`
+	}
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	tst.AssertNoErr(t, os.MkdirAll(dbdir, os.ModePerm))
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb)
+	db.RegisterDefaultConverter()
+
+	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
+	tst.AssertNoErr(t, err)
+
+	_, err = InsertSingle(ctx, db, "requests", RequestData{
+		ID:        "001",
+		Timestamp: time.Date(2000, 06, 15, 12, 0, 0, 0, time.UTC),
+	})
+	tst.AssertNoErr(t, err)
+}
+
+func TestTypeConverter2(t *testing.T) {
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	type RequestData struct {
+		ID        string                `db:"id"`
+		Timestamp rfctime.UnixMilliTime `db:"timestamp"`
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	tst.AssertNoErr(t, os.MkdirAll(dbdir, os.ModePerm))
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb)
+	db.RegisterDefaultConverter()
+
+	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
+	tst.AssertNoErr(t, err)
+
+	t0 := rfctime.NewUnixMilli(time.Date(2012, 03, 01, 16, 0, 0, 0, time.UTC))
+
+	_, err = InsertSingle(ctx, db, "requests", RequestData{
+		ID:        "002",
+		Timestamp: t0,
+	})
+	tst.AssertNoErr(t, err)
+
+	r, err := QuerySingle[RequestData](ctx, db, "SELECT * FROM requests WHERE id = '002'", PP{}, SModeExtended, Safe)
+	tst.AssertNoErr(t, err)
+
+	fmt.Printf("%+v\n", r)
+
+	tst.AssertEqual(t, "002", r.ID)
+	tst.AssertEqual(t, t0.UnixNano(), r.Timestamp.UnixNano())
+}

sq/structscanner.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/jmoiron/sqlx"
 	"github.com/jmoiron/sqlx/reflectx"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 )
 
@@ -15,9 +16,10 @@ type StructScanner struct {
 	Mapper *reflectx.Mapper
 	unsafe bool
 
-	fields  [][]int
-	values  []any
-	columns []string
+	fields    [][]int
+	values    []any
+	converter []DBTypeConverter
+	columns   []string
 }
 
 func NewStructScanner(rows *sqlx.Rows, unsafe bool) *StructScanner {
@@ -47,13 +49,15 @@ func (r *StructScanner) Start(dest any) error {
 		return fmt.Errorf("missing destination name %s in %T", columns[f], dest)
 	}
 	r.values = make([]interface{}, len(columns))
+	r.converter = make([]DBTypeConverter, len(columns))
 
 	return nil
 }
 
 // StructScanExt forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
-// does also wok with nullabel structs (from LEFT JOIN's)
-func (r *StructScanner) StructScanExt(dest any) error {
+// does also work with nullabel structs (from LEFT JOIN's)
+// does also work with custom value converters
+func (r *StructScanner) StructScanExt(q Queryable, dest any) error {
 	v := reflect.ValueOf(dest)
 
 	if v.Kind() != reflect.Ptr {
@@ -64,7 +68,7 @@ func (r *StructScanner) StructScanExt(dest any) error {
 
 	v = v.Elem()
 
-	err := fieldsByTraversalExtended(v, r.fields, r.values)
+	err := fieldsByTraversalExtended(q, v, r.fields, r.values, r.converter)
 	if err != nil {
 		return err
 	}
@@ -131,7 +135,6 @@ func (r *StructScanner) StructScanExt(dest any) error {
 
 		val1 := reflect.ValueOf(r.values[i])
 		val2 := val1.Elem()
-		val3 := val2.Elem()
 
 		if val2.IsNil() {
 			if f.Kind() != reflect.Pointer {
@@ -140,7 +143,16 @@ func (r *StructScanner) StructScanExt(dest any) error {
 
 			f.Set(reflect.Zero(f.Type())) // set to nil
 		} else {
-			f.Set(val3)
+			if r.converter[i] != nil {
+				val3 := val2.Elem().Interface()
+				conv3, err := r.converter[i].DBToModel(val3)
+				if err != nil {
+					return err
+				}
+				f.Set(reflect.ValueOf(conv3))
+			} else {
+				f.Set(val2.Elem())
+			}
 		}
 
 	}
@@ -172,7 +184,7 @@ func (r *StructScanner) StructScanBase(dest any) error {
 }
 
 // fieldsByTraversal forked from github.com/jmoiron/sqlx@v1.3.5/sqlx.go
-func fieldsByTraversalExtended(v reflect.Value, traversals [][]int, values []interface{}) error {
+func fieldsByTraversalExtended(q Queryable, v reflect.Value, traversals [][]int, values []interface{}, converter []DBTypeConverter) error {
 	v = reflect.Indirect(v)
 	if v.Kind() != reflect.Struct {
 		return errors.New("argument not a struct")
@@ -185,7 +197,23 @@ func fieldsByTraversalExtended(v reflect.Value, traversals [][]int, values []int
 		}
 		f := reflectx.FieldByIndexes(v, traversal)
 
-		values[i] = reflect.New(reflect.PointerTo(f.Type())).Interface()
+		typeStr := f.Type().String()
+
+		foundConverter := false
+		for _, conv := range q.ListConverter() {
+			if conv.ModelTypeString() == typeStr {
+				_v := langext.Ptr[any](nil)
+				values[i] = _v
+				foundConverter = true
+				converter[i] = conv
+				break
+			}
+		}
+
+		if !foundConverter {
+			values[i] = reflect.New(reflect.PointerTo(f.Type())).Interface()
+			converter[i] = nil
+		}
 	}
 	return nil
 }

sq/transaction.go

@@ -17,35 +17,35 @@ const (
 )
 
 type Tx interface {
+	Queryable
+
 	Rollback() error
 	Commit() error
 	Status() TxStatus
-	Exec(ctx context.Context, sql string, prep PP) (sql.Result, error)
-	Query(ctx context.Context, sql string, prep PP) (*sqlx.Rows, error)
 }
 
 type transaction struct {
 	tx       *sqlx.Tx
 	id       uint16
-	lstr     []Listener
 	status   TxStatus
 	execCtr  int
 	queryCtr int
+	db       *database
 }
 
-func NewTransaction(xtx *sqlx.Tx, txid uint16, lstr []Listener) Tx {
+func NewTransaction(xtx *sqlx.Tx, txid uint16, db *database) Tx {
 	return &transaction{
 		tx:       xtx,
 		id:       txid,
-		lstr:     lstr,
 		status:   TxStatusInitial,
 		execCtr:  0,
 		queryCtr: 0,
+		db:       db,
 	}
 }
 
 func (tx *transaction) Rollback() error {
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		err := v.PreTxRollback(tx.id)
 		if err != nil {
 			return err
@@ -58,7 +58,7 @@ func (tx *transaction) Rollback() error {
 		tx.status = TxStatusRollback
 	}
 
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		v.PostTxRollback(tx.id, result)
 	}
 
@@ -66,7 +66,7 @@ func (tx *transaction) Rollback() error {
 }
 
 func (tx *transaction) Commit() error {
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		err := v.PreTxCommit(tx.id)
 		if err != nil {
 			return err
@@ -79,7 +79,7 @@ func (tx *transaction) Commit() error {
 		tx.status = TxStatusComitted
 	}
 
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		v.PostTxRollback(tx.id, result)
 	}
 
@@ -88,7 +88,7 @@ func (tx *transaction) Commit() error {
 
 func (tx *transaction) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Result, error) {
 	origsql := sqlstr
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		err := v.PreExec(ctx, langext.Ptr(tx.id), &sqlstr, &prep)
 		if err != nil {
 			return nil, err
@@ -101,7 +101,7 @@ func (tx *transaction) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Re
 		tx.status = TxStatusActive
 	}
 
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		v.PostExec(langext.Ptr(tx.id), origsql, sqlstr, prep)
 	}
 
@@ -113,7 +113,7 @@ func (tx *transaction) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Re
 
 func (tx *transaction) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx.Rows, error) {
 	origsql := sqlstr
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		err := v.PreQuery(ctx, langext.Ptr(tx.id), &sqlstr, &prep)
 		if err != nil {
 			return nil, err
@@ -126,7 +126,7 @@ func (tx *transaction) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx
 		tx.status = TxStatusActive
 	}
 
-	for _, v := range tx.lstr {
+	for _, v := range tx.db.lstr {
 		v.PostQuery(langext.Ptr(tx.id), origsql, sqlstr, prep)
 	}
 
@@ -140,6 +140,10 @@ func (tx *transaction) Status() TxStatus {
 	return tx.status
 }
 
+func (tx *transaction) ListConverter() []DBTypeConverter {
+	return tx.db.conv
+}
+
 func (tx *transaction) Traffic() (int, int) {
 	return tx.execCtr, tx.queryCtr
 }

wmo/collection.go

@@ -45,7 +45,7 @@ type Coll[TData any] struct {
 	customDecoder       *func(ctx context.Context, dec Decodable) (TData, error) // custom decoding function (useful if TData is an interface)
 	isInterfaceDataType bool                                                     // true if TData is an interface (not a struct)
 	unmarshalHooks      []func(d TData) TData                                    // called for every object after unmarshalling
-	extraModPipeline    mongo.Pipeline                                           // appended to pipelines after filter/limit/skip/sort, used for $lookup, $set, $unset, $project, etc
+	extraModPipeline    []func(ctx context.Context) mongo.Pipeline               // appended to pipelines after filter/limit/skip/sort, used for $lookup, $set, $unset, $project, etc
 }
 
 func (c *Coll[TData]) Collection() *mongo.Collection {
@@ -83,7 +83,13 @@ func (c *Coll[TData]) WithUnmarshalHook(fn func(d TData) TData) *Coll[TData] {
 }
 
 func (c *Coll[TData]) WithModifyingPipeline(p mongo.Pipeline) *Coll[TData] {
-	c.extraModPipeline = append(c.extraModPipeline, p...)
+	c.extraModPipeline = append(c.extraModPipeline, func(ctx context.Context) mongo.Pipeline { return p })
+
+	return c
+}
+
+func (c *Coll[TData]) WithModifyingPipelineFunc(fn func(ctx context.Context) mongo.Pipeline) *Coll[TData] {
+	c.extraModPipeline = append(c.extraModPipeline, fn)
 
 	return c
 }

wmo/queryAggregate.go

@@ -10,7 +10,9 @@ import (
 
 func (c *Coll[TData]) Aggregate(ctx context.Context, pipeline mongo.Pipeline, opts ...*options.AggregateOptions) ([]TData, error) {
 
-	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+	for _, ppl := range c.extraModPipeline {
+		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
+	}
 
 	cursor, err := c.coll.Aggregate(ctx, pipeline, opts...)
 	if err != nil {
@@ -27,7 +29,9 @@ func (c *Coll[TData]) Aggregate(ctx context.Context, pipeline mongo.Pipeline, op
 
 func (c *Coll[TData]) AggregateOneOpt(ctx context.Context, pipeline mongo.Pipeline, opts ...*options.AggregateOptions) (*TData, error) {
 
-	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+	for _, ppl := range c.extraModPipeline {
+		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
+	}
 
 	cursor, err := c.coll.Aggregate(ctx, pipeline, opts...)
 	if err != nil {
@@ -47,7 +51,9 @@ func (c *Coll[TData]) AggregateOneOpt(ctx context.Context, pipeline mongo.Pipeli
 
 func (c *Coll[TData]) AggregateOne(ctx context.Context, pipeline mongo.Pipeline, opts ...*options.AggregateOptions) (TData, error) {
 
-	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+	for _, ppl := range c.extraModPipeline {
+		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
+	}
 
 	cursor, err := c.coll.Aggregate(ctx, pipeline, opts...)
 	if err != nil {

wmo/queryFind.go

@@ -32,7 +32,9 @@ func (c *Coll[TData]) Find(ctx context.Context, filter bson.M, opts ...*options.
 		}
 	}
 
-	pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+	for _, ppl := range c.extraModPipeline {
+		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
+	}
 
 	for _, opt := range opts {
 		if opt != nil && opt.Projection != nil {

wmo/queryFindOne.go

@@ -71,7 +71,9 @@ func (c *Coll[TData]) findOneInternal(ctx context.Context, filter bson.M, allowN
 		pipeline = append(pipeline, bson.D{{Key: "$match", Value: filter}})
 		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: 1}})
 
-		pipeline = langext.ArrConcat(pipeline, c.extraModPipeline)
+		for _, ppl := range c.extraModPipeline {
+			pipeline = langext.ArrConcat(pipeline, ppl(ctx))
+		}
 
 		cursor, err := c.coll.Aggregate(ctx, pipeline)
 		if err != nil {

wmo/queryList.go

@@ -6,6 +6,7 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	ct "gogs.mikescher.com/BlackForestBytes/goext/cursortoken"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 )
 
 func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int, inTok ct.CursorToken) ([]TData, ct.CursorToken, error) {
@@ -50,7 +51,10 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 	}
 
 	pipeline = append(pipeline, paginationPipeline...)
-	pipeline = append(pipeline, c.extraModPipeline...)
+
+	for _, ppl := range c.extraModPipeline {
+		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
+	}
 
 	cursor, err := c.coll.Aggregate(ctx, pipeline)
 	if err != nil {

wmo/queryPaginate.go

@@ -42,7 +42,12 @@ func (c *Coll[TData]) Paginate(ctx context.Context, filter pag.Filter, page int,
 	pipelineCount := mongo.Pipeline{}
 	pipelineCount = append(pipelineCount, bson.D{{Key: "$count", Value: "count"}})
 
-	pipelineList := langext.ArrConcat(pipelineFilter, pipelineSort, pipelinePaginate, c.extraModPipeline, pipelineSort)
+	extrModPipelineResolved := mongo.Pipeline{}
+	for _, ppl := range c.extraModPipeline {
+		extrModPipelineResolved = langext.ArrConcat(extrModPipelineResolved, ppl(ctx))
+	}
+
+	pipelineList := langext.ArrConcat(pipelineFilter, pipelineSort, pipelinePaginate, extrModPipelineResolved, pipelineSort)
 	pipelineTotalCount := langext.ArrConcat(pipelineFilter, pipelineCount)
 
 	cursorList, err := c.coll.Aggregate(ctx, pipelineList)