v0.0.445 added CtxData() and ExtendGinMeta/ExtendContextMeta to exerr

bfcodegen/id-generate.go

@@ -38,7 +38,7 @@ var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator =
 //go:embed id-generate.template
 var templateIDGenerateText string
 
-func GenerateIDSpecs(sourceDir string, destFile string, opt *IDGenOptions) error {
+func GenerateIDSpecs(sourceDir string, destFile string, opt IDGenOptions) error {
 
 	debugOutput := langext.Coalesce(opt.DebugOutput, false)
 

bfcodegen/id-generate_test.go

@@ -34,10 +34,10 @@ func TestGenerateIDSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", &IDGenOptions{DebugOutput: langext.PTrue})
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", IDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
-	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", &IDGenOptions{DebugOutput: langext.PTrue})
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", IDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

dataext/optional.go

@@ -0,0 +1,59 @@
+package dataext
+
+import (
+	"encoding/json"
+	"errors"
+)
+
+type JsonOpt[T any] struct {
+	isSet bool
+	value T
+}
+
+// MarshalJSON returns m as the JSON encoding of m.
+func (m JsonOpt[T]) MarshalJSON() ([]byte, error) {
+	if !m.isSet {
+		return []byte("null"), nil // actually this would be undefined - but undefined is not valid JSON
+	}
+
+	return json.Marshal(m.value)
+}
+
+// UnmarshalJSON sets *m to a copy of data.
+func (m *JsonOpt[T]) UnmarshalJSON(data []byte) error {
+	if m == nil {
+		return errors.New("JsonOpt: UnmarshalJSON on nil pointer")
+	}
+
+	m.isSet = true
+	return json.Unmarshal(data, &m.value)
+}
+
+func (m JsonOpt[T]) IsSet() bool {
+	return m.isSet
+}
+
+func (m JsonOpt[T]) IsUnset() bool {
+	return !m.isSet
+}
+
+func (m JsonOpt[T]) Value() (T, bool) {
+	if !m.isSet {
+		return *new(T), false
+	}
+	return m.value, true
+}
+
+func (m JsonOpt[T]) ValueOrNil() *T {
+	if !m.isSet {
+		return nil
+	}
+	return &m.value
+}
+
+func (m JsonOpt[T]) MustValue() T {
+	if !m.isSet {
+		panic("value not set")
+	}
+	return m.value
+}

exerr/builder.go

@@ -68,10 +68,11 @@ func init() {
 }
 
 type Builder struct {
-	wrappedErr      error
-	errorData       *ExErr
-	containsGinData bool
-	noLog           bool
+	wrappedErr          error
+	errorData           *ExErr
+	containsGinData     bool
+	containsContextData bool
+	noLog               bool
 }
 
 func Get(err error) *Builder {
@@ -309,27 +310,27 @@ func (b *Builder) Errs(key string, val []error) *Builder {
 func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request) *Builder {
 	if v := ctx.Value("start_timestamp"); v != nil {
 		if t, ok := v.(time.Time); ok {
-			b.Time("ctx.startTimestamp", t)
-			b.Time("ctx.endTimestamp", time.Now())
+			b.Time("ctx_startTimestamp", t)
+			b.Time("ctx_endTimestamp", time.Now())
 		}
 	}
-	b.Str("gin.method", req.Method)
-	b.Str("gin.path", g.FullPath())
-	b.Strs("gin.header", extractHeader(g.Request.Header))
+	b.Str("gin_method", req.Method)
+	b.Str("gin_path", g.FullPath())
+	b.Strs("gin_header", extractHeader(g.Request.Header))
 	if req.URL != nil {
-		b.Str("gin.url", req.URL.String())
+		b.Str("gin_url", req.URL.String())
 	}
 	if ctxVal := g.GetString("apiversion"); ctxVal != "" {
-		b.Str("gin.context.apiversion", ctxVal)
+		b.Str("gin_context_apiversion", ctxVal)
 	}
 	if ctxVal := g.GetString("uid"); ctxVal != "" {
-		b.Str("gin.context.uid", ctxVal)
+		b.Str("gin_context_uid", ctxVal)
 	}
 	if ctxVal := g.GetString("fcmId"); ctxVal != "" {
-		b.Str("gin.context.fcmid", ctxVal)
+		b.Str("gin_context_fcmid", ctxVal)
 	}
 	if ctxVal := g.GetString("reqid"); ctxVal != "" {
-		b.Str("gin.context.reqid", ctxVal)
+		b.Str("gin_context_reqid", ctxVal)
 	}
 	if req.Method != "GET" && req.Body != nil {
 
@@ -340,12 +341,12 @@ func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request)
 						var prettyJSON bytes.Buffer
 						err = json.Indent(&prettyJSON, bin, "", "  ")
 						if err == nil {
-							b.Str("gin.body", string(prettyJSON.Bytes()))
+							b.Str("gin_body", string(prettyJSON.Bytes()))
 						} else {
-							b.Bytes("gin.body", bin)
+							b.Bytes("gin_body", bin)
 						}
 					} else {
-						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+						b.Str("gin_body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
 					}
 				}
 			}
@@ -355,9 +356,9 @@ func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request)
 			if brc, ok := req.Body.(dataext.BufferedReadCloser); ok {
 				if bin, err := brc.BufferedAll(); err == nil {
 					if len(bin) < 16*1024 {
-						b.Bytes("gin.body", bin)
+						b.Bytes("gin_body", bin)
 					} else {
-						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+						b.Str("gin_body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
 					}
 				}
 			}
@@ -365,10 +366,20 @@ func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request)
 
 	}
 
+	pkgconfig.ExtendGinMeta(ctx, b, g, req)
+
 	b.containsGinData = true
 	return b
 }
 
+func (b *Builder) CtxData(method Method, ctx context.Context) *Builder {
+	pkgconfig.ExtendContextMeta(b, method, ctx)
+
+	b.containsContextData = true
+
+	return b
+}
+
 func formatHeader(header map[string][]string) string {
 	ml := 1
 	for k, _ := range header {
@@ -412,9 +423,13 @@ func extractHeader(header map[string][]string) []string {
 // If the errors is not SevWarn or SevInfo it gets also logged (in short form, without stacktrace) onto stdout
 // Can be gloablly configured with ZeroLogErrTraces and ZeroLogAllTraces
 // Can be locally suppressed with Builder.NoLog()
-func (b *Builder) Build() error {
+func (b *Builder) Build(ctxs ...context.Context) error {
 	warnOnPkgConfigNotInitialized()
 
+	for _, dctx := range ctxs {
+		b.CtxData(MethodBuild, dctx)
+	}
+
 	if pkgconfig.DisableErrorWrapping && b.wrappedErr != nil {
 		return b.wrappedErr
 	}
@@ -439,11 +454,13 @@ func (b *Builder) Output(ctx context.Context, g *gin.Context) {
 		b.GinReq(ctx, g, g.Request)
 	}
 
+	b.CtxData(MethodOutput, ctx)
+
 	b.errorData.Output(g)
 
-	if b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal {
+	if (b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal) && (pkgconfig.ZeroLogErrGinOutput || pkgconfig.ZeroLogAllGinOutput) {
 		b.errorData.Log(stackSkipLogger.Error())
-	} else if b.errorData.Severity == SevWarn {
+	} else if (b.errorData.Severity == SevWarn) && (pkgconfig.ZeroLogAllGinOutput) {
 		b.errorData.Log(stackSkipLogger.Warn())
 	}
 
@@ -452,7 +469,13 @@ func (b *Builder) Output(ctx context.Context, g *gin.Context) {
 
 // Print prints the error
 // If the error is SevErr we also send it to the error-service
-func (b *Builder) Print() {
+func (b *Builder) Print(ctxs ...context.Context) {
+	warnOnPkgConfigNotInitialized()
+
+	for _, dctx := range ctxs {
+		b.CtxData(MethodPrint, dctx)
+	}
+
 	if b.errorData.Severity == SevErr || b.errorData.Severity == SevFatal {
 		b.errorData.Log(stackSkipLogger.Error())
 	} else if b.errorData.Severity == SevWarn {
@@ -468,7 +491,12 @@ func (b *Builder) Format(level LogPrintLevel) string {
 
 // Fatal prints the error and terminates the program
 // If the error is SevErr we also send it to the error-service
-func (b *Builder) Fatal() {
+func (b *Builder) Fatal(ctxs ...context.Context) {
+
+	for _, dctx := range ctxs {
+		b.CtxData(MethodFatal, dctx)
+	}
+
 	b.errorData.Severity = SevFatal
 	b.errorData.Log(stackSkipLogger.WithLevel(zerolog.FatalLevel))
 

exerr/constructor.go

@@ -181,7 +181,7 @@ func getReflectedMetaValues(value interface{}, remainingDepth int) map[string]Me
 
 	jsonval, err := json.Marshal(value)
 	if err != nil {
-		panic(err) // gets recovered later up
+		return map[string]MetaValue{"": {DataType: MDTString, Value: fmt.Sprintf("Failed to Marshal %T:\n%+v", value, value)}}
 	}
 
 	return map[string]MetaValue{"": {DataType: MDTString, Value: string(jsonval)}}

exerr/errinit.go

@@ -1,19 +1,26 @@
 package exerr
 
 import (
+	"context"
 	"fmt"
+	"github.com/gin-gonic/gin"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"net/http"
 )
 
 type ErrorPackageConfig struct {
-	ZeroLogErrTraces       bool                                             // autom print zerolog logs on .Build()  (for SevErr and SevFatal)
-	ZeroLogAllTraces       bool                                             // autom print zerolog logs on .Build()  (for all Severities)
-	RecursiveErrors        bool                                             // errors contains their Origin-Error
-	ExtendedGinOutput      bool                                             // Log extended data (trace, meta, ...) to gin in err.Output()
-	IncludeMetaInGinOutput bool                                             // Log meta fields ( from e.g. `.Str(key, val).Build()` ) to gin in err.Output()
-	ExtendGinOutput        func(err *ExErr, json map[string]any)            // (Optionally) extend the gin output with more fields
-	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any) // (Optionally) extend the gin `__data` output with more fields
-	DisableErrorWrapping   bool                                             // Disables the exerr.Wrap()...Build() function - will always return the original error
+	ZeroLogErrTraces       bool                                                                     // autom print zerolog logs on .Build()  (for SevErr and SevFatal)
+	ZeroLogAllTraces       bool                                                                     // autom print zerolog logs on .Build()  (for all Severities)
+	RecursiveErrors        bool                                                                     // errors contains their Origin-Error
+	ExtendedGinOutput      bool                                                                     // Log extended data (trace, meta, ...) to gin in err.Output()
+	IncludeMetaInGinOutput bool                                                                     // Log meta fields ( from e.g. `.Str(key, val).Build()` ) to gin in err.Output()
+	ExtendGinOutput        func(err *ExErr, json map[string]any)                                    // (Optionally) extend the gin output with more fields
+	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any)                         // (Optionally) extend the gin `__data` output with more fields
+	DisableErrorWrapping   bool                                                                     // Disables the exerr.Wrap()...Build() function - will always return the original error
+	ZeroLogErrGinOutput    bool                                                                     // autom print zerolog logs on ginext.Error() / .Output(gin)  (for SevErr and SevFatal)
+	ZeroLogAllGinOutput    bool                                                                     // autom print zerolog logs on ginext.Error() / .Output(gin)  (for all Severities)
+	ExtendGinMeta          func(ctx context.Context, b *Builder, g *gin.Context, req *http.Request) // (Optionally) extend the final error meta values with additional data from the gin context (a few are automatically added, here more can be included)
+	ExtendContextMeta      func(b *Builder, method Method, dctx context.Context)                    // (Optionally) extend the final error meta values with additional data from the context (a few are automatically added, here more can be included)
 }
 
 type ErrorPackageConfigInit struct {
@@ -25,6 +32,10 @@ type ErrorPackageConfigInit struct {
 	ExtendGinOutput        func(err *ExErr, json map[string]any)
 	ExtendGinDataOutput    func(err *ExErr, depth int, json map[string]any)
 	DisableErrorWrapping   *bool
+	ZeroLogErrGinOutput    *bool
+	ZeroLogAllGinOutput    *bool
+	ExtendGinMeta          func(ctx context.Context, b *Builder, g *gin.Context, req *http.Request)
+	ExtendContextMeta      func(b *Builder, method Method, dctx context.Context)
 }
 
 var initialized = false
@@ -38,6 +49,10 @@ var pkgconfig = ErrorPackageConfig{
 	ExtendGinOutput:        func(err *ExErr, json map[string]any) {},
 	ExtendGinDataOutput:    func(err *ExErr, depth int, json map[string]any) {},
 	DisableErrorWrapping:   false,
+	ZeroLogErrGinOutput:    true,
+	ZeroLogAllGinOutput:    false,
+	ExtendGinMeta:          func(ctx context.Context, b *Builder, g *gin.Context, req *http.Request) {},
+	ExtendContextMeta:      func(b *Builder, method Method, dctx context.Context) {},
 }
 
 // Init initializes the exerr packages
@@ -50,6 +65,8 @@ func Init(cfg ErrorPackageConfigInit) {
 
 	ego := func(err *ExErr, json map[string]any) {}
 	egdo := func(err *ExErr, depth int, json map[string]any) {}
+	egm := func(ctx context.Context, b *Builder, g *gin.Context, req *http.Request) {}
+	egcm := func(b *Builder, method Method, dctx context.Context) {}
 
 	if cfg.ExtendGinOutput != nil {
 		ego = cfg.ExtendGinOutput
@@ -57,6 +74,12 @@ func Init(cfg ErrorPackageConfigInit) {
 	if cfg.ExtendGinDataOutput != nil {
 		egdo = cfg.ExtendGinDataOutput
 	}
+	if cfg.ExtendGinMeta != nil {
+		egm = cfg.ExtendGinMeta
+	}
+	if cfg.ExtendContextMeta != nil {
+		egcm = cfg.ExtendContextMeta
+	}
 
 	pkgconfig = ErrorPackageConfig{
 		ZeroLogErrTraces:       langext.Coalesce(cfg.ZeroLogErrTraces, pkgconfig.ZeroLogErrTraces),
@@ -67,6 +90,10 @@ func Init(cfg ErrorPackageConfigInit) {
 		ExtendGinOutput:        ego,
 		ExtendGinDataOutput:    egdo,
 		DisableErrorWrapping:   langext.Coalesce(cfg.DisableErrorWrapping, pkgconfig.DisableErrorWrapping),
+		ZeroLogAllGinOutput:    langext.Coalesce(cfg.ZeroLogAllGinOutput, pkgconfig.ZeroLogAllGinOutput),
+		ZeroLogErrGinOutput:    langext.Coalesce(cfg.ZeroLogErrGinOutput, pkgconfig.ZeroLogErrGinOutput),
+		ExtendGinMeta:          egm,
+		ExtendContextMeta:      egcm,
 	}
 
 	initialized = true

exerr/gin.go

@@ -15,10 +15,10 @@ func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) la
 		ginJson["id"] = ee.UniqueID
 	}
 	if ee.Category != CatWrap {
-		ginJson["category"] = ee.Category
+		ginJson["category"] = ee.Category.Category
 	}
 	if ee.Type != TypeWrap {
-		ginJson["type"] = ee.Type
+		ginJson["type"] = ee.Type.Key
 	}
 	if ee.StatusCode != nil {
 		ginJson["statuscode"] = ee.StatusCode
@@ -30,7 +30,7 @@ func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) la
 		ginJson["caller"] = ee.Caller
 	}
 	if ee.Severity != SevErr {
-		ginJson["severity"] = ee.Severity
+		ginJson["severity"] = ee.Severity.Severity
 	}
 	if ee.Timestamp != (time.Time{}) {
 		ginJson["time"] = ee.Timestamp.Format(time.RFC3339)

ginext/engine.go

@@ -19,6 +19,7 @@ type GinWrapper struct {
 	engine          *gin.Engine
 	suppressGinLogs bool
 
+	opt                   Options
 	allowCors             bool
 	ginDebug              bool
 	bufferBody            bool
@@ -37,12 +38,15 @@ type ginRouteSpec struct {
 }
 
 type Options struct {
-	AllowCors             *bool                                     // Add cors handler to allow all CORS requests on the default http methods
-	GinDebug              *bool                                     // Set gin.debug to true (adds more logs)
-	BufferBody            *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
-	Timeout               *time.Duration                            // The default handler timeout
-	ListenerBeforeRequest []func(g *gin.Context)                    // Register listener that are called before the handler method
-	ListenerAfterRequest  []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+	AllowCors                *bool                                     // Add cors handler to allow all CORS requests on the default http methods
+	GinDebug                 *bool                                     // Set gin.debug to true (adds more logs)
+	SuppressGinLogs          *bool                                     // Suppress our custom gin logs (even if GinDebug == true)
+	BufferBody               *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
+	Timeout                  *time.Duration                            // The default handler timeout
+	ListenerBeforeRequest    []func(g *gin.Context)                    // Register listener that are called before the handler method
+	ListenerAfterRequest     []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+	DebugTrimHandlerPrefixes []string                                  // Trim these prefixes from the handler names in the debug print
+	DebugReplaceHandlerNames map[string]string                         // Replace handler names in debug output
 }
 
 // NewEngine creates a new (wrapped) ginEngine
@@ -51,7 +55,8 @@ func NewEngine(opt Options) *GinWrapper {
 
 	wrapper := &GinWrapper{
 		engine:                engine,
-		suppressGinLogs:       false,
+		opt:                   opt,
+		suppressGinLogs:       langext.Coalesce(opt.SuppressGinLogs, false),
 		allowCors:             langext.Coalesce(opt.AllowCors, false),
 		ginDebug:              langext.Coalesce(opt.GinDebug, true),
 		bufferBody:            langext.Coalesce(opt.BufferBody, false),
@@ -73,12 +78,12 @@ func NewEngine(opt Options) *GinWrapper {
 	if !wrapper.ginDebug {
 		gin.SetMode(gin.ReleaseMode)
 
-		ginlogger := gin.Logger()
-		engine.Use(func(context *gin.Context) {
-			if !wrapper.suppressGinLogs {
+		if !wrapper.suppressGinLogs {
+			ginlogger := gin.Logger()
+			engine.Use(func(context *gin.Context) {
 				ginlogger(context)
-			}
-		})
+			})
+		}
 	} else {
 		gin.SetMode(gin.DebugMode)
 	}
@@ -184,6 +189,18 @@ func (w *GinWrapper) cleanMiddlewareName(fname string) string {
 		}
 	}
 
+	for _, pfx := range w.opt.DebugTrimHandlerPrefixes {
+		if strings.HasPrefix(fname, pfx) {
+			fname = fname[len(pfx):]
+		}
+	}
+
+	for k, v := range langext.ForceMap(w.opt.DebugReplaceHandlerNames) {
+		if strings.EqualFold(fname, k) {
+			fname = v
+		}
+	}
+
 	return fname
 }
 
@@ -193,3 +210,9 @@ func (w *GinWrapper) ServeHTTP(req *http.Request) *httptest.ResponseRecorder {
 	w.engine.ServeHTTP(respRec, req)
 	return respRec
 }
+
+// ForwardRequest manually inserts a request into this router
+// = behaves as if the request came from the outside (and writes the response to `writer`)
+func (w *GinWrapper) ForwardRequest(writer http.ResponseWriter, req *http.Request) {
+	w.engine.ServeHTTP(writer, req)
+}

ginext/response.go

@@ -1,6 +1,7 @@
 package ginext
 
 import (
+	"context"
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
@@ -410,7 +411,8 @@ func (j jsonAPIErrResponse) Write(g *gin.Context) {
 	for _, v := range j.cookies {
 		g.SetCookie(v.name, v.value, v.maxAge, v.path, v.domain, v.secure, v.httpOnly)
 	}
-	j.err.Output(g)
+
+	exerr.Get(j.err).Output(context.Background(), g)
 
 	j.err.CallListener(exerr.MethodOutput)
 }

go.mod

@@ -1,34 +1,37 @@
 module gogs.mikescher.com/BlackForestBytes/goext
 
-go 1.21
+go 1.22
 
 require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.-
-	github.com/jmoiron/sqlx v1.3.5
+	github.com/jmoiron/sqlx v1.4.0
 	github.com/rs/xid v1.5.0
 	github.com/rs/zerolog v1.32.0
-	go.mongodb.org/mongo-driver v1.14.0
-	golang.org/x/crypto v0.21.0
-	golang.org/x/sys v0.18.0
-	golang.org/x/term v0.18.0
+	go.mongodb.org/mongo-driver v1.15.0
+	golang.org/x/crypto v0.22.0
+	golang.org/x/sys v0.19.0
+	golang.org/x/term v0.19.0
 )
 
 require (
-	github.com/bytedance/sonic v1.11.2 // indirect
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.19.0 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -36,19 +39,19 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect
 	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.34.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.37.6 // indirect
 	modernc.org/mathutil v1.6.0 // indirect

go.sum

@@ -1,3 +1,4 @@
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
@@ -8,6 +9,17 @@ github.com/bytedance/sonic v1.11.1 h1:JC0+6c9FoWYYxakaoa+c5QTtJeiSZNeByOBhXtAFSn
 github.com/bytedance/sonic v1.11.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A=
 github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.3 h1:jRN+yEjakWh8aK5FzrciUHG8OFXK+4/KrAX/ysEtHAA=
+github.com/bytedance/sonic v1.11.3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.4 h1:8+OMLSSDDm2/qJc6ld5K5Sm62NK9VHcUKk0NzBoMAM4=
+github.com/bytedance/sonic v1.11.4/go.mod h1:YrWEqYtlBPS6LUA0vpuG79a1trsh4Ae41uWUWUreHhE=
+github.com/bytedance/sonic v1.11.5 h1:G00FYjjqll5iQ1PYXynbg/hyzqBqavH8Mo9/oTopd9k=
+github.com/bytedance/sonic v1.11.5/go.mod h1:X2PC2giUdj/Cv2lliWFLk6c/DUQok5rViJSemeB0wDw=
+github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bytedance/sonic/loader v0.1.0/go.mod h1:UmRT+IRTGKz/DAkzcEGzyVqQFJ7H9BqwBO3pm9H/+HY=
+github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
@@ -15,6 +27,17 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpV
 github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/cloudwego/base64x v0.1.0 h1:Tg5q9tq1khq9Y9UwfoC6zkHK0FypN2GLDvhqFceOL8U=
+github.com/cloudwego/base64x v0.1.0/go.mod h1:lM8nFiNbg74QgesNo6EAtv8N9tlRjBWExmHoNDa3PkU=
+github.com/cloudwego/base64x v0.1.3 h1:b5J/l8xolB7dyDTTmhJP2oTs5LdrjyrUFuNxdfq5hAg=
+github.com/cloudwego/base64x v0.1.3/go.mod h1:1+1K5BUHIQzyapgpF7LwvOGAEDicKtt1umPV+aN8pi8=
+github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
+github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
+github.com/cloudwego/iasm v0.0.9/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/cloudwego/iasm v0.1.0 h1:q0OuhwWDMyi3nlrQ6kIr0Yx0c3FI6cq/OZWKodIDdz8=
+github.com/cloudwego/iasm v0.1.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -43,8 +66,12 @@ github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtP
 github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
 github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
+github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -62,6 +89,8 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -71,6 +100,8 @@ github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2e
 github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
 github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -83,6 +114,7 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -92,6 +124,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
 github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -102,6 +135,12 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
 github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo=
+github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg=
+github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -116,6 +155,7 @@ github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -124,6 +164,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
@@ -137,11 +179,15 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
+github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 h1:tBiBTKHnIjovYoLX/TPkcf+OjqqKGQrPtGT3Foz+Pgo=
+github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76/go.mod h1:SQliXeA7Dhkt//vS29v3zpbEwoa+zb2Cn5xj5uO4K5U=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
 go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
 go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
 go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
+go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
+go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
 golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
@@ -159,6 +205,8 @@ golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
 golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
 golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -173,10 +221,14 @@ golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -194,6 +246,8 @@ golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
@@ -202,6 +256,8 @@ golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -221,6 +277,8 @@ google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4=
+google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.403"
+const GoextVersion = "0.0.445"
 
-const GoextVersionTimestamp = "2024-03-10T12:58:59+0100"
+const GoextVersionTimestamp = "2024-05-03T15:28:53+0200"

gojson/decode.go

@@ -217,6 +217,7 @@ type decodeState struct {
 	savedError            error
 	useNumber             bool
 	disallowUnknownFields bool
+	tagkey                *string
 }
 
 // readIndex returns the position of the last byte read.
@@ -652,7 +653,11 @@ func (d *decodeState) object(v reflect.Value) error {
 			v.Set(reflect.MakeMap(t))
 		}
 	case reflect.Struct:
-		fields = cachedTypeFields(t)
+		tagkey := "json"
+		if d.tagkey != nil {
+			tagkey = *d.tagkey
+		}
+		fields = cachedTypeFields(t, tagkey)
 		// ok
 	default:
 		d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)})

gojson/encode.go

@@ -382,7 +382,12 @@ func isEmptyValue(v reflect.Value) bool {
 }
 
 func (e *encodeState) reflectValue(v reflect.Value, opts encOpts) {
-	valueEncoder(v)(e, v, opts)
+	tagkey := "json"
+	if opts.tagkey != nil {
+		tagkey = *opts.tagkey
+	}
+
+	valueEncoder(v, tagkey)(e, v, opts)
 }
 
 type encOpts struct {
@@ -397,20 +402,22 @@ type encOpts struct {
 	// filter matches jsonfilter tag of struct
 	// marshals if no jsonfilter is set or otherwise if jsonfilter has the filter value
 	filter *string
+	// use different tag instead of "json"
+	tagkey *string
 }
 
 type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
 
 var encoderCache sync.Map // map[reflect.Type]encoderFunc
 
-func valueEncoder(v reflect.Value) encoderFunc {
+func valueEncoder(v reflect.Value, tagkey string) encoderFunc {
 	if !v.IsValid() {
 		return invalidValueEncoder
 	}
-	return typeEncoder(v.Type())
+	return typeEncoder(v.Type(), tagkey)
 }
 
-func typeEncoder(t reflect.Type) encoderFunc {
+func typeEncoder(t reflect.Type, tagkey string) encoderFunc {
 	if fi, ok := encoderCache.Load(t); ok {
 		return fi.(encoderFunc)
 	}
@@ -433,7 +440,7 @@ func typeEncoder(t reflect.Type) encoderFunc {
 	}
 
 	// Compute the real encoder and replace the indirect func with it.
-	f = newTypeEncoder(t, true)
+	f = newTypeEncoder(t, true, tagkey)
 	wg.Done()
 	encoderCache.Store(t, f)
 	return f
@@ -446,19 +453,19 @@ var (
 
 // newTypeEncoder constructs an encoderFunc for a type.
 // The returned encoder only checks CanAddr when allowAddr is true.
-func newTypeEncoder(t reflect.Type, allowAddr bool) encoderFunc {
+func newTypeEncoder(t reflect.Type, allowAddr bool, tagkey string) encoderFunc {
 	// If we have a non-pointer value whose type implements
 	// Marshaler with a value receiver, then we're better off taking
 	// the address of the value - otherwise we end up with an
 	// allocation as we cast the value to an interface.
 	if t.Kind() != reflect.Pointer && allowAddr && reflect.PointerTo(t).Implements(marshalerType) {
-		return newCondAddrEncoder(addrMarshalerEncoder, newTypeEncoder(t, false))
+		return newCondAddrEncoder(addrMarshalerEncoder, newTypeEncoder(t, false, tagkey))
 	}
 	if t.Implements(marshalerType) {
 		return marshalerEncoder
 	}
 	if t.Kind() != reflect.Pointer && allowAddr && reflect.PointerTo(t).Implements(textMarshalerType) {
-		return newCondAddrEncoder(addrTextMarshalerEncoder, newTypeEncoder(t, false))
+		return newCondAddrEncoder(addrTextMarshalerEncoder, newTypeEncoder(t, false, tagkey))
 	}
 	if t.Implements(textMarshalerType) {
 		return textMarshalerEncoder
@@ -480,15 +487,15 @@ func newTypeEncoder(t reflect.Type, allowAddr bool) encoderFunc {
 	case reflect.Interface:
 		return interfaceEncoder
 	case reflect.Struct:
-		return newStructEncoder(t)
+		return newStructEncoder(t, tagkey)
 	case reflect.Map:
-		return newMapEncoder(t)
+		return newMapEncoder(t, tagkey)
 	case reflect.Slice:
-		return newSliceEncoder(t)
+		return newSliceEncoder(t, tagkey)
 	case reflect.Array:
-		return newArrayEncoder(t)
+		return newArrayEncoder(t, tagkey)
 	case reflect.Pointer:
-		return newPtrEncoder(t)
+		return newPtrEncoder(t, tagkey)
 	default:
 		return unsupportedTypeEncoder
 	}
@@ -801,8 +808,8 @@ FieldLoop:
 	}
 }
 
-func newStructEncoder(t reflect.Type) encoderFunc {
-	se := structEncoder{fields: cachedTypeFields(t)}
+func newStructEncoder(t reflect.Type, tagkey string) encoderFunc {
+	se := structEncoder{fields: cachedTypeFields(t, tagkey)}
 	return se.encode
 }
 
@@ -855,7 +862,7 @@ func (me mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
 	e.ptrLevel--
 }
 
-func newMapEncoder(t reflect.Type) encoderFunc {
+func newMapEncoder(t reflect.Type, tagkey string) encoderFunc {
 	switch t.Key().Kind() {
 	case reflect.String,
 		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
@@ -865,7 +872,7 @@ func newMapEncoder(t reflect.Type) encoderFunc {
 			return unsupportedTypeEncoder
 		}
 	}
-	me := mapEncoder{typeEncoder(t.Elem())}
+	me := mapEncoder{typeEncoder(t.Elem(), tagkey)}
 	return me.encode
 }
 
@@ -936,7 +943,7 @@ func (se sliceEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
 	e.ptrLevel--
 }
 
-func newSliceEncoder(t reflect.Type) encoderFunc {
+func newSliceEncoder(t reflect.Type, tagkey string) encoderFunc {
 	// Byte slices get special treatment; arrays don't.
 	if t.Elem().Kind() == reflect.Uint8 {
 		p := reflect.PointerTo(t.Elem())
@@ -944,7 +951,7 @@ func newSliceEncoder(t reflect.Type) encoderFunc {
 			return encodeByteSlice
 		}
 	}
-	enc := sliceEncoder{newArrayEncoder(t)}
+	enc := sliceEncoder{newArrayEncoder(t, tagkey)}
 	return enc.encode
 }
 
@@ -964,8 +971,8 @@ func (ae arrayEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
 	e.WriteByte(']')
 }
 
-func newArrayEncoder(t reflect.Type) encoderFunc {
-	enc := arrayEncoder{typeEncoder(t.Elem())}
+func newArrayEncoder(t reflect.Type, tagkey string) encoderFunc {
+	enc := arrayEncoder{typeEncoder(t.Elem(), tagkey)}
 	return enc.encode
 }
 
@@ -992,8 +999,8 @@ func (pe ptrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
 	e.ptrLevel--
 }
 
-func newPtrEncoder(t reflect.Type) encoderFunc {
-	enc := ptrEncoder{typeEncoder(t.Elem())}
+func newPtrEncoder(t reflect.Type, tagkey string) encoderFunc {
+	enc := ptrEncoder{typeEncoder(t.Elem(), tagkey)}
 	return enc.encode
 }
 
@@ -1270,7 +1277,7 @@ func (x byIndex) Less(i, j int) bool {
 // typeFields returns a list of fields that JSON should recognize for the given type.
 // The algorithm is breadth-first search over the set of structs to include - the top struct
 // and then any reachable anonymous structs.
-func typeFields(t reflect.Type) structFields {
+func typeFields(t reflect.Type, tagkey string) structFields {
 	// Anonymous fields to explore at the current level and the next.
 	current := []field{}
 	next := []field{{typ: t}}
@@ -1315,7 +1322,7 @@ func typeFields(t reflect.Type) structFields {
 					// Ignore unexported non-embedded fields.
 					continue
 				}
-				tag := sf.Tag.Get("json")
+				tag := sf.Tag.Get(tagkey)
 				if tag == "-" {
 					continue
 				}
@@ -1449,7 +1456,7 @@ func typeFields(t reflect.Type) structFields {
 
 	for i := range fields {
 		f := &fields[i]
-		f.encoder = typeEncoder(typeByIndex(t, f.index))
+		f.encoder = typeEncoder(typeByIndex(t, f.index), tagkey)
 	}
 	nameIndex := make(map[string]int, len(fields))
 	for i, field := range fields {
@@ -1474,13 +1481,26 @@ func dominantField(fields []field) (field, bool) {
 	return fields[0], true
 }
 
-var fieldCache sync.Map // map[reflect.Type]structFields
+var fieldCache sync.Map // map[string]map[reflect.Type]structFields
 
 // cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
-func cachedTypeFields(t reflect.Type) structFields {
-	if f, ok := fieldCache.Load(t); ok {
+func cachedTypeFields(t reflect.Type, tagkey string) structFields {
+	if m0, ok := fieldCache.Load(tagkey); ok {
+
+		if f, ok := m0.(*sync.Map).Load(t); ok {
+			return f.(structFields)
+		}
+		f, _ := m0.(*sync.Map).LoadOrStore(t, typeFields(t, tagkey))
+		return f.(structFields)
+
+	} else {
+
+		m0 := &sync.Map{}
+		f, _ := m0.LoadOrStore(t, typeFields(t, tagkey))
+
+		fieldCache.Store(tagkey, m0)
+
 		return f.(structFields)
 	}
-	f, _ := fieldCache.LoadOrStore(t, typeFields(t))
-	return f.(structFields)
+
 }

gojson/stream.go

@@ -41,6 +41,9 @@ func (dec *Decoder) UseNumber() { dec.d.useNumber = true }
 // non-ignored, exported fields in the destination.
 func (dec *Decoder) DisallowUnknownFields() { dec.d.disallowUnknownFields = true }
 
+// TagKey sets a different TagKey (instead of "json")
+func (dec *Decoder) TagKey(v string) { dec.d.tagkey = &v }
+
 // Decode reads the next JSON-encoded value from its
 // input and stores it in the value pointed to by v.
 //

langext/array.go

@@ -453,6 +453,15 @@ func ArrConcat[T any](arr ...[]T) []T {
 	return r
 }
 
+// ArrAppend works similar to append(x, y, z) - but doe snot touch the old array and creates a new one
+func ArrAppend[T any](arr []T, add ...T) []T {
+	r := ArrCopy(arr)
+	for _, v := range add {
+		r = append(r, v)
+	}
+	return r
+}
+
 // ArrCopy does a shallow copy of the 'in' array
 func ArrCopy[T any](in []T) []T {
 	out := make([]T, len(in))
@@ -468,6 +477,10 @@ func ArrRemove[T comparable](arr []T, needle T) []T {
 	return arr
 }
 
+func ArrRemoveAt[T any](arr []T, idx int) []T {
+	return append(arr[:idx], arr[idx+1:]...)
+}
+
 func ArrExcept[T comparable](arr []T, needles ...T) []T {
 	r := make([]T, 0, len(arr))
 	rmlist := ArrToSet(needles)

langext/coalesce.go

@@ -5,12 +5,76 @@ import (
 	"time"
 )
 
-func Coalesce[T any](v *T, def T) T {
-	if v == nil {
-		return def
-	} else {
-		return *v
+func Coalesce[T any](v1 *T, def T) T {
+	if v1 != nil {
+		return *v1
 	}
+
+	return def
+}
+
+func CoalesceOpt[T any](v1 *T, v2 *T) *T {
+	if v1 != nil {
+		return v1
+	}
+
+	return v2
+}
+
+func Coalesce3[T any](v1 *T, v2 *T, def T) T {
+	if v1 != nil {
+		return *v1
+	}
+
+	if v2 != nil {
+		return *v2
+	}
+
+	return def
+}
+
+func Coalesce3Opt[T any](v1 *T, v2 *T, v3 *T) *T {
+	if v1 != nil {
+		return v1
+	}
+
+	if v2 != nil {
+		return v2
+	}
+
+	return v3
+}
+
+func Coalesce4[T any](v1 *T, v2 *T, v3 *T, def T) T {
+	if v1 != nil {
+		return *v1
+	}
+
+	if v2 != nil {
+		return *v2
+	}
+
+	if v3 != nil {
+		return *v3
+	}
+
+	return def
+}
+
+func Coalesce4Opt[T any](v1 *T, v2 *T, v3 *T, v4 *T) *T {
+	if v1 != nil {
+		return v1
+	}
+
+	if v2 != nil {
+		return v2
+	}
+
+	if v3 != nil {
+		return v3
+	}
+
+	return v4
 }
 
 func CoalesceString(s *string, def string) string {

langext/json.go

@@ -63,3 +63,51 @@ func PatchRemJson[JV string | []byte](rawjson JV, key string) (JV, error) {
 
 	return JV(newjson), nil
 }
+
+func MarshalJsonOrPanic(v any) string {
+	bin, err := json.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	return string(bin)
+}
+
+func MarshalJsonOrDefault(v any, def string) string {
+	bin, err := json.Marshal(v)
+	if err != nil {
+		return def
+	}
+	return string(bin)
+}
+
+func MarshalJsonOrNil(v any) *string {
+	bin, err := json.Marshal(v)
+	if err != nil {
+		return nil
+	}
+	return Ptr(string(bin))
+}
+
+func MarshalJsonIndentOrPanic(v any, prefix, indent string) string {
+	bin, err := json.MarshalIndent(v, prefix, indent)
+	if err != nil {
+		panic(err)
+	}
+	return string(bin)
+}
+
+func MarshalJsonIndentOrDefault(v any, prefix, indent string, def string) string {
+	bin, err := json.MarshalIndent(v, prefix, indent)
+	if err != nil {
+		return def
+	}
+	return string(bin)
+}
+
+func MarshalJsonIndentOrNil(v any, prefix, indent string) *string {
+	bin, err := json.MarshalIndent(v, prefix, indent)
+	if err != nil {
+		return nil
+	}
+	return Ptr(string(bin))
+}

langext/object.go

@@ -0,0 +1,19 @@
+package langext
+
+import "encoding/json"
+
+func DeepCopyByJson[T any](v T) (T, error) {
+
+	bin, err := json.Marshal(v)
+	if err != nil {
+		return *new(T), err
+	}
+
+	var result T
+	err = json.Unmarshal(bin, &result)
+	if err != nil {
+		return *new(T), err
+	}
+
+	return result, nil
+}

langext/pointer.go

@@ -27,6 +27,10 @@ func DblPtrNil[T any]() **T {
 	return &v
 }
 
+func ArrPtr[T any](v ...T) *[]T {
+	return &v
+}
+
 func PtrInt32(v int32) *int32 {
 	return &v
 }

reflectext/convertToMap.go

@@ -1,19 +1,45 @@
 package reflectext
 
-import "reflect"
+import (
+	"encoding/json"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"reflect"
+)
 
-func ConvertStructToMap(v any) any {
-	return reflectToMap(reflect.ValueOf(v))
+type ConvertStructToMapOpt struct {
+	KeepJsonMarshalTypes bool
+	MaxDepth             *int
 }
 
-func reflectToMap(fv reflect.Value) any {
+func ConvertStructToMap(v any, opts ...ConvertStructToMapOpt) map[string]any {
+	opt := ConvertStructToMapOpt{}
+	if len(opts) > 0 {
+		opt = opts[0]
+	}
+
+	res := reflectToMap(reflect.ValueOf(v), 1, opt)
+
+	if v, ok := res.(map[string]any); ok {
+		return v
+	} else if langext.IsNil(res) {
+		return nil
+	} else {
+		panic("not an object")
+	}
+}
+
+func reflectToMap(fv reflect.Value, depth int, opt ConvertStructToMapOpt) any {
+
+	if opt.MaxDepth != nil && depth > *opt.MaxDepth {
+		return fv.Interface()
+	}
 
 	if fv.Kind() == reflect.Ptr {
 
 		if fv.IsNil() {
 			return nil
 		} else {
-			return reflectToMap(fv.Elem())
+			return reflectToMap(fv.Elem(), depth, opt)
 		}
 
 	}
@@ -30,7 +56,7 @@ func reflectToMap(fv reflect.Value) any {
 		arrlen := fv.Len()
 		arr := make([]any, arrlen)
 		for i := 0; i < arrlen; i++ {
-			arr[i] = reflectToMap(fv.Index(i))
+			arr[i] = reflectToMap(fv.Index(i), depth+1, opt)
 		}
 		return arr
 
@@ -41,7 +67,7 @@ func reflectToMap(fv reflect.Value) any {
 		arrlen := fv.Len()
 		arr := make([]any, arrlen)
 		for i := 0; i < arrlen; i++ {
-			arr[i] = reflectToMap(fv.Index(i))
+			arr[i] = reflectToMap(fv.Index(i), depth+1, opt)
 		}
 		return arr
 
@@ -56,11 +82,15 @@ func reflectToMap(fv reflect.Value) any {
 
 	if fv.Kind() == reflect.Struct {
 
+		if opt.KeepJsonMarshalTypes && fv.Type().Implements(reflect.TypeFor[json.Marshaler]()) {
+			return fv.Interface()
+		}
+
 		res := make(map[string]any)
 
 		for i := 0; i < fv.NumField(); i++ {
 			if fv.Type().Field(i).IsExported() {
-				res[fv.Type().Field(i).Name] = reflectToMap(fv.Field(i))
+				res[fv.Type().Field(i).Name] = reflectToMap(fv.Field(i), depth+1, opt)
 			}
 		}
 

reflectext/convertToMap_test.go

@@ -0,0 +1,42 @@
+package reflectext
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"testing"
+	"time"
+)
+
+func TestConvertStructToMap(t *testing.T) {
+
+	type tst struct {
+		FieldA  int
+		FieldB  string
+		FieldC  time.Time
+		FieldD  []float64
+		FieldE1 *int
+		FieldE2 **int
+		FieldE3 *int
+		FieldE4 **int
+		FieldE5 *int
+		FieldE6 **int
+	}
+
+	value := tst{
+		FieldA:  123,
+		FieldB:  "hello",
+		FieldC:  time.Date(2020, 05, 12, 8, 30, 0, 0, time.UTC),
+		FieldD:  []float64{1, 2, 3, 4, 5, 6, 7},
+		FieldE1: nil,
+		FieldE2: nil,
+		FieldE3: langext.Ptr(12),
+		FieldE4: langext.DblPtr(12),
+		FieldE5: nil,
+		FieldE6: langext.DblPtrNil[int](),
+	}
+
+	valueOut := ConvertStructToMap(value, ConvertStructToMapOpt{KeepJsonMarshalTypes: true})
+
+	fmt.Printf("%+v\n", valueOut)
+
+}

reflectext/mapAccess_test.go

@@ -47,9 +47,3 @@ func TestGetMapField(t *testing.T) {
 	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany2, "Test")), "12 true")
 	tst.AssertEqual(t, fmt.Sprint(GetMapField[PseudoInt](maany2, "Test2")), "0 false")
 }
-
-func main2() {
-}
-
-func main() {
-}

rfctime/date.go

@@ -9,6 +9,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
 	"reflect"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -65,36 +67,20 @@ func (t *Date) UnmarshalJSON(data []byte) error {
 	if err := json.Unmarshal(data, &str); err != nil {
 		return err
 	}
-	t0, err := time.Parse(t.FormatStr(), str)
-	if err != nil {
-		return err
-	}
-	t.Year = t0.Year()
-	t.Month = int(t0.Month())
-	t.Day = t0.Day()
-	return nil
+	return t.ParseString(str)
 }
 
 func (t Date) MarshalJSON() ([]byte, error) {
-	str := t.TimeUTC().Format(t.FormatStr())
+	str := t.String()
 	return json.Marshal(str)
 }
 
 func (t Date) MarshalText() ([]byte, error) {
-	b := make([]byte, 0, len(t.FormatStr()))
-	return t.TimeUTC().AppendFormat(b, t.FormatStr()), nil
+	return []byte(t.String()), nil
 }
 
 func (t *Date) UnmarshalText(data []byte) error {
-	var err error
-	v, err := time.Parse(t.FormatStr(), string(data))
-	if err != nil {
-		return err
-	}
-	t.Year = v.Year()
-	t.Month = int(v.Month())
-	t.Day = v.Day()
-	return nil
+	return t.ParseString(string(data))
 }
 
 func (t *Date) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
@@ -116,6 +102,13 @@ func (t *Date) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
 		return err
 	}
 
+	if tt == "" {
+		t.Year = 0
+		t.Month = 0
+		t.Day = 0
+		return nil
+	}
+
 	v, err := time.Parse(t.FormatStr(), tt)
 	if err != nil {
 		return err
@@ -128,7 +121,10 @@ func (t *Date) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
 }
 
 func (t Date) MarshalBSONValue() (bsontype.Type, []byte, error) {
-	return bson.MarshalValue(t.TimeUTC().Format(t.FormatStr()))
+	if t.IsZero() {
+		return bson.MarshalValue("")
+	}
+	return bson.MarshalValue(t.String())
 }
 
 func (t Date) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val reflect.Value) error {
@@ -164,7 +160,7 @@ func (t Date) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val
 }
 
 func (t Date) Serialize() string {
-	return t.TimeUTC().Format(t.FormatStr())
+	return t.String()
 }
 
 func (t Date) FormatStr() string {
@@ -212,11 +208,52 @@ func (t Date) Format(layout string) string {
 }
 
 func (t Date) GoString() string {
-	return t.TimeUTC().GoString()
+	return fmt.Sprintf("rfctime.Date{Year: %d, Month: %d, Day: %d}", t.Year, t.Month, t.Day)
 }
 
 func (t Date) String() string {
-	return t.TimeUTC().String()
+	return fmt.Sprintf("%04d-%02d-%02d", t.Year, t.Month, t.Day)
+}
+
+func (t *Date) ParseString(v string) error {
+	split := strings.Split(v, "-")
+	if len(split) != 3 {
+		return errors.New("invalid date format: " + v)
+	}
+	year, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	month, err := strconv.ParseInt(split[1], 10, 32)
+	if err != nil {
+		return errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	day, err := strconv.ParseInt(split[2], 10, 32)
+	if err != nil {
+		return errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+
+	if year < 0 {
+		return errors.New("invalid date format: " + v + ": year is negative")
+	}
+
+	if month < 1 || month > 12 {
+		return errors.New("invalid date format: " + v + ": month is out of range")
+	}
+
+	if day < 1 || day > 31 {
+		return errors.New("invalid date format: " + v + ": day is out of range")
+	}
+
+	t.Year = int(year)
+	t.Month = int(month)
+	t.Day = int(day)
+
+	return nil
+}
+
+func (t Date) IsZero() bool {
+	return t.Year == 0 && t.Month == 0 && t.Day == 0
 }
 
 func NewDate(t time.Time) Date {

rfctime/rfc3339.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"time"
 )
@@ -245,6 +246,13 @@ func NewRFC3339(t time.Time) RFC3339Time {
 	return RFC3339Time(t)
 }
 
+func NewRFC3339Ptr(t *time.Time) *RFC3339Time {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(RFC3339Time(*t))
+}
+
 func NowRFC3339() RFC3339Time {
 	return RFC3339Time(time.Now())
 }

rfctime/rfc3339Nano.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"time"
 )
@@ -245,6 +246,13 @@ func NewRFC3339Nano(t time.Time) RFC3339NanoTime {
 	return RFC3339NanoTime(t)
 }
 
+func NewRFC3339NanoPtr(t *time.Time) *RFC3339NanoTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(RFC3339NanoTime(*t))
+}
+
 func NowRFC3339Nano() RFC3339NanoTime {
 	return RFC3339NanoTime(time.Now())
 }

rfctime/unix.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"strconv"
 	"time"
@@ -239,6 +240,13 @@ func NewUnix(t time.Time) UnixTime {
 	return UnixTime(t)
 }
 
+func NewUnixPtr(t *time.Time) *UnixTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(UnixTime(*t))
+}
+
 func NowUnix() UnixTime {
 	return UnixTime(time.Now())
 }

rfctime/unixMilli.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"strconv"
 	"time"
@@ -239,6 +240,13 @@ func NewUnixMilli(t time.Time) UnixMilliTime {
 	return UnixMilliTime(t)
 }
 
+func NewUnixMilliPtr(t *time.Time) *UnixMilliTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(UnixMilliTime(*t))
+}
+
 func NowUnixMilli() UnixMilliTime {
 	return UnixMilliTime(time.Now())
 }

rfctime/unixNano.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"strconv"
 	"time"
@@ -239,6 +240,13 @@ func NewUnixNano(t time.Time) UnixNanoTime {
 	return UnixNanoTime(t)
 }
 
+func NewUnixNanoPtr(t *time.Time) *UnixNanoTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(UnixNanoTime(*t))
+}
+
 func NowUnixNano() UnixNanoTime {
 	return UnixNanoTime(time.Now())
 }

sq/converterDefault.go

@@ -7,8 +7,6 @@ import (
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
 	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
-	"strconv"
-	"strings"
 	"time"
 )
 
@@ -79,24 +77,12 @@ var ConverterRFC339NanoTimeToString = NewDBTypeConverter[rfctime.RFC3339NanoTime
 var ConverterRFCDateToString = NewDBTypeConverter[rfctime.Date, string](func(v rfctime.Date) (string, error) {
 	return fmt.Sprintf("%04d-%02d-%02d", v.Year, v.Month, v.Day), nil
 }, func(v string) (rfctime.Date, error) {
-	split := strings.Split(v, "-")
-	if len(split) != 3 {
-		return rfctime.Date{}, errors.New("invalid date format: " + v)
+	d := rfctime.Date{}
+	if err := d.ParseString(v); err != nil {
+		return rfctime.Date{}, err
+	} else {
+		return d, nil
 	}
-	year, err := strconv.ParseInt(split[0], 10, 32)
-	if err != nil {
-		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
-	}
-	month, err := strconv.ParseInt(split[0], 10, 32)
-	if err != nil {
-		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
-	}
-	day, err := strconv.ParseInt(split[0], 10, 32)
-	if err != nil {
-		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
-	}
-
-	return rfctime.Date{Year: int(year), Month: int(month), Day: int(day)}, nil
 })
 
 var ConverterRFCTimeToString = NewDBTypeConverter[rfctime.Time, string](func(v rfctime.Time) (string, error) {

sq/filter.go

@@ -47,3 +47,10 @@ func NewSimplePaginateFilter(filterClause string, filterParams PP, sort []Filter
 		},
 	}
 }
+
+func NewEmptyPaginateFilter() PaginateFilter {
+	return genericPaginateFilter{
+		sql:  func(params PP) (string, string, []string) { return "1=1", "", nil },
+		sort: func() []FilterSort { return make([]FilterSort, 0) },
+	}
+}

sq/list.go

@@ -0,0 +1,48 @@
+package sq
+
+import (
+	"context"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+)
+
+func Iterate[TData any](ctx context.Context, q Queryable, table string, filter PaginateFilter, scanMode StructScanMode, scanSec StructScanSafety, page int, limit *int, consumer func(ctx context.Context, v TData) error) (int, error) {
+	if filter == nil {
+		filter = NewEmptyPaginateFilter()
+	}
+
+	prepParams := PP{}
+
+	sortOrder := filter.Sort()
+	sortCond := ""
+	if len(sortOrder) > 0 {
+		sortCond = "ORDER BY "
+		for i, v := range sortOrder {
+			if i > 0 {
+				sortCond += ", "
+			}
+			sortCond += v.Field + " " + string(v.Direction)
+		}
+	}
+
+	pageCond := ""
+	if limit != nil {
+		pageCond += fmt.Sprintf("LIMIT :%s OFFSET :%s", prepParams.Add(*limit+1), prepParams.Add(*limit*(page-1)))
+	}
+
+	filterCond, joinCond, joinTables := filter.SQL(prepParams)
+
+	selectCond := table + ".*"
+	for _, v := range joinTables {
+		selectCond += ", " + v + ".*"
+	}
+
+	sqlQueryData := "SELECT " + selectCond + " FROM " + table + " " + joinCond + " WHERE ( " + filterCond + " ) " + sortCond + " " + pageCond
+
+	rows, err := q.Query(ctx, sqlQueryData, prepParams)
+	if err != nil {
+		return 0, exerr.Wrap(err, "failed to list paginated entries from DB").Str("table", table).Any("filter", filter).Int("page", page).Any("limit", limit).Build()
+	}
+
+	return IterateAll[TData](ctx, q, rows, scanMode, scanSec, true, consumer)
+}

sq/paginate.go

@@ -9,6 +9,10 @@ import (
 )
 
 func Paginate[TData any](ctx context.Context, q Queryable, table string, filter PaginateFilter, scanMode StructScanMode, scanSec StructScanSafety, page int, limit *int) ([]TData, pag.Pagination, error) {
+	if filter == nil {
+		filter = NewEmptyPaginateFilter()
+	}
+
 	prepParams := PP{}
 
 	sortOrder := filter.Sort()
@@ -90,6 +94,10 @@ func Paginate[TData any](ctx context.Context, q Queryable, table string, filter
 }
 
 func Count(ctx context.Context, q Queryable, table string, filter PaginateFilter) (int, error) {
+	if filter == nil {
+		filter = NewEmptyPaginateFilter()
+	}
+
 	prepParams := PP{}
 
 	filterCond, joinCond, _ := filter.SQL(prepParams)

sq/scanner.go

@@ -333,3 +333,79 @@ func ScanAll[TData any](ctx context.Context, q Queryable, rows *sqlx.Rows, mode
 	}
 	return res, nil
 }
+
+func IterateAll[TData any](ctx context.Context, q Queryable, rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool, consumer func(ctx context.Context, v TData) error) (int, error) {
+	var strscan *StructScanner
+
+	if sec == Safe {
+		strscan = NewStructScanner(rows, false)
+		var data TData
+		err := strscan.Start(&data)
+		if err != nil {
+			return 0, err
+		}
+	} else if sec == Unsafe {
+		strscan = NewStructScanner(rows, true)
+		var data TData
+		err := strscan.Start(&data)
+		if err != nil {
+			return 0, err
+		}
+	} else {
+		return 0, errors.New("unknown value for <sec>")
+	}
+
+	rcount := 0
+
+	for rows.Next() {
+
+		if err := ctx.Err(); err != nil {
+			return rcount, err
+		}
+
+		if mode == SModeFast {
+			var data TData
+			err := strscan.StructScanBase(&data)
+			if err != nil {
+				return rcount, err
+			}
+
+			err = consumer(ctx, data)
+			if err != nil {
+				return rcount, exerr.Wrap(err, "").Build()
+			}
+
+			rcount++
+
+		} else if mode == SModeExtended {
+			var data TData
+			err := strscan.StructScanExt(q, &data)
+			if err != nil {
+				return rcount, err
+			}
+
+			err = consumer(ctx, data)
+			if err != nil {
+				return rcount, exerr.Wrap(err, "").Build()
+			}
+
+			rcount++
+
+		} else {
+			return rcount, errors.New("unknown value for <mode>")
+		}
+	}
+
+	if close {
+		err := strscan.rows.Close()
+		if err != nil {
+			return rcount, err
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return rcount, err
+	}
+
+	return rcount, nil
+}

timeext/time.go

@@ -146,3 +146,37 @@ func UnixFloatSeconds(v float64) time.Time {
 func FloorTime(t time.Time) time.Time {
 	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
 }
+
+func SubtractYears(t time.Time, yearCount float64, tz *time.Location) time.Time {
+	t = t.In(tz)
+
+	if yearCount < 0 {
+		return AddYears(t, -yearCount, tz)
+	}
+
+	intCount, floatCount := math.Modf(yearCount)
+
+	t.AddDate(-int(intCount), 0, 0)
+
+	t0 := TimeToYearStart(t, tz)
+	t1 := TimeToYearEnd(t, tz)
+
+	return t.Add(time.Duration(float64(t1.Sub(t0)) * floatCount * -1))
+}
+
+func AddYears(t time.Time, yearCount float64, tz *time.Location) time.Time {
+	t = t.In(tz)
+
+	if yearCount < 0 {
+		return SubtractYears(t, -yearCount, tz)
+	}
+
+	intCount, floatCount := math.Modf(yearCount)
+
+	t.AddDate(int(intCount), 0, 0)
+
+	t0 := TimeToYearStart(t, tz)
+	t1 := TimeToYearEnd(t, tz)
+
+	return t.Add(time.Duration(float64(t1.Sub(t0)) * floatCount))
+}

wmo/collection.go

@@ -125,3 +125,17 @@ func (c *Coll[TData]) createToken(fieldPrimary string, dirPrimary ct.SortDirecti
 		Extra:          ct.Extra{},
 	}, nil
 }
+
+func (c *Coll[TData]) needsDoubleSort(ctx context.Context) bool {
+	for _, ppl := range c.extraModPipeline {
+		for _, stage := range ppl(ctx) {
+			for _, bsone := range stage {
+				if bsone.Key == "$group" {
+					// a group stage in extraModPipeline results in unsorted data, which means the caller must sort again after these pipeline stages...
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

wmo/queryFind.go

@@ -36,6 +36,14 @@ func (c *Coll[TData]) Find(ctx context.Context, filter bson.M, opts ...*options.
 		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
 	}
 
+	if c.needsDoubleSort(ctx) {
+		for _, opt := range opts {
+			if opt != nil && opt.Sort != nil {
+				pipeline = append(pipeline, bson.D{{Key: "$sort", Value: opt.Sort}})
+			}
+		}
+	}
+
 	for _, opt := range opts {
 		if opt != nil && opt.Projection != nil {
 			pipeline = append(pipeline, bson.D{{Key: "$project", Value: opt.Projection}})

wmo/queryList.go

@@ -14,6 +14,10 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		return make([]TData, 0), ct.End(), nil
 	}
 
+	if pageSize != nil && *pageSize == 0 {
+		return make([]TData, 0), inTok, nil // fast track, we return an empty list and do not advance the cursor token
+	}
+
 	pipeline := mongo.Pipeline{}
 	pf1 := "_id"
 	pd1 := ct.SortASC
@@ -35,7 +39,7 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		sortDirSecondary = nil
 	}
 
-	paginationPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize)
+	paginationPipeline, doubleSortPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize)
 	if err != nil {
 		return nil, ct.CursorToken{}, exerr.
 			Wrap(err, "failed to create pagination").
@@ -56,6 +60,10 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
 	}
 
+	if c.needsDoubleSort(ctx) {
+		pipeline = langext.ArrConcat(pipeline, doubleSortPipeline)
+	}
+
 	cursor, err := c.coll.Aggregate(ctx, pipeline)
 	if err != nil {
 		return nil, ct.CursorToken{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
@@ -136,14 +144,50 @@ func (c *Coll[TData]) ListWithCount(ctx context.Context, filter ct.Filter, pageS
 	return data, token, count, nil
 }
 
-func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, error) {
+func (c *Coll[TData]) ListAllIDs(ctx context.Context, filter ct.RawFilter) ([]string, error) {
+	type idObject struct {
+		ID string `bson:"_id"`
+	}
+
+	pipelineFilter := mongo.Pipeline{}
+
+	if filter != nil {
+		pipelineFilter = filter.FilterQuery()
+	}
+
+	extrModPipelineResolved := mongo.Pipeline{}
+	for _, ppl := range c.extraModPipeline {
+		extrModPipelineResolved = langext.ArrConcat(extrModPipelineResolved, ppl(ctx))
+	}
+
+	pipelineProjectIDs := mongo.Pipeline{}
+	pipelineProjectIDs = append(pipelineProjectIDs, bson.D{{Key: "$project", Value: bson.M{"_id": 1}}})
+
+	pipelineList := langext.ArrConcat(pipelineFilter, extrModPipelineResolved, pipelineProjectIDs)
+
+	cursorList, err := c.coll.Aggregate(ctx, pipelineList)
+	if err != nil {
+		return nil, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipelineList).Str("collection", c.Name()).Build()
+	}
+
+	var res []idObject
+
+	err = cursorList.All(ctx, &res)
+	if err != nil {
+		return nil, exerr.Wrap(err, "failed to decode entities").Any("pipeline", pipelineList).Str("collection", c.Name()).Build()
+	}
+
+	return langext.ArrMap(res, func(v idObject) string { return v.ID }), nil
+}
+
+func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, []bson.D, error) {
 
 	cond := bson.A{}
 	sort := bson.D{}
 
 	valuePrimary, err := coll.getTokenValueAsMongoType(token.ValuePrimary, fieldPrimary)
 	if err != nil {
-		return nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
+		return nil, nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
 	}
 
 	if sortPrimary == ct.SortASC {
@@ -160,7 +204,7 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken
 
 		valueSecondary, err := coll.getTokenValueAsMongoType(token.ValueSecondary, *fieldSecondary)
 		if err != nil {
-			return nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
+			return nil, nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
 		}
 
 		if *sortSecondary == ct.SortASC {
@@ -171,7 +215,7 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken
 				bson.M{*fieldSecondary: bson.M{"$gt": valueSecondary}},
 			}})
 
-			sort = append(sort, bson.E{Key: fieldPrimary, Value: +1})
+			sort = append(sort, bson.E{Key: *fieldSecondary, Value: +1})
 
 		} else if *sortSecondary == ct.SortDESC {
 
@@ -181,7 +225,7 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken
 				bson.M{*fieldSecondary: bson.M{"$lt": valueSecondary}},
 			}})
 
-			sort = append(sort, bson.E{Key: fieldPrimary, Value: -1})
+			sort = append(sort, bson.E{Key: *fieldSecondary, Value: -1})
 
 		}
 	}
@@ -203,15 +247,17 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken
 
 	} else {
 
-		return nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build()
+		return nil, nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build()
 
 	}
 
 	pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}})
 
+	pipelineSort := mongo.Pipeline{bson.D{{Key: "$sort", Value: sort}}}
+
 	if pageSize != nil {
 		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}})
 	}
 
-	return pipeline, nil
+	return pipeline, pipelineSort, nil
 }