v0.0.444 change gin values in exerr auto meta to not include dots in keys (fucks up mongo)

exerr/builder.go

@@ -309,27 +309,27 @@ func (b *Builder) Errs(key string, val []error) *Builder {
 func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request) *Builder {
 	if v := ctx.Value("start_timestamp"); v != nil {
 		if t, ok := v.(time.Time); ok {
-			b.Time("ctx.startTimestamp", t)
-			b.Time("ctx.endTimestamp", time.Now())
+			b.Time("ctx_startTimestamp", t)
+			b.Time("ctx_endTimestamp", time.Now())
 		}
 	}
-	b.Str("gin.method", req.Method)
-	b.Str("gin.path", g.FullPath())
-	b.Strs("gin.header", extractHeader(g.Request.Header))
+	b.Str("gin_method", req.Method)
+	b.Str("gin_path", g.FullPath())
+	b.Strs("gin_header", extractHeader(g.Request.Header))
 	if req.URL != nil {
-		b.Str("gin.url", req.URL.String())
+		b.Str("gin_url", req.URL.String())
 	}
 	if ctxVal := g.GetString("apiversion"); ctxVal != "" {
-		b.Str("gin.context.apiversion", ctxVal)
+		b.Str("gin_context_apiversion", ctxVal)
 	}
 	if ctxVal := g.GetString("uid"); ctxVal != "" {
-		b.Str("gin.context.uid", ctxVal)
+		b.Str("gin_context_uid", ctxVal)
 	}
 	if ctxVal := g.GetString("fcmId"); ctxVal != "" {
-		b.Str("gin.context.fcmid", ctxVal)
+		b.Str("gin_context_fcmid", ctxVal)
 	}
 	if ctxVal := g.GetString("reqid"); ctxVal != "" {
-		b.Str("gin.context.reqid", ctxVal)
+		b.Str("gin_context_reqid", ctxVal)
 	}
 	if req.Method != "GET" && req.Body != nil {
 
@@ -340,12 +340,12 @@ func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request)
 						var prettyJSON bytes.Buffer
 						err = json.Indent(&prettyJSON, bin, "", "  ")
 						if err == nil {
-							b.Str("gin.body", string(prettyJSON.Bytes()))
+							b.Str("gin_body", string(prettyJSON.Bytes()))
 						} else {
-							b.Bytes("gin.body", bin)
+							b.Bytes("gin_body", bin)
 						}
 					} else {
-						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+						b.Str("gin_body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
 					}
 				}
 			}
@@ -355,9 +355,9 @@ func (b *Builder) GinReq(ctx context.Context, g *gin.Context, req *http.Request)
 			if brc, ok := req.Body.(dataext.BufferedReadCloser); ok {
 				if bin, err := brc.BufferedAll(); err == nil {
 					if len(bin) < 16*1024 {
-						b.Bytes("gin.body", bin)
+						b.Bytes("gin_body", bin)
 					} else {
-						b.Str("gin.body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
+						b.Str("gin_body", fmt.Sprintf("[[%v bytes | %s]]", len(bin), req.Header.Get("Content-Type")))
 					}
 				}
 			}

exerr/gin.go

@@ -15,10 +15,10 @@ func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) la
 		ginJson["id"] = ee.UniqueID
 	}
 	if ee.Category != CatWrap {
-		ginJson["category"] = ee.Category
+		ginJson["category"] = ee.Category.Category
 	}
 	if ee.Type != TypeWrap {
-		ginJson["type"] = ee.Type
+		ginJson["type"] = ee.Type.Key
 	}
 	if ee.StatusCode != nil {
 		ginJson["statuscode"] = ee.StatusCode
@@ -30,7 +30,7 @@ func (ee *ExErr) toJson(depth int, applyExtendListener bool, outputMeta bool) la
 		ginJson["caller"] = ee.Caller
 	}
 	if ee.Severity != SevErr {
-		ginJson["severity"] = ee.Severity
+		ginJson["severity"] = ee.Severity.Severity
 	}
 	if ee.Timestamp != (time.Time{}) {
 		ginJson["time"] = ee.Timestamp.Format(time.RFC3339)

ginext/engine.go

@@ -19,6 +19,7 @@ type GinWrapper struct {
 	engine          *gin.Engine
 	suppressGinLogs bool
 
+	opt                   Options
 	allowCors             bool
 	ginDebug              bool
 	bufferBody            bool
@@ -37,13 +38,15 @@ type ginRouteSpec struct {
 }
 
 type Options struct {
-	AllowCors             *bool                                     // Add cors handler to allow all CORS requests on the default http methods
-	GinDebug              *bool                                     // Set gin.debug to true (adds more logs)
-	SuppressGinLogs       *bool                                     // Suppress our custom gin logs (even if GinDebug == true)
-	BufferBody            *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
-	Timeout               *time.Duration                            // The default handler timeout
-	ListenerBeforeRequest []func(g *gin.Context)                    // Register listener that are called before the handler method
-	ListenerAfterRequest  []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+	AllowCors                *bool                                     // Add cors handler to allow all CORS requests on the default http methods
+	GinDebug                 *bool                                     // Set gin.debug to true (adds more logs)
+	SuppressGinLogs          *bool                                     // Suppress our custom gin logs (even if GinDebug == true)
+	BufferBody               *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
+	Timeout                  *time.Duration                            // The default handler timeout
+	ListenerBeforeRequest    []func(g *gin.Context)                    // Register listener that are called before the handler method
+	ListenerAfterRequest     []func(g *gin.Context, resp HTTPResponse) // Register listener that are called after the handler method
+	DebugTrimHandlerPrefixes []string                                  // Trim these prefixes from the handler names in the debug print
+	DebugReplaceHandlerNames map[string]string                         // Replace handler names in debug output
 }
 
 // NewEngine creates a new (wrapped) ginEngine
@@ -52,6 +55,7 @@ func NewEngine(opt Options) *GinWrapper {
 
 	wrapper := &GinWrapper{
 		engine:                engine,
+		opt:                   opt,
 		suppressGinLogs:       langext.Coalesce(opt.SuppressGinLogs, false),
 		allowCors:             langext.Coalesce(opt.AllowCors, false),
 		ginDebug:              langext.Coalesce(opt.GinDebug, true),
@@ -185,6 +189,18 @@ func (w *GinWrapper) cleanMiddlewareName(fname string) string {
 		}
 	}
 
+	for _, pfx := range w.opt.DebugTrimHandlerPrefixes {
+		if strings.HasPrefix(fname, pfx) {
+			fname = fname[len(pfx):]
+		}
+	}
+
+	for k, v := range langext.ForceMap(w.opt.DebugReplaceHandlerNames) {
+		if strings.EqualFold(fname, k) {
+			fname = v
+		}
+	}
+
 	return fname
 }
 

go.mod

@@ -5,7 +5,7 @@ go 1.22
 require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.-
-	github.com/jmoiron/sqlx v1.3.5
+	github.com/jmoiron/sqlx v1.4.0
 	github.com/rs/xid v1.5.0
 	github.com/rs/zerolog v1.32.0
 	go.mongodb.org/mongo-driver v1.15.0
@@ -15,17 +15,18 @@ require (
 )
 
 require (
-	github.com/bytedance/sonic v1.11.4 // indirect
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.1 // indirect
-	github.com/cloudwego/base64x v0.1.0 // indirect
-	github.com/cloudwego/iasm v0.1.0 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.19.0 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.5.0 // indirect
@@ -38,19 +39,19 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect
 	golang.org/x/arch v0.7.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.34.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.37.6 // indirect
 	modernc.org/mathutil v1.6.0 // indirect

go.sum

@@ -1,3 +1,4 @@
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
@@ -12,6 +13,13 @@ github.com/bytedance/sonic v1.11.3 h1:jRN+yEjakWh8aK5FzrciUHG8OFXK+4/KrAX/ysEtHA
 github.com/bytedance/sonic v1.11.3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/bytedance/sonic v1.11.4 h1:8+OMLSSDDm2/qJc6ld5K5Sm62NK9VHcUKk0NzBoMAM4=
 github.com/bytedance/sonic v1.11.4/go.mod h1:YrWEqYtlBPS6LUA0vpuG79a1trsh4Ae41uWUWUreHhE=
+github.com/bytedance/sonic v1.11.5 h1:G00FYjjqll5iQ1PYXynbg/hyzqBqavH8Mo9/oTopd9k=
+github.com/bytedance/sonic v1.11.5/go.mod h1:X2PC2giUdj/Cv2lliWFLk6c/DUQok5rViJSemeB0wDw=
+github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bytedance/sonic/loader v0.1.0/go.mod h1:UmRT+IRTGKz/DAkzcEGzyVqQFJ7H9BqwBO3pm9H/+HY=
+github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
@@ -21,9 +29,15 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0
 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/cloudwego/base64x v0.1.0 h1:Tg5q9tq1khq9Y9UwfoC6zkHK0FypN2GLDvhqFceOL8U=
 github.com/cloudwego/base64x v0.1.0/go.mod h1:lM8nFiNbg74QgesNo6EAtv8N9tlRjBWExmHoNDa3PkU=
+github.com/cloudwego/base64x v0.1.3 h1:b5J/l8xolB7dyDTTmhJP2oTs5LdrjyrUFuNxdfq5hAg=
+github.com/cloudwego/base64x v0.1.3/go.mod h1:1+1K5BUHIQzyapgpF7LwvOGAEDicKtt1umPV+aN8pi8=
+github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
+github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
 github.com/cloudwego/iasm v0.0.9/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/cloudwego/iasm v0.1.0 h1:q0OuhwWDMyi3nlrQ6kIr0Yx0c3FI6cq/OZWKodIDdz8=
 github.com/cloudwego/iasm v0.1.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -52,8 +66,12 @@ github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtP
 github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
 github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
+github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -71,6 +89,8 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
 github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -94,6 +114,7 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -103,6 +124,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
 github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -117,6 +139,8 @@ github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeB
 github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg=
 github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -155,6 +179,8 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
+github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 h1:tBiBTKHnIjovYoLX/TPkcf+OjqqKGQrPtGT3Foz+Pgo=
+github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76/go.mod h1:SQliXeA7Dhkt//vS29v3zpbEwoa+zb2Cn5xj5uO4K5U=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
 go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
@@ -251,6 +277,8 @@ google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4=
+google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.433"
+const GoextVersion = "0.0.444"
 
-const GoextVersionTimestamp = "2024-04-15T10:25:30+0200"
+const GoextVersionTimestamp = "2024-05-03T13:24:08+0200"

langext/array.go

@@ -477,6 +477,10 @@ func ArrRemove[T comparable](arr []T, needle T) []T {
 	return arr
 }
 
+func ArrRemoveAt[T any](arr []T, idx int) []T {
+	return append(arr[:idx], arr[idx+1:]...)
+}
+
 func ArrExcept[T comparable](arr []T, needles ...T) []T {
 	r := make([]T, 0, len(arr))
 	rmlist := ArrToSet(needles)

langext/pointer.go

@@ -27,6 +27,10 @@ func DblPtrNil[T any]() **T {
 	return &v
 }
 
+func ArrPtr[T any](v ...T) *[]T {
+	return &v
+}
+
 func PtrInt32(v int32) *int32 {
 	return &v
 }

reflectext/convertToMap.go

@@ -8,6 +8,7 @@ import (
 
 type ConvertStructToMapOpt struct {
 	KeepJsonMarshalTypes bool
+	MaxDepth             *int
 }
 
 func ConvertStructToMap(v any, opts ...ConvertStructToMapOpt) map[string]any {
@@ -16,7 +17,7 @@ func ConvertStructToMap(v any, opts ...ConvertStructToMapOpt) map[string]any {
 		opt = opts[0]
 	}
 
-	res := reflectToMap(reflect.ValueOf(v), opt)
+	res := reflectToMap(reflect.ValueOf(v), 1, opt)
 
 	if v, ok := res.(map[string]any); ok {
 		return v
@@ -27,14 +28,18 @@ func ConvertStructToMap(v any, opts ...ConvertStructToMapOpt) map[string]any {
 	}
 }
 
-func reflectToMap(fv reflect.Value, opt ConvertStructToMapOpt) any {
+func reflectToMap(fv reflect.Value, depth int, opt ConvertStructToMapOpt) any {
+
+	if opt.MaxDepth != nil && depth > *opt.MaxDepth {
+		return fv.Interface()
+	}
 
 	if fv.Kind() == reflect.Ptr {
 
 		if fv.IsNil() {
 			return nil
 		} else {
-			return reflectToMap(fv.Elem(), opt)
+			return reflectToMap(fv.Elem(), depth, opt)
 		}
 
 	}
@@ -51,7 +56,7 @@ func reflectToMap(fv reflect.Value, opt ConvertStructToMapOpt) any {
 		arrlen := fv.Len()
 		arr := make([]any, arrlen)
 		for i := 0; i < arrlen; i++ {
-			arr[i] = reflectToMap(fv.Index(i), opt)
+			arr[i] = reflectToMap(fv.Index(i), depth+1, opt)
 		}
 		return arr
 
@@ -62,7 +67,7 @@ func reflectToMap(fv reflect.Value, opt ConvertStructToMapOpt) any {
 		arrlen := fv.Len()
 		arr := make([]any, arrlen)
 		for i := 0; i < arrlen; i++ {
-			arr[i] = reflectToMap(fv.Index(i), opt)
+			arr[i] = reflectToMap(fv.Index(i), depth+1, opt)
 		}
 		return arr
 
@@ -85,7 +90,7 @@ func reflectToMap(fv reflect.Value, opt ConvertStructToMapOpt) any {
 
 		for i := 0; i < fv.NumField(); i++ {
 			if fv.Type().Field(i).IsExported() {
-				res[fv.Type().Field(i).Name] = reflectToMap(fv.Field(i), opt)
+				res[fv.Type().Field(i).Name] = reflectToMap(fv.Field(i), depth+1, opt)
 			}
 		}
 

rfctime/rfc3339.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"time"
 )
@@ -245,6 +246,13 @@ func NewRFC3339(t time.Time) RFC3339Time {
 	return RFC3339Time(t)
 }
 
+func NewRFC3339Ptr(t *time.Time) *RFC3339Time {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(RFC3339Time(*t))
+}
+
 func NowRFC3339() RFC3339Time {
 	return RFC3339Time(time.Now())
 }

rfctime/rfc3339Nano.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"time"
 )
@@ -245,6 +246,13 @@ func NewRFC3339Nano(t time.Time) RFC3339NanoTime {
 	return RFC3339NanoTime(t)
 }
 
+func NewRFC3339NanoPtr(t *time.Time) *RFC3339NanoTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(RFC3339NanoTime(*t))
+}
+
 func NowRFC3339Nano() RFC3339NanoTime {
 	return RFC3339NanoTime(time.Now())
 }

rfctime/unix.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"strconv"
 	"time"
@@ -239,6 +240,13 @@ func NewUnix(t time.Time) UnixTime {
 	return UnixTime(t)
 }
 
+func NewUnixPtr(t *time.Time) *UnixTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(UnixTime(*t))
+}
+
 func NowUnix() UnixTime {
 	return UnixTime(time.Now())
 }

rfctime/unixMilli.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"strconv"
 	"time"
@@ -239,6 +240,13 @@ func NewUnixMilli(t time.Time) UnixMilliTime {
 	return UnixMilliTime(t)
 }
 
+func NewUnixMilliPtr(t *time.Time) *UnixMilliTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(UnixMilliTime(*t))
+}
+
 func NowUnixMilli() UnixMilliTime {
 	return UnixMilliTime(time.Now())
 }

rfctime/unixNano.go

@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsoncodec"
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
 	"strconv"
 	"time"
@@ -239,6 +240,13 @@ func NewUnixNano(t time.Time) UnixNanoTime {
 	return UnixNanoTime(t)
 }
 
+func NewUnixNanoPtr(t *time.Time) *UnixNanoTime {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(UnixNanoTime(*t))
+}
+
 func NowUnixNano() UnixNanoTime {
 	return UnixNanoTime(time.Now())
 }

wmo/collection.go

@@ -125,3 +125,17 @@ func (c *Coll[TData]) createToken(fieldPrimary string, dirPrimary ct.SortDirecti
 		Extra:          ct.Extra{},
 	}, nil
 }
+
+func (c *Coll[TData]) needsDoubleSort(ctx context.Context) bool {
+	for _, ppl := range c.extraModPipeline {
+		for _, stage := range ppl(ctx) {
+			for _, bsone := range stage {
+				if bsone.Key == "$group" {
+					// a group stage in extraModPipeline results in unsorted data, which means the caller must sort again after these pipeline stages...
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

wmo/queryFind.go

@@ -36,6 +36,14 @@ func (c *Coll[TData]) Find(ctx context.Context, filter bson.M, opts ...*options.
 		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
 	}
 
+	if c.needsDoubleSort(ctx) {
+		for _, opt := range opts {
+			if opt != nil && opt.Sort != nil {
+				pipeline = append(pipeline, bson.D{{Key: "$sort", Value: opt.Sort}})
+			}
+		}
+	}
+
 	for _, opt := range opts {
 		if opt != nil && opt.Projection != nil {
 			pipeline = append(pipeline, bson.D{{Key: "$project", Value: opt.Projection}})

wmo/queryList.go

@@ -14,6 +14,10 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		return make([]TData, 0), ct.End(), nil
 	}
 
+	if pageSize != nil && *pageSize == 0 {
+		return make([]TData, 0), inTok, nil // fast track, we return an empty list and do not advance the cursor token
+	}
+
 	pipeline := mongo.Pipeline{}
 	pf1 := "_id"
 	pd1 := ct.SortASC
@@ -35,7 +39,7 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		sortDirSecondary = nil
 	}
 
-	paginationPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize)
+	paginationPipeline, doubleSortPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize)
 	if err != nil {
 		return nil, ct.CursorToken{}, exerr.
 			Wrap(err, "failed to create pagination").
@@ -56,6 +60,10 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int,
 		pipeline = langext.ArrConcat(pipeline, ppl(ctx))
 	}
 
+	if c.needsDoubleSort(ctx) {
+		pipeline = langext.ArrConcat(pipeline, doubleSortPipeline)
+	}
+
 	cursor, err := c.coll.Aggregate(ctx, pipeline)
 	if err != nil {
 		return nil, ct.CursorToken{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Str("collection", c.Name()).Build()
@@ -136,14 +144,50 @@ func (c *Coll[TData]) ListWithCount(ctx context.Context, filter ct.Filter, pageS
 	return data, token, count, nil
 }
 
-func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, error) {
+func (c *Coll[TData]) ListAllIDs(ctx context.Context, filter ct.RawFilter) ([]string, error) {
+	type idObject struct {
+		ID string `bson:"_id"`
+	}
+
+	pipelineFilter := mongo.Pipeline{}
+
+	if filter != nil {
+		pipelineFilter = filter.FilterQuery()
+	}
+
+	extrModPipelineResolved := mongo.Pipeline{}
+	for _, ppl := range c.extraModPipeline {
+		extrModPipelineResolved = langext.ArrConcat(extrModPipelineResolved, ppl(ctx))
+	}
+
+	pipelineProjectIDs := mongo.Pipeline{}
+	pipelineProjectIDs = append(pipelineProjectIDs, bson.D{{Key: "$project", Value: bson.M{"_id": 1}}})
+
+	pipelineList := langext.ArrConcat(pipelineFilter, extrModPipelineResolved, pipelineProjectIDs)
+
+	cursorList, err := c.coll.Aggregate(ctx, pipelineList)
+	if err != nil {
+		return nil, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipelineList).Str("collection", c.Name()).Build()
+	}
+
+	var res []idObject
+
+	err = cursorList.All(ctx, &res)
+	if err != nil {
+		return nil, exerr.Wrap(err, "failed to decode entities").Any("pipeline", pipelineList).Str("collection", c.Name()).Build()
+	}
+
+	return langext.ArrMap(res, func(v idObject) string { return v.ID }), nil
+}
+
+func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, []bson.D, error) {
 
 	cond := bson.A{}
 	sort := bson.D{}
 
 	valuePrimary, err := coll.getTokenValueAsMongoType(token.ValuePrimary, fieldPrimary)
 	if err != nil {
-		return nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
+		return nil, nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
 	}
 
 	if sortPrimary == ct.SortASC {
@@ -160,7 +204,7 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken
 
 		valueSecondary, err := coll.getTokenValueAsMongoType(token.ValueSecondary, *fieldSecondary)
 		if err != nil {
-			return nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
+			return nil, nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
 		}
 
 		if *sortSecondary == ct.SortASC {
@@ -203,15 +247,17 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken
 
 	} else {
 
-		return nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build()
+		return nil, nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build()
 
 	}
 
 	pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}})
 
+	pipelineSort := mongo.Pipeline{bson.D{{Key: "$sort", Value: sort}}}
+
 	if pageSize != nil {
 		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}})
 	}
 
-	return pipeline, nil
+	return pipeline, pipelineSort, nil
 }