v0.0.641 Handle cursortokens with non-decodable values gracefully

bfcodegen/id-generate.template

@@ -4,6 +4,7 @@ package {{.PkgName}}
 
 import "go.mongodb.org/mongo-driver/v2/bson"
 import "git.blackforestbytes.com/BlackForestBytes/goext/exerr"
+import "git.blackforestbytes.com/BlackForestBytes/goext/wmo"
 
 const ChecksumIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
 
@@ -52,4 +53,6 @@ func New{{.Name}}() {{.Name}} {
 	return {{.Name}}(bson.NewObjectID().Hex())
 }
 
+var _ wmo.MongoEntityID = (*{{.Name}})(nil)
+
 {{end}}

cursortoken/token.go

@@ -3,10 +3,11 @@ package cursortoken
 import (
 	"encoding/base32"
 	"encoding/json"
-	"git.blackforestbytes.com/BlackForestBytes/goext/exerr"
 	"strconv"
 	"strings"
 	"time"
+
+	"git.blackforestbytes.com/BlackForestBytes/goext/exerr"
 )
 
 type CursorToken interface {

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/rs/xid v1.6.0
 	github.com/rs/zerolog v1.35.1
 	go.mongodb.org/mongo-driver/v2 v2.6.0
-	golang.org/x/crypto v0.51.0
-	golang.org/x/sys v0.44.0
+	golang.org/x/crypto v0.52.0
+	golang.org/x/sys v0.45.0
 	golang.org/x/term v0.43.0
 )
 
@@ -63,8 +63,8 @@ require (
 	github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	golang.org/x/arch v0.27.0 // indirect
-	golang.org/x/image v0.40.0 // indirect
-	golang.org/x/net v0.54.0 // indirect
+	golang.org/x/image v0.41.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/text v0.37.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	modernc.org/libc v1.72.0 // indirect

go.sum

@@ -164,12 +164,16 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
 golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
 golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
 golang.org/x/image v0.40.0 h1:Tw4GyDXMo+daZN1znreBRC3VayR1aLFUyUEOLUdW1a8=
 golang.org/x/image v0.40.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
+golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
+golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
@@ -181,6 +185,8 @@ golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
 golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
 golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
@@ -194,6 +200,8 @@ golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
 golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.640"
+const GoextVersion = "0.0.641"
 
-const GoextVersionTimestamp = "2026-05-12T10:54:15+0200"
+const GoextVersionTimestamp = "2026-05-22T22:06:05+0200"

wmo/collection.go

@@ -7,14 +7,23 @@ import (
 	ct "git.blackforestbytes.com/BlackForestBytes/goext/cursortoken"
 	"git.blackforestbytes.com/BlackForestBytes/goext/exerr"
 	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"go.mongodb.org/mongo-driver/v2/bson"
 	"go.mongodb.org/mongo-driver/v2/mongo"
 )
 
 type EntityID interface {
 	MarshalBSONValue() (byte, []byte, error)
+	IsZero() bool
 	String() string
 }
 
+type MongoEntityID interface {
+	EntityID
+
+	ObjID() (bson.ObjectID, error)
+	Valid() bool
+}
+
 type Decodable interface {
 	Decode(v any) error
 }

wmo/queryList.go

@@ -401,6 +401,8 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 		return nil, nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build()
 	}
 
+	if isValidTokenValue(valuePrimary) {
+
 		if sortPrimary == ct.SortASC {
 			// We sort ASC on <field> - so we want all entries newer ($gt) than the $primary
 			cond = append(cond, bson.M{fieldPrimary: bson.M{"$gt": valuePrimary}})
@@ -411,6 +413,8 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 			sort = append(sort, bson.E{Key: fieldPrimary, Value: -1})
 		}
 
+	}
+
 	if fieldSecondary != nil && sortSecondary != nil && *fieldSecondary != fieldPrimary {
 
 		valueSecondary, err := coll.getTokenValueAsMongoType(token.ValueSecondary, *fieldSecondary)
@@ -418,6 +422,8 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 			return nil, nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build()
 		}
 
+		if isValidTokenValue(valueSecondary) {
+
 			if *sortSecondary == ct.SortASC {
 
 				// the conflict-resolution condition, for entries with the _same_ <field> as the $primary we take the ones with a greater $secondary (= newer)
@@ -439,6 +445,8 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 				sort = append(sort, bson.E{Key: *fieldSecondary, Value: -1})
 
 			}
+
+		}
 	}
 
 	pipeline := make([]bson.D, 0, 3)
@@ -449,7 +457,9 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 
 	} else if token.Mode == ct.CTMNormal {
 
+		if len(cond) > 0 {
 			pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$or": cond}}})
+		}
 
 	} else if token.Mode == ct.CTMEnd {
 
@@ -462,9 +472,15 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 
 	}
 
+	if len(sort) > 0 {
 		pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}})
+	}
 
-	pipelineSort := mongo.Pipeline{bson.D{{Key: "$sort", Value: sort}}}
+	pipelineSort := mongo.Pipeline{}
+
+	if len(sort) > 0 {
+		pipelineSort = append(pipelineSort, bson.D{{Key: "$sort", Value: sort}})
+	}
 
 	if pageSize != nil {
 		pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}})
@@ -473,6 +489,20 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort,
 	return pipeline, pipelineSort, nil
 }
 
+func isValidTokenValue(val any) bool {
+
+	// special handling for Mongo EntityIDs
+	// We want to prevent failing late (in mongoDB layer) when client sends invalid IDs as token values
+
+	if mongoID, ok := val.(MongoEntityID); ok {
+		if !mongoID.Valid() {
+			return false
+		}
+	}
+
+	return true
+}
+
 func createSortOnlyPipeline(fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection) ([]bson.D, error) {
 
 	sort := bson.D{}

wmo/queryList_test.go

@@ -0,0 +1,107 @@
+package wmo
+
+import (
+	"encoding/json"
+	"testing"
+
+	ct "git.blackforestbytes.com/BlackForestBytes/goext/cursortoken"
+	"git.blackforestbytes.com/BlackForestBytes/goext/exerr"
+	"git.blackforestbytes.com/BlackForestBytes/goext/langext"
+	"git.blackforestbytes.com/BlackForestBytes/goext/rfctime"
+	"git.blackforestbytes.com/BlackForestBytes/goext/tst"
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+type TestExampleID string //@id:type
+
+func (i TestExampleID) MarshalBSONValue() (byte, []byte, error) {
+	if objId, err := bson.ObjectIDFromHex(string(i)); err == nil {
+		tp, data, err := bson.MarshalValue(objId)
+		return byte(tp), data, err
+	} else {
+		return 0, nil, exerr.New(exerr.TypeMarshalEntityID, "Failed to marshal UserID("+i.String()+") to ObjectId").Str("value", string(i)).Type("type", i).Build()
+	}
+}
+
+func (i TestExampleID) String() string {
+	return string(i)
+}
+
+func (i TestExampleID) ObjID() (bson.ObjectID, error) {
+	return bson.ObjectIDFromHex(string(i))
+}
+
+func (i TestExampleID) Valid() bool {
+	_, err := bson.ObjectIDFromHex(string(i))
+	return err == nil
+}
+
+func (i TestExampleID) IsZero() bool {
+	return i == ""
+}
+
+var _ MongoEntityID = (*TestExampleID)(nil)
+
+func TestCreatePaginationPipelineWithInvalidObjectIDValue(t *testing.T) {
+	//
+	// {"v1":"2026-05-16T21:44:13.16Z","v2":"6a08e52d144a332c2e2d067b","dir":"DESC","dir2":"DESC","size":50}
+	strtok := "tok_PMRHMMJCHIRDEMBSGYWTANJNGE3FIMRRHI2DIORRGMXDCNS2EIWCE5RSEI5CENTBGA4GKNJSMQYTINDBGMZTEYZSMUZGIMBWG5RCELBCMRUXEIR2EJCEKU2DEIWCEZDJOIZCEORCIRCVGQZCFQRHG2L2MURDUNJQPU======"
+
+	tok, err := ct.Decode(strtok)
+	if err != nil {
+		t.Error("Failed to decode token:", err)
+		return
+	}
+
+	type obj struct {
+		ID      TestExampleID           `bson:"_id"`
+		Created rfctime.RFC3339NanoTime `bson:"created"`
+	}
+
+	coll := Coll[obj]{coll: nil}
+	coll.init()
+
+	paginationPipeline, doubleSortPipeline, err := createPaginationPipeline[obj](&coll, tok.(ct.CTKeySort), "_id", ct.SortDESC, new("_id"), new(ct.SortDESC), new(15))
+	if err != nil {
+		t.Error("Failed to create pagination Pipeline:", err)
+		return
+	}
+
+	//fmt.Printf("# paginationPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(paginationPipeline))))
+	//fmt.Printf("# doubleSortPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(doubleSortPipeline))))
+
+	tst.AssertEqual(t, string(langext.Must(json.Marshal(paginationPipeline))), `[{"$limit":16}]`) // wrong (_id with wrong date type) fields are not used
+	tst.AssertEqual(t, string(langext.Must(json.Marshal(doubleSortPipeline))), `[]`)              // wrong (_id with wrong date type) fields are not used
+}
+
+func TestCreatePaginationPipelineWithValidObjectIDValue(t *testing.T) {
+	//
+	// {"v1":"6a08e52d144a332c2e2d067b","v2":"6a08e52d144a332c2e2d067b","dir":"DESC","dir2":"DESC","size":50}
+	strtok := "tok_PMFCAIBAEARHMMJCHIQCENTBGA4GKNJSMQYTINDBGMZTEYZSMUZGIMBWG5RCELAKEAQCAIBCOYZCEORAEI3GCMBYMU2TEZBRGQ2GCMZTGJRTEZJSMQYDMN3CEIWAUIBAEAQCEZDJOIRDUIBCIRCVGQZCFQFCAIBAEARGI2LSGIRDUIBCIRCVGQZCFQFCAIBAEARHG2L2MURDUIBVGAFH2==="
+
+	tok, err := ct.Decode(strtok)
+	if err != nil {
+		t.Error("Failed to decode token:", err)
+		return
+	}
+
+	type obj struct {
+		ID      TestExampleID           `bson:"_id"`
+		Created rfctime.RFC3339NanoTime `bson:"created"`
+	}
+
+	coll := Coll[obj]{coll: nil}
+	coll.init()
+
+	paginationPipeline, doubleSortPipeline, err := createPaginationPipeline[obj](&coll, tok.(ct.CTKeySort), "_id", ct.SortDESC, new("_id"), new(ct.SortDESC), new(15))
+	if err != nil {
+		t.Error("Failed to create pagination Pipeline:", err)
+		return
+	}
+
+	//fmt.Printf("# paginationPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(paginationPipeline))))
+	//fmt.Printf("# doubleSortPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(doubleSortPipeline))))
+
+	tst.AssertEqual(t, string(langext.Must(json.Marshal(paginationPipeline))), `[{"$match":{"$or":[{"_id":{"$lt":"6a08e52d144a332c2e2d067b"}}]}},{"$sort":{"_id":-1}},{"$limit":16}]`)
+	tst.AssertEqual(t, string(langext.Must(json.Marshal(doubleSortPipeline))), `[{"$sort":{"_id":-1}}]`)
+}