From fad2e4ff6d1ef91ee4ed4f44edfd4eb5d39017f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Fri, 22 May 2026 22:06:05 +0200 Subject: [PATCH] v0.0.641 Handle cursortokens with non-decodable values gracefully --- bfcodegen/id-generate.template | 3 + cursortoken/token.go | 3 +- go.mod | 8 +-- go.sum | 8 +++ goextVersion.go | 4 +- wmo/collection.go | 9 +++ wmo/queryList.go | 80 ++++++++++++++++-------- wmo/queryList_test.go | 107 +++++++++++++++++++++++++++++++++ 8 files changed, 190 insertions(+), 32 deletions(-) create mode 100644 wmo/queryList_test.go diff --git a/bfcodegen/id-generate.template b/bfcodegen/id-generate.template index eddf863..d0b9f8a 100644 --- a/bfcodegen/id-generate.template +++ b/bfcodegen/id-generate.template @@ -4,6 +4,7 @@ package {{.PkgName}} import "go.mongodb.org/mongo-driver/v2/bson" import "git.blackforestbytes.com/BlackForestBytes/goext/exerr" +import "git.blackforestbytes.com/BlackForestBytes/goext/wmo" const ChecksumIDGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}} @@ -52,4 +53,6 @@ func New{{.Name}}() {{.Name}} { return {{.Name}}(bson.NewObjectID().Hex()) } +var _ wmo.MongoEntityID = (*{{.Name}})(nil) + {{end}} \ No newline at end of file diff --git a/cursortoken/token.go b/cursortoken/token.go index e17e2a9..7957899 100644 --- a/cursortoken/token.go +++ b/cursortoken/token.go @@ -3,10 +3,11 @@ package cursortoken import ( "encoding/base32" "encoding/json" - "git.blackforestbytes.com/BlackForestBytes/goext/exerr" "strconv" "strings" "time" + + "git.blackforestbytes.com/BlackForestBytes/goext/exerr" ) type CursorToken interface { diff --git a/go.mod b/go.mod index be97082..6f78d14 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.35.1 go.mongodb.org/mongo-driver/v2 v2.6.0 - golang.org/x/crypto v0.51.0 - golang.org/x/sys v0.44.0 + golang.org/x/crypto v0.52.0 + golang.org/x/sys v0.45.0 golang.org/x/term v0.43.0 ) @@ -63,8 +63,8 @@ require ( github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/arch v0.27.0 // indirect - golang.org/x/image v0.40.0 // indirect - golang.org/x/net v0.54.0 // indirect + golang.org/x/image v0.41.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/protobuf v1.36.11 // indirect modernc.org/libc v1.72.0 // indirect diff --git a/go.sum b/go.sum index c0c4a4e..53979ee 100644 --- a/go.sum +++ b/go.sum @@ -164,12 +164,16 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww= golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA= golang.org/x/image v0.40.0 h1:Tw4GyDXMo+daZN1znreBRC3VayR1aLFUyUEOLUdW1a8= golang.org/x/image v0.40.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= +golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo= +golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= @@ -181,6 +185,8 @@ golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= @@ -194,6 +200,8 @@ golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= diff --git a/goextVersion.go b/goextVersion.go index 2a19d7e..df583fc 100644 --- a/goextVersion.go +++ b/goextVersion.go @@ -1,5 +1,5 @@ package goext -const GoextVersion = "0.0.640" +const GoextVersion = "0.0.641" -const GoextVersionTimestamp = "2026-05-12T10:54:15+0200" +const GoextVersionTimestamp = "2026-05-22T22:06:05+0200" diff --git a/wmo/collection.go b/wmo/collection.go index bd18119..83ce199 100644 --- a/wmo/collection.go +++ b/wmo/collection.go @@ -7,14 +7,23 @@ import ( ct "git.blackforestbytes.com/BlackForestBytes/goext/cursortoken" "git.blackforestbytes.com/BlackForestBytes/goext/exerr" "git.blackforestbytes.com/BlackForestBytes/goext/langext" + "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" ) type EntityID interface { MarshalBSONValue() (byte, []byte, error) + IsZero() bool String() string } +type MongoEntityID interface { + EntityID + + ObjID() (bson.ObjectID, error) + Valid() bool +} + type Decodable interface { Decode(v any) error } diff --git a/wmo/queryList.go b/wmo/queryList.go index 8dde640..74c211e 100644 --- a/wmo/queryList.go +++ b/wmo/queryList.go @@ -401,14 +401,18 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort, return nil, nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build() } - if sortPrimary == ct.SortASC { - // We sort ASC on - so we want all entries newer ($gt) than the $primary - cond = append(cond, bson.M{fieldPrimary: bson.M{"$gt": valuePrimary}}) - sort = append(sort, bson.E{Key: fieldPrimary, Value: +1}) - } else if sortPrimary == ct.SortDESC { - // We sort DESC on - so we want all entries older ($lt) than the $primary - cond = append(cond, bson.M{fieldPrimary: bson.M{"$lt": valuePrimary}}) - sort = append(sort, bson.E{Key: fieldPrimary, Value: -1}) + if isValidTokenValue(valuePrimary) { + + if sortPrimary == ct.SortASC { + // We sort ASC on - so we want all entries newer ($gt) than the $primary + cond = append(cond, bson.M{fieldPrimary: bson.M{"$gt": valuePrimary}}) + sort = append(sort, bson.E{Key: fieldPrimary, Value: +1}) + } else if sortPrimary == ct.SortDESC { + // We sort DESC on - so we want all entries older ($lt) than the $primary + cond = append(cond, bson.M{fieldPrimary: bson.M{"$lt": valuePrimary}}) + sort = append(sort, bson.E{Key: fieldPrimary, Value: -1}) + } + } if fieldSecondary != nil && sortSecondary != nil && *fieldSecondary != fieldPrimary { @@ -418,25 +422,29 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort, return nil, nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build() } - if *sortSecondary == ct.SortASC { + if isValidTokenValue(valueSecondary) { - // the conflict-resolution condition, for entries with the _same_ as the $primary we take the ones with a greater $secondary (= newer) - cond = append(cond, bson.M{"$and": bson.A{ - bson.M{"$or": bson.A{bson.M{fieldPrimary: valuePrimary}, bson.M{fieldPrimary: nil}, bson.M{fieldPrimary: bson.M{"$exists": false}}}}, - bson.M{*fieldSecondary: bson.M{"$gt": valueSecondary}}, - }}) + if *sortSecondary == ct.SortASC { - sort = append(sort, bson.E{Key: *fieldSecondary, Value: +1}) + // the conflict-resolution condition, for entries with the _same_ as the $primary we take the ones with a greater $secondary (= newer) + cond = append(cond, bson.M{"$and": bson.A{ + bson.M{"$or": bson.A{bson.M{fieldPrimary: valuePrimary}, bson.M{fieldPrimary: nil}, bson.M{fieldPrimary: bson.M{"$exists": false}}}}, + bson.M{*fieldSecondary: bson.M{"$gt": valueSecondary}}, + }}) - } else if *sortSecondary == ct.SortDESC { + sort = append(sort, bson.E{Key: *fieldSecondary, Value: +1}) - // the conflict-resolution condition, for entries with the _same_ as the $primary we take the ones with a smaller $secondary (= older) - cond = append(cond, bson.M{"$and": bson.A{ - bson.M{"$or": bson.A{bson.M{fieldPrimary: valuePrimary}, bson.M{fieldPrimary: nil}, bson.M{fieldPrimary: bson.M{"$exists": false}}}}, - bson.M{*fieldSecondary: bson.M{"$lt": valueSecondary}}, - }}) + } else if *sortSecondary == ct.SortDESC { - sort = append(sort, bson.E{Key: *fieldSecondary, Value: -1}) + // the conflict-resolution condition, for entries with the _same_ as the $primary we take the ones with a smaller $secondary (= older) + cond = append(cond, bson.M{"$and": bson.A{ + bson.M{"$or": bson.A{bson.M{fieldPrimary: valuePrimary}, bson.M{fieldPrimary: nil}, bson.M{fieldPrimary: bson.M{"$exists": false}}}}, + bson.M{*fieldSecondary: bson.M{"$lt": valueSecondary}}, + }}) + + sort = append(sort, bson.E{Key: *fieldSecondary, Value: -1}) + + } } } @@ -449,7 +457,9 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort, } else if token.Mode == ct.CTMNormal { - pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$or": cond}}}) + if len(cond) > 0 { + pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"$or": cond}}}) + } } else if token.Mode == ct.CTMEnd { @@ -462,9 +472,15 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort, } - pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}}) + if len(sort) > 0 { + pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}}) + } - pipelineSort := mongo.Pipeline{bson.D{{Key: "$sort", Value: sort}}} + pipelineSort := mongo.Pipeline{} + + if len(sort) > 0 { + pipelineSort = append(pipelineSort, bson.D{{Key: "$sort", Value: sort}}) + } if pageSize != nil { pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}}) @@ -473,6 +489,20 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CTKeySort, return pipeline, pipelineSort, nil } +func isValidTokenValue(val any) bool { + + // special handling for Mongo EntityIDs + // We want to prevent failing late (in mongoDB layer) when client sends invalid IDs as token values + + if mongoID, ok := val.(MongoEntityID); ok { + if !mongoID.Valid() { + return false + } + } + + return true +} + func createSortOnlyPipeline(fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection) ([]bson.D, error) { sort := bson.D{} diff --git a/wmo/queryList_test.go b/wmo/queryList_test.go new file mode 100644 index 0000000..5fd2576 --- /dev/null +++ b/wmo/queryList_test.go @@ -0,0 +1,107 @@ +package wmo + +import ( + "encoding/json" + "testing" + + ct "git.blackforestbytes.com/BlackForestBytes/goext/cursortoken" + "git.blackforestbytes.com/BlackForestBytes/goext/exerr" + "git.blackforestbytes.com/BlackForestBytes/goext/langext" + "git.blackforestbytes.com/BlackForestBytes/goext/rfctime" + "git.blackforestbytes.com/BlackForestBytes/goext/tst" + "go.mongodb.org/mongo-driver/v2/bson" +) + +type TestExampleID string //@id:type + +func (i TestExampleID) MarshalBSONValue() (byte, []byte, error) { + if objId, err := bson.ObjectIDFromHex(string(i)); err == nil { + tp, data, err := bson.MarshalValue(objId) + return byte(tp), data, err + } else { + return 0, nil, exerr.New(exerr.TypeMarshalEntityID, "Failed to marshal UserID("+i.String()+") to ObjectId").Str("value", string(i)).Type("type", i).Build() + } +} + +func (i TestExampleID) String() string { + return string(i) +} + +func (i TestExampleID) ObjID() (bson.ObjectID, error) { + return bson.ObjectIDFromHex(string(i)) +} + +func (i TestExampleID) Valid() bool { + _, err := bson.ObjectIDFromHex(string(i)) + return err == nil +} + +func (i TestExampleID) IsZero() bool { + return i == "" +} + +var _ MongoEntityID = (*TestExampleID)(nil) + +func TestCreatePaginationPipelineWithInvalidObjectIDValue(t *testing.T) { + // + // {"v1":"2026-05-16T21:44:13.16Z","v2":"6a08e52d144a332c2e2d067b","dir":"DESC","dir2":"DESC","size":50} + strtok := "tok_PMRHMMJCHIRDEMBSGYWTANJNGE3FIMRRHI2DIORRGMXDCNS2EIWCE5RSEI5CENTBGA4GKNJSMQYTINDBGMZTEYZSMUZGIMBWG5RCELBCMRUXEIR2EJCEKU2DEIWCEZDJOIZCEORCIRCVGQZCFQRHG2L2MURDUNJQPU======" + + tok, err := ct.Decode(strtok) + if err != nil { + t.Error("Failed to decode token:", err) + return + } + + type obj struct { + ID TestExampleID `bson:"_id"` + Created rfctime.RFC3339NanoTime `bson:"created"` + } + + coll := Coll[obj]{coll: nil} + coll.init() + + paginationPipeline, doubleSortPipeline, err := createPaginationPipeline[obj](&coll, tok.(ct.CTKeySort), "_id", ct.SortDESC, new("_id"), new(ct.SortDESC), new(15)) + if err != nil { + t.Error("Failed to create pagination Pipeline:", err) + return + } + + //fmt.Printf("# paginationPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(paginationPipeline)))) + //fmt.Printf("# doubleSortPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(doubleSortPipeline)))) + + tst.AssertEqual(t, string(langext.Must(json.Marshal(paginationPipeline))), `[{"$limit":16}]`) // wrong (_id with wrong date type) fields are not used + tst.AssertEqual(t, string(langext.Must(json.Marshal(doubleSortPipeline))), `[]`) // wrong (_id with wrong date type) fields are not used +} + +func TestCreatePaginationPipelineWithValidObjectIDValue(t *testing.T) { + // + // {"v1":"6a08e52d144a332c2e2d067b","v2":"6a08e52d144a332c2e2d067b","dir":"DESC","dir2":"DESC","size":50} + strtok := "tok_PMFCAIBAEARHMMJCHIQCENTBGA4GKNJSMQYTINDBGMZTEYZSMUZGIMBWG5RCELAKEAQCAIBCOYZCEORAEI3GCMBYMU2TEZBRGQ2GCMZTGJRTEZJSMQYDMN3CEIWAUIBAEAQCEZDJOIRDUIBCIRCVGQZCFQFCAIBAEARGI2LSGIRDUIBCIRCVGQZCFQFCAIBAEARHG2L2MURDUIBVGAFH2===" + + tok, err := ct.Decode(strtok) + if err != nil { + t.Error("Failed to decode token:", err) + return + } + + type obj struct { + ID TestExampleID `bson:"_id"` + Created rfctime.RFC3339NanoTime `bson:"created"` + } + + coll := Coll[obj]{coll: nil} + coll.init() + + paginationPipeline, doubleSortPipeline, err := createPaginationPipeline[obj](&coll, tok.(ct.CTKeySort), "_id", ct.SortDESC, new("_id"), new(ct.SortDESC), new(15)) + if err != nil { + t.Error("Failed to create pagination Pipeline:", err) + return + } + + //fmt.Printf("# paginationPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(paginationPipeline)))) + //fmt.Printf("# doubleSortPipeline:\n%+v\n\n", string(langext.Must(json.Marshal(doubleSortPipeline)))) + + tst.AssertEqual(t, string(langext.Must(json.Marshal(paginationPipeline))), `[{"$match":{"$or":[{"_id":{"$lt":"6a08e52d144a332c2e2d067b"}}]}},{"$sort":{"_id":-1}},{"$limit":16}]`) + tst.AssertEqual(t, string(langext.Must(json.Marshal(doubleSortPipeline))), `[{"$sort":{"_id":-1}}]`) +}