v0.0.402 add PackageName() and TypeName() to enums_codegen

.idea/.gitignore

@@ -6,3 +6,5 @@
 # Datasource local storage ignored files
 /dataSources/
 /dataSources.local.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

README.md

@@ -30,7 +30,7 @@ Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
 | confext     | Mike       | Parses environment configuration into structs                                                                 |
 | cmdext      | Mike       | Runner for external commands/processes                                                                        |
 |             |            |                                                                                                               |
-| sq          | Mike       | Utility functions for sql based databases                                                                     |
+| sq          | Mike       | Utility functions for sql based databases (primarily sqlite)                                                  |
 | tst         | Mike       | Utility functions for unit tests                                                                              |
 |             |            |                                                                                                               |
 | rfctime     | Mike       | Classes for time seriallization, with different marshallign method for mongo and json                         |

TODO.md

@@ -2,6 +2,8 @@
 
  - cronext
 
- - rfctime.DateOnly
  - rfctime.HMSTimeOnly
- - rfctime.NanoTimeOnly
+ - rfctime.NanoTimeOnly
+
+ - remove sqlx dependency from sq  (unmaintained, and mostly superseeded by our own stuff?)
+ - Move DBLogger and DBPreprocessor to sq

bfcodegen/csid-generate.go

@@ -26,6 +26,10 @@ type CSIDDef struct {
 	Prefix       string
 }
 
+type CSIDGenOptions struct {
+	DebugOutput *bool
+}
+
 var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
@@ -35,7 +39,9 @@ var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGen
 //go:embed csid-generate.template
 var templateCSIDGenerateText string
 
-func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
+func GenerateCharsetIDSpecs(sourceDir string, destFile string, opt CSIDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -81,13 +87,18 @@ func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allIDs = append(allIDs, fileIDs...)
 
@@ -113,7 +124,7 @@ func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
+func processCSIDFile(basedir string, fn string, debugOutput bool) ([]CSIDDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -155,7 +166,11 @@ func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
 				Name:         match.GroupByName("name").Value(),
 				Prefix:       match.GroupByName("prefix").Value(),
 			}
-			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
 			ids = append(ids, def)
 		}
 	}

bfcodegen/csid-generate_test.go

@@ -34,10 +34,10 @@ func TestGenerateCSIDSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
-	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

bfcodegen/enum-generate.go

@@ -37,6 +37,11 @@ type EnumDef struct {
 	Values       []EnumDefVal
 }
 
+type EnumGenOptions struct {
+	DebugOutput *bool
+	GoFormat    *bool
+}
+
 var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
@@ -48,7 +53,7 @@ var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerato
 //go:embed enum-generate.template
 var templateEnumGenerateText string
 
-func GenerateEnumSpecs(sourceDir string, destFile string) error {
+func GenerateEnumSpecs(sourceDir string, destFile string, opt EnumGenOptions) error {
 
 	oldChecksum := "N/A"
 	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
@@ -61,7 +66,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		}
 	}
 
-	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, true)
+	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, langext.Coalesce(opt.GoFormat, true), langext.Coalesce(opt.DebugOutput, false))
 	if err != nil {
 		return err
 	}
@@ -78,7 +83,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool) (string, string, bool, error) {
+func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool, debugOutput bool) (string, string, bool, error) {
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -113,13 +118,18 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return "", "", false, err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allEnums = append(allEnums, fileEnums...)
 
@@ -146,7 +156,7 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 	return string(fdata), newChecksum, true, nil
 }
 
-func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
+func processEnumFile(basedir string, fn string, debugOutput bool) ([]EnumDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -190,7 +200,10 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 				Values:       make([]EnumDefVal, 0),
 			}
 			enums = append(enums, def)
-			fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+
+			if debugOutput {
+				fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+			}
 		}
 
 		if match, ok := rexEnumValueDef.MatchFirst(line); ok {
@@ -230,16 +243,21 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 				if v.EnumTypeName == typename {
 					enums[i].Values = append(enums[i].Values, def)
 					found = true
-					if def.Description != nil {
-						fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
-					} else {
-						fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+
+					if debugOutput {
+						if def.Description != nil {
+							fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
+						} else {
+							fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+						}
 					}
 					break
 				}
 			}
 			if !found {
-				fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				if debugOutput {
+					fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				}
 			}
 		}
 	}

bfcodegen/enum-generate.template

@@ -7,6 +7,8 @@ import "gogs.mikescher.com/BlackForestBytes/goext/enums"
 
 const ChecksumEnumGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
 
+{{ $pkgname  := .PkgName }}
+
 {{range .Enums}}
 
 {{ $hasStr   := ( . | hasStr   ) }}
@@ -97,6 +99,14 @@ func (e {{.EnumTypeName}}) VarName() string {
 	return ""
 }
 
+func (e {{.EnumTypeName}}) TypeName() string {
+	return "{{$typename}}"
+}
+
+func (e {{.EnumTypeName}}) PackageName() string {
+	return "{{$pkgname }}"
+}
+
 func (e {{.EnumTypeName}}) Meta() enums.EnumMetaValue {
     {{if $hasDescr}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())} {{else}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil} {{end}}
 }
@@ -134,4 +144,12 @@ func {{.EnumTypeName}}ValuesDescriptionMeta() []enums.EnumDescriptionMetaValue {
 }
 {{end}}
 
-{{end}}
+{{end}}
+
+// ================================ ================= ================================
+
+func AllPackageEnums() []enums.Enum {
+    return []enums.Enum{ {{range .Enums}}
+        {{.EnumTypeName}}, {{end}}
+    }
+}

bfcodegen/enum-generate_test.go

@@ -37,10 +37,10 @@ func TestGenerateEnumSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
 	tst.AssertNoErr(t, err)
 
-	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
 	tst.AssertNoErr(t, err)
 
 	tst.AssertEqual(t, cs1, cs2)
@@ -76,7 +76,7 @@ func TestGenerateEnumSpecsData(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true)
+	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true, true)
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

bfcodegen/id-generate.go

@@ -25,6 +25,10 @@ type IDDef struct {
 	Name         string
 }
 
+type IDGenOptions struct {
+	DebugOutput *bool
+}
+
 var rexIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@id:type).*$`))
@@ -34,7 +38,9 @@ var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator =
 //go:embed id-generate.template
 var templateIDGenerateText string
 
-func GenerateIDSpecs(sourceDir string, destFile string) error {
+func GenerateIDSpecs(sourceDir string, destFile string, opt *IDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -80,13 +86,18 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allIDs = append(allIDs, fileIDs...)
 
@@ -112,7 +123,7 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
+func processIDFile(basedir string, fn string, debugOutput bool) ([]IDDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -153,7 +164,11 @@ func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
 				FileRelative: rfp,
 				Name:         match.GroupByName("name").Value(),
 			}
-			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
 			ids = append(ids, def)
 		}
 	}

bfcodegen/id-generate_test.go

@@ -34,10 +34,10 @@ func TestGenerateIDSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", &IDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
-	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", &IDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

cryptext/pronouncablePassword.go

@@ -0,0 +1,263 @@
+package cryptext
+
+import (
+	"crypto/rand"
+	"io"
+	"math/big"
+	mathrand "math/rand"
+	"strings"
+)
+
+const (
+	ppStartChar            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppEndChar              = "ABDEFIKMNORSTUXYZ"
+	ppVowel                = "AEIOUY"
+	ppConsonant            = "BCDFGHJKLMNPQRSTVWXZ"
+	ppSegmentLenMin        = 3
+	ppSegmentLenMax        = 7
+	ppMaxRepeatedVowel     = 2
+	ppMaxRepeatedConsonant = 2
+)
+
+var ppContinuation = map[uint8]string{
+	'A': "BCDFGHJKLMNPRSTVWXYZ",
+	'B': "ADFIKLMNORSTUY",
+	'C': "AEIKOUY",
+	'D': "AEILORSUYZ",
+	'E': "BCDFGHJKLMNPRSTVWXYZ",
+	'F': "ADEGIKLOPRTUY",
+	'G': "ABDEFHILMNORSTUY",
+	'H': "AEIOUY",
+	'I': "BCDFGHJKLMNPRSTVWXZ",
+	'J': "AEIOUY",
+	'K': "ADEFHILMNORSTUY",
+	'L': "ADEFGIJKMNOPSTUVWYZ",
+	'M': "ABEFIKOPSTUY",
+	'N': "ABEFIKOPSTUY",
+	'O': "BCDFGHJKLMNPRSTVWXYZ",
+	'P': "AEFIJLORSTUY",
+	'Q': "AEIOUY",
+	'R': "ADEFGHIJKLMNOPSTUVYZ",
+	'S': "ACDEIKLOPTUYZ",
+	'T': "AEHIJOPRSUWY",
+	'U': "BCDFGHJKLMNPRSTVWXZ",
+	'V': "AEIOUY",
+	'W': "AEIOUY",
+	'X': "AEIOUY",
+	'Y': "ABCDFGHKLMNPRSTVXZ",
+	'Z': "AEILOTUY",
+}
+
+var ppLog2Map = map[int]float64{
+	1:  0.00000000,
+	2:  1.00000000,
+	3:  1.58496250,
+	4:  2.00000000,
+	5:  2.32192809,
+	6:  2.58496250,
+	7:  2.80735492,
+	8:  3.00000000,
+	9:  3.16992500,
+	10: 3.32192809,
+	11: 3.45943162,
+	12: 3.58496250,
+	13: 3.70043972,
+	14: 3.80735492,
+	15: 3.90689060,
+	16: 4.00000000,
+	17: 4.08746284,
+	18: 4.16992500,
+	19: 4.24792751,
+	20: 4.32192809,
+	21: 4.39231742,
+	22: 4.45943162,
+	23: 4.52356196,
+	24: 4.58496250,
+	25: 4.64385619,
+	26: 4.70043972,
+	27: 4.75488750,
+	28: 4.80735492,
+	29: 4.85798100,
+	30: 4.90689060,
+	31: 4.95419631,
+	32: 5.00000000,
+}
+
+var (
+	ppVowelMap     = ppMakeSet(ppVowel)
+	ppConsonantMap = ppMakeSet(ppConsonant)
+	ppEndCharMap   = ppMakeSet(ppEndChar)
+)
+
+func ppMakeSet(v string) map[uint8]bool {
+	mp := make(map[uint8]bool, len(v))
+	for _, chr := range v {
+		mp[uint8(chr)] = true
+	}
+	return mp
+}
+
+func ppRandInt(rng io.Reader, max int) int {
+	v, err := rand.Int(rng, big.NewInt(int64(max)))
+	if err != nil {
+		panic(err)
+	}
+	return int(v.Int64())
+}
+
+func ppRand(rng io.Reader, chars string, entropy *float64) uint8 {
+	chr := chars[ppRandInt(rng, len(chars))]
+
+	*entropy = *entropy + ppLog2Map[len(chars)]
+
+	return chr
+}
+
+func ppCharType(chr uint8) (bool, bool) {
+	_, ok1 := ppVowelMap[chr]
+	_, ok2 := ppConsonantMap[chr]
+
+	return ok1, ok2
+}
+
+func ppCharsetRemove(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; !ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func ppCharsetFilter(cs string, set map[uint8]bool, allowEmpty bool) string {
+	result := ""
+	for _, chr := range cs {
+		if _, ok := set[uint8(chr)]; ok {
+			result += string(chr)
+		}
+	}
+	if result == "" && !allowEmpty {
+		return cs
+	}
+	return result
+}
+
+func PronouncablePasswordExt(rng io.Reader, pwlen int) (string, float64) {
+
+	// kinda pseudo markov-chain - with a few extra rules and no weights...
+
+	if pwlen <= 0 {
+		return "", 0
+	}
+
+	vowelCount := 0
+	consoCount := 0
+	entropy := float64(0)
+
+	startChar := ppRand(rng, ppStartChar, &entropy)
+
+	result := string(startChar)
+	currentChar := startChar
+
+	isVowel, isConsonant := ppCharType(currentChar)
+	if isVowel {
+		vowelCount = 1
+	}
+	if isConsonant {
+		consoCount = ppMaxRepeatedConsonant
+	}
+
+	segmentLen := 1
+
+	segmentLenTarget := ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+
+	for len(result) < pwlen {
+
+		charset := ppContinuation[currentChar]
+		if vowelCount >= ppMaxRepeatedVowel {
+			charset = ppCharsetRemove(charset, ppVowelMap, false)
+		}
+		if consoCount >= ppMaxRepeatedConsonant {
+			charset = ppCharsetRemove(charset, ppConsonantMap, false)
+		}
+
+		lastOfSegment := false
+		newSegment := false
+
+		if len(result)+1 == pwlen {
+			// last of result
+			charset = ppCharsetFilter(charset, ppEndCharMap, false)
+		} else if segmentLen+1 == segmentLenTarget {
+			// last of segment
+			charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+			if charsetNew != "" {
+				charset = charsetNew
+				lastOfSegment = true
+			}
+		} else if segmentLen >= segmentLenTarget {
+			// (perhaps) start of new segment
+			if _, ok := ppEndCharMap[currentChar]; ok {
+				charset = ppStartChar
+				newSegment = true
+			} else {
+				// continue segment for one more char to (hopefully) find an end-char
+				charsetNew := ppCharsetFilter(charset, ppEndCharMap, true)
+				if charsetNew != "" {
+					charset = charsetNew
+					lastOfSegment = true
+				}
+			}
+		} else {
+			// normal continuation
+		}
+
+		newChar := ppRand(rng, charset, &entropy)
+		if lastOfSegment {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		} else if newSegment {
+			currentChar = newChar
+			segmentLen = 1
+			result += strings.ToUpper(string(newChar))
+			segmentLenTarget = ppSegmentLenMin + ppRandInt(rng, ppSegmentLenMax-ppSegmentLenMin)
+			vowelCount = 0
+			consoCount = 0
+		} else {
+			currentChar = newChar
+			segmentLen++
+			result += strings.ToLower(string(newChar))
+		}
+
+		isVowel, isConsonant := ppCharType(currentChar)
+		if isVowel {
+			vowelCount++
+			consoCount = 0
+		}
+		if isConsonant {
+			vowelCount = 0
+			if newSegment {
+				consoCount = ppMaxRepeatedConsonant
+			} else {
+				consoCount++
+			}
+		}
+	}
+
+	return result, entropy
+}
+
+func PronouncablePassword(len int) string {
+	v, _ := PronouncablePasswordExt(rand.Reader, len)
+	return v
+}
+
+func PronouncablePasswordSeeded(seed int64, len int) string {
+
+	v, _ := PronouncablePasswordExt(mathrand.New(mathrand.NewSource(seed)), len)
+	return v
+}

cryptext/pronouncablePassword_test.go

@@ -0,0 +1,35 @@
+package cryptext
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+)
+
+func TestPronouncablePasswordExt(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw, entropy := PronouncablePasswordExt(rand.New(rand.NewSource(int64(i))), 16)
+		fmt.Printf("[%.2f] => %s\n", entropy, pw)
+	}
+}
+
+func TestPronouncablePasswordSeeded(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw := PronouncablePasswordSeeded(int64(i), 8)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePassword(t *testing.T) {
+	for i := 0; i < 20; i++ {
+		pw := PronouncablePassword(i + 1)
+		fmt.Printf("%s\n", pw)
+	}
+}
+
+func TestPronouncablePasswordWrongLen(t *testing.T) {
+	PronouncablePassword(0)
+	PronouncablePassword(-1)
+	PronouncablePassword(-2)
+	PronouncablePassword(-3)
+}

enums/enum.go

@@ -5,6 +5,8 @@ type Enum interface {
 	ValuesAny() []any
 	ValuesMeta() []EnumMetaValue
 	VarName() string
+	TypeName() string
+	PackageName() string
 }
 
 type StringEnum interface {

exerr/data.go

@@ -48,8 +48,9 @@ var (
 	TypeMongoReflection   = NewType("MONGO_REFLECTION", langext.Ptr(500))
 	TypeMongoInvalidOpt   = NewType("MONGO_INVALIDOPT", langext.Ptr(500))
 
-	TypeSQLQuery = NewType("SQL_QUERY", langext.Ptr(500))
-	TypeSQLBuild = NewType("SQL_BUILD", langext.Ptr(500))
+	TypeSQLQuery  = NewType("SQL_QUERY", langext.Ptr(500))
+	TypeSQLBuild  = NewType("SQL_BUILD", langext.Ptr(500))
+	TypeSQLDecode = NewType("SQL_DECODE", langext.Ptr(500))
 
 	TypeWrap = NewType("Wrap", nil)
 

exerr/meta.go

@@ -9,6 +9,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"math"
 	"strconv"
 	"strings"
 	"time"
@@ -667,6 +668,28 @@ func (v MetaValue) rawValueForJson() any {
 		}
 		return v.Value.(EnumWrap).ValueString
 	}
+	if v.DataType == MDTFloat32 {
+		if math.IsNaN(float64(v.Value.(float32))) {
+			return "float64::NaN"
+		} else if math.IsInf(float64(v.Value.(float32)), +1) {
+			return "float64::+inf"
+		} else if math.IsInf(float64(v.Value.(float32)), -1) {
+			return "float64::-inf"
+		} else {
+			return v.Value
+		}
+	}
+	if v.DataType == MDTFloat64 {
+		if math.IsNaN(v.Value.(float64)) {
+			return "float64::NaN"
+		} else if math.IsInf(v.Value.(float64), +1) {
+			return "float64::+inf"
+		} else if math.IsInf(v.Value.(float64), -1) {
+			return "float64::-inf"
+		} else {
+			return v.Value
+		}
+	}
 	return v.Value
 }
 

ginext/preContext.go

@@ -163,16 +163,16 @@ func (pctx PreContext) Start() (*AppContext, *gin.Context, *HTTPResponse) {
 
 	ictx, cancel := context.WithTimeout(context.Background(), langext.Coalesce(pctx.timeout, pctx.wrapper.requestTimeout))
 
+	actx := CreateAppContext(pctx.ginCtx, ictx, cancel)
+
 	if pctx.persistantData.sessionObj != nil {
-		err := pctx.persistantData.sessionObj.Init(pctx.ginCtx, ictx)
+		err := pctx.persistantData.sessionObj.Init(pctx.ginCtx, actx)
 		if err != nil {
-			cancel()
+			actx.Cancel()
 			return nil, nil, langext.Ptr(Error(exerr.Wrap(err, "Failed to init session").Build()))
 		}
 	}
 
-	actx := CreateAppContext(pctx.ginCtx, ictx, cancel)
-
 	return actx, pctx.ginCtx, nil
 }
 

ginext/routes.go

@@ -53,15 +53,11 @@ func (w *GinRoutesWrapper) Group(relativePath string) *GinRoutesWrapper {
 func (w *GinRoutesWrapper) Use(middleware ...gin.HandlerFunc) *GinRoutesWrapper {
 	defHandler := langext.ArrCopy(w.defaultHandler)
 	defHandler = append(defHandler, middleware...)
-	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler, absPath: w.absPath}
 }
 
 func (w *GinRoutesWrapper) WithJSONFilter(filter string) *GinRoutesWrapper {
-	defHandler := langext.ArrCopy(w.defaultHandler)
-	defHandler = append(defHandler, func(g *gin.Context) {
-		g.Set("goext.jsonfilter", filter)
-	})
-	return &GinRoutesWrapper{wrapper: w.wrapper, routes: w.routes, defaultHandler: defHandler}
+	return w.Use(func(g *gin.Context) { g.Set("goext.jsonfilter", filter) })
 }
 
 func (w *GinRoutesWrapper) GET(relativePath string) *GinRouteBuilder {
@@ -116,10 +112,7 @@ func (w *GinRouteBuilder) Use(middleware ...gin.HandlerFunc) *GinRouteBuilder {
 }
 
 func (w *GinRouteBuilder) WithJSONFilter(filter string) *GinRouteBuilder {
-	w.handlers = append(w.handlers, func(g *gin.Context) {
-		g.Set("goext.jsonfilter", filter)
-	})
-	return w
+	return w.Use(func(g *gin.Context) { g.Set("goext.jsonfilter", filter) })
 }
 
 func (w *GinRouteBuilder) Handle(handler WHandlerFunc) {

ginext/session.go

@@ -6,6 +6,6 @@ import (
 )
 
 type SessionObject interface {
-	Init(g *gin.Context, ctx context.Context) error
+	Init(g *gin.Context, ctx *AppContext) error
 	Finish(ctx context.Context, resp HTTPResponse) error
 }

go.mod

@@ -7,15 +7,15 @@ require (
 	github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.-
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/rs/xid v1.5.0
-	github.com/rs/zerolog v1.31.0
-	go.mongodb.org/mongo-driver v1.13.1
-	golang.org/x/crypto v0.18.0
-	golang.org/x/sys v0.16.0
-	golang.org/x/term v0.16.0
+	github.com/rs/zerolog v1.32.0
+	go.mongodb.org/mongo-driver v1.14.0
+	golang.org/x/crypto v0.21.0
+	golang.org/x/sys v0.18.0
+	golang.org/x/term v0.18.0
 )
 
 require (
-	github.com/bytedance/sonic v1.10.2 // indirect
+	github.com/bytedance/sonic v1.11.2 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -23,14 +23,14 @@ require (
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.16.0 // indirect
+	github.com/go-playground/validator/v10 v10.19.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -45,10 +45,10 @@ require (
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.37.6 // indirect
 	modernc.org/mathutil v1.6.0 // indirect

go.sum

@@ -2,6 +2,12 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
 github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.0 h1:FwNNv6Vu4z2Onf1++LNzxB/QhitD8wuTdpZzMTGITWo=
+github.com/bytedance/sonic v1.11.0/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.1 h1:JC0+6c9FoWYYxakaoa+c5QTtJeiSZNeByOBhXtAFSn4=
+github.com/bytedance/sonic v1.11.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A=
+github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
@@ -31,6 +37,12 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
 github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74=
+github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U=
+github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
+github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
@@ -55,12 +67,20 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
 github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
+github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -91,6 +111,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
+github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -118,6 +140,8 @@ github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
 go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
+go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
+go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
 golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
@@ -129,6 +153,12 @@ golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
+golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -139,6 +169,10 @@ golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
@@ -156,10 +190,18 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
 golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -177,6 +219,8 @@ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSm
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.370"
+const GoextVersion = "0.0.402"
 
-const GoextVersionTimestamp = "2024-01-13T14:10:25+0100"
+const GoextVersionTimestamp = "2024-03-10T12:49:31+0100"

langext/array.go

@@ -265,6 +265,15 @@ func ArrFirstIndex[T comparable](arr []T, needle T) int {
 	return -1
 }
 
+func ArrFirstIndexFunc[T any](arr []T, comp func(v T) bool) int {
+	for i, v := range arr {
+		if comp(v) {
+			return i
+		}
+	}
+	return -1
+}
+
 func ArrLastIndex[T comparable](arr []T, needle T) int {
 	result := -1
 	for i, v := range arr {
@@ -275,6 +284,16 @@ func ArrLastIndex[T comparable](arr []T, needle T) int {
 	return result
 }
 
+func ArrLastIndexFunc[T any](arr []T, comp func(v T) bool) int {
+	result := -1
+	for i, v := range arr {
+		if comp(v) {
+			result = i
+		}
+	}
+	return result
+}
+
 func AddToSet[T comparable](set []T, add T) []T {
 	for _, v := range set {
 		if v == add {
@@ -479,3 +498,33 @@ func JoinString(arr []string, delimiter string) string {
 
 	return str
 }
+
+// ArrChunk splits the array into buckets of max-size `chunkSize`
+// order is being kept.
+// The last chunk may contain less than length elements.
+//
+// (chunkSize == -1) means no chunking
+//
+// see https://www.php.net/manual/en/function.array-chunk.php
+func ArrChunk[T any](arr []T, chunkSize int) [][]T {
+	if chunkSize == -1 {
+		return [][]T{arr}
+	}
+
+	res := make([][]T, 0, 1+len(arr)/chunkSize)
+
+	i := 0
+	for i < len(arr) {
+
+		right := i + chunkSize
+		if right >= len(arr) {
+			right = len(arr)
+		}
+
+		res = append(res, arr[i:right])
+
+		i = right
+	}
+
+	return res
+}

langext/maps.go

@@ -71,3 +71,19 @@ func ForceMap[K comparable, V any](v map[K]V) map[K]V {
 		return v
 	}
 }
+
+func MapMerge[K comparable, V any](base map[K]V, arr ...map[K]V) map[K]V {
+	res := make(map[K]V, len(base)*(1+len(arr)))
+
+	for k, v := range base {
+		res[k] = v
+	}
+
+	for _, m := range arr {
+		for k, v := range m {
+			res[k] = v
+		}
+	}
+
+	return res
+}

langext/must.go

@@ -0,0 +1,21 @@
+package langext
+
+// Must returns a value and panics on error
+//
+// Usage: Must(methodWithError(...))
+func Must[T any](v T, err error) T {
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// MustBool returns a value and panics on missing
+//
+// Usage: MustBool(methodWithOkayReturn(...))
+func MustBool[T any](v T, ok bool) T {
+	if !ok {
+		panic("not ok")
+	}
+	return v
+}

langext/sort.go

@@ -8,12 +8,28 @@ func Sort[T OrderedConstraint](arr []T) {
 	})
 }
 
+func AsSorted[T OrderedConstraint](arr []T) []T {
+	arr = ArrCopy(arr)
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+	return arr
+}
+
 func SortStable[T OrderedConstraint](arr []T) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return arr[i1] < arr[i2]
 	})
 }
 
+func AsSortedStable[T OrderedConstraint](arr []T) []T {
+	arr = ArrCopy(arr)
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return arr[i1] < arr[i2]
+	})
+	return arr
+}
+
 func IsSorted[T OrderedConstraint](arr []T) bool {
 	return sort.SliceIsSorted(arr, func(i1, i2 int) bool {
 		return arr[i1] < arr[i2]
@@ -26,12 +42,28 @@ func SortSlice[T any](arr []T, less func(v1, v2 T) bool) {
 	})
 }
 
+func AsSortedSlice[T any](arr []T, less func(v1, v2 T) bool) []T {
+	arr = ArrCopy(arr)
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+	return arr
+}
+
 func SortSliceStable[T any](arr []T, less func(v1, v2 T) bool) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return less(arr[i1], arr[i2])
 	})
 }
 
+func AsSortedSliceStable[T any](arr []T, less func(v1, v2 T) bool) []T {
+	arr = ArrCopy(arr)
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return less(arr[i1], arr[i2])
+	})
+	return arr
+}
+
 func IsSliceSorted[T any](arr []T, less func(v1, v2 T) bool) bool {
 	return sort.SliceIsSorted(arr, func(i1, i2 int) bool {
 		return less(arr[i1], arr[i2])
@@ -44,12 +76,28 @@ func SortBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TEle
 	})
 }
 
+func AsSortedBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) []TElem {
+	arr = ArrCopy(arr)
+	sort.Slice(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+	return arr
+}
+
 func SortByStable[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return selector(arr[i1]) < selector(arr[i2])
 	})
 }
 
+func AsSortedByStable[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) []TElem {
+	arr = ArrCopy(arr)
+	sort.SliceStable(arr, func(i1, i2 int) bool {
+		return selector(arr[i1]) < selector(arr[i2])
+	})
+	return arr
+}
+
 func IsSortedBy[TElem any, TSel OrderedConstraint](arr []TElem, selector func(v TElem) TSel) {
 	sort.SliceStable(arr, func(i1, i2 int) bool {
 		return selector(arr[i1]) < selector(arr[i2])

langext/url.go

@@ -0,0 +1,29 @@
+package langext
+
+import (
+	"fmt"
+	"strings"
+)
+
+func BuildUrl(url, path string, params *map[string]string) string {
+	if path[:1] == "/" && url[len(url)-1:] == "/" {
+		url += path[1:]
+	} else if path[:1] != "/" && url[len(url)-1:] != "/" {
+		url += "/" + path
+	} else {
+		url += path
+	}
+
+	if params == nil {
+		return url
+	}
+
+	for key, value := range *params {
+		if strings.Contains(url, "?") {
+			url += fmt.Sprintf("&%s=%s", key, value)
+		} else {
+			url += fmt.Sprintf("?%s=%s", key, value)
+		}
+	}
+	return url
+}

langext/url_test.go

@@ -0,0 +1,45 @@
+package langext
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"testing"
+)
+
+func TestBuildUrl(t *testing.T) {
+	tests := []struct {
+		Url    string
+		Path   string
+		Params *map[string]string
+		Want   string
+	}{
+		{
+			Url:    "https://test.heydyno.de/",
+			Path:   "/testing-01",
+			Params: &map[string]string{"param1": "value1"},
+			Want:   "https://test.heydyno.de/testing-01?param1=value1",
+		},
+		{
+			Url:    "https://test.heydyno.de",
+			Path:   "testing-01",
+			Params: &map[string]string{"param1": "value1"},
+			Want:   "https://test.heydyno.de/testing-01?param1=value1",
+		},
+		{
+			Url:    "https://test.heydyno.de",
+			Path:   "/testing-01",
+			Params: nil,
+			Want:   "https://test.heydyno.de/testing-01",
+		},
+		{
+			Url:    "https://test.heydyno.de/",
+			Path:   "testing-01",
+			Params: nil,
+			Want:   "https://test.heydyno.de/testing-01",
+		},
+	}
+
+	for _, test := range tests {
+		res := BuildUrl(test.Url, test.Path, test.Params)
+		tst.AssertEqual(t, res, test.Want)
+	}
+}

pagination/filter.go

@@ -5,7 +5,7 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
-type Filter interface {
+type MongoFilter interface {
 	FilterQuery() mongo.Pipeline
 	Sort() bson.D
 }
@@ -23,6 +23,6 @@ func (d dynamicFilter) Sort() bson.D {
 	return d.sort
 }
 
-func CreateFilter(pipeline mongo.Pipeline, sort bson.D) Filter {
+func CreateFilter(pipeline mongo.Pipeline, sort bson.D) MongoFilter {
 	return dynamicFilter{pipeline: pipeline, sort: sort}
 }

reflectext/convertToMap.go

@@ -0,0 +1,72 @@
+package reflectext
+
+import "reflect"
+
+func ConvertStructToMap(v any) any {
+	return reflectToMap(reflect.ValueOf(v))
+}
+
+func reflectToMap(fv reflect.Value) any {
+
+	if fv.Kind() == reflect.Ptr {
+
+		if fv.IsNil() {
+			return nil
+		} else {
+			return reflectToMap(fv.Elem())
+		}
+
+	}
+
+	if fv.Kind() == reflect.Func {
+
+		// skip
+		return nil
+
+	}
+
+	if fv.Kind() == reflect.Array {
+
+		arrlen := fv.Len()
+		arr := make([]any, arrlen)
+		for i := 0; i < arrlen; i++ {
+			arr[i] = reflectToMap(fv.Index(i))
+		}
+		return arr
+
+	}
+
+	if fv.Kind() == reflect.Slice {
+
+		arrlen := fv.Len()
+		arr := make([]any, arrlen)
+		for i := 0; i < arrlen; i++ {
+			arr[i] = reflectToMap(fv.Index(i))
+		}
+		return arr
+
+	}
+
+	if fv.Kind() == reflect.Chan {
+
+		// skip
+		return nil
+
+	}
+
+	if fv.Kind() == reflect.Struct {
+
+		res := make(map[string]any)
+
+		for i := 0; i < fv.NumField(); i++ {
+			if fv.Type().Field(i).IsExported() {
+				res[fv.Type().Field(i).Name] = reflectToMap(fv.Field(i))
+			}
+		}
+
+		return res
+
+	}
+
+	return fv.Interface()
+}

rfctime/time.go

@@ -0,0 +1,148 @@
+package rfctime
+
+import (
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Time struct {
+	Hour       int
+	Minute     int
+	Second     int
+	NanoSecond int
+}
+
+func (t Time) Serialize() string {
+	return fmt.Sprintf("%04d:%02d:%02d.%09d", t.Hour, t.Minute, t.Second, t.NanoSecond)
+}
+
+func (t Time) SerializeShort() string {
+	if t.NanoSecond == 0 && t.Second == 0 {
+		return fmt.Sprintf("%02d:%02d", t.Hour, t.Minute)
+	} else if t.NanoSecond == 0 {
+		return fmt.Sprintf("%02d:%02d:%02d", t.Hour, t.Minute, t.Second)
+	} else {
+		return fmt.Sprintf("%02d:%02d:%02d.%09d", t.Hour, t.Minute, t.Second, t.NanoSecond)
+	}
+}
+
+func (t *Time) Deserialize(v string) error {
+
+	var h, m, s, ns string
+
+	split1 := strings.Split(v, ".")
+
+	if len(split1) == 2 {
+
+		split2 := strings.Split(split1[0], ":")
+		if len(split2) == 3 {
+
+			h = split2[0]
+			m = split2[1]
+			s = split2[2]
+			ns = split1[1]
+
+		} else {
+			return fmt.Errorf("invalid time format: %s", v)
+		}
+
+	} else if len(split1) == 1 {
+
+		split2 := strings.Split(split1[0], ":")
+		if len(split2) == 2 {
+
+			h = split2[0]
+			m = split2[1]
+			s = "0"
+			ns = "0"
+
+		} else if len(split2) == 3 {
+
+			h = split2[0]
+			m = split2[1]
+			s = split2[2]
+			ns = "0"
+
+		} else {
+			return fmt.Errorf("invalid time format: %s", v)
+		}
+
+	} else {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	ns = langext.StrPadRight(ns, "0", 9)
+
+	hh, err := strconv.ParseInt(h, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	mm, err := strconv.ParseInt(m, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	ss, err := strconv.ParseInt(s, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	nss, err := strconv.ParseInt(ns, 10, 32)
+	if err != nil {
+		return fmt.Errorf("invalid time format: %s", v)
+	}
+
+	t.Hour = int(hh)
+	t.Minute = int(mm)
+	t.Second = int(ss)
+	t.NanoSecond = int(nss)
+
+	return nil
+}
+
+func (t Time) FormatStr() string {
+	return "15:04:05.999999999"
+}
+
+func (t Time) GoString() string {
+	return fmt.Sprintf("rfctime.NewTime(%d, %d, %d, %d)", t.Hour, t.Minute, t.Second, t.NanoSecond)
+}
+
+func (t Time) String() string {
+	return fmt.Sprintf("%04d:%02d:%02d.%09d", t.Hour, t.Minute, t.Second, t.NanoSecond)
+}
+
+func NewTime(h int, m int, s int, ns int) Time {
+	return Time{
+		Hour:       h,
+		Minute:     m,
+		Second:     s,
+		NanoSecond: ns,
+	}
+}
+
+func NewTimeFromTS(t time.Time) Time {
+	return Time{
+		Hour:       t.Hour(),
+		Minute:     t.Minute(),
+		Second:     t.Second(),
+		NanoSecond: t.Nanosecond(),
+	}
+}
+
+func NowTime(loc *time.Location) Time {
+	now := time.Now().In(loc)
+	return NewTime(now.Hour(), now.Minute(), now.Second(), now.Nanosecond())
+}
+
+func NowTimeLoc() Time {
+	return NowTime(time.UTC)
+}
+
+func NowTimeUTC() Time {
+	return NowTime(time.Local)
+}

sq/builder.go

@@ -1,13 +1,14 @@
 package sq
 
 import (
+	"errors"
 	"fmt"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	"reflect"
 	"strings"
 )
 
-func BuildUpdateStatement(q Queryable, tableName string, obj any, idColumn string) (string, PP, error) {
+func BuildUpdateStatement[TData any](q Queryable, tableName string, obj TData, idColumn string) (string, PP, error) {
 	rval := reflect.ValueOf(obj)
 	rtyp := rval.Type()
 
@@ -69,3 +70,130 @@ func BuildUpdateStatement(q Queryable, tableName string, obj any, idColumn strin
 	//goland:noinspection SqlNoDataSourceInspection
 	return fmt.Sprintf("UPDATE %s SET %s WHERE %s", tableName, strings.Join(setClauses, ", "), matchClause), params, nil
 }
+
+func BuildInsertStatement[TData any](q Queryable, tableName string, obj TData) (string, PP, error) {
+	rval := reflect.ValueOf(obj)
+	rtyp := rval.Type()
+
+	params := PP{}
+
+	fields := make([]string, 0)
+	values := make([]string, 0)
+
+	for i := 0; i < rtyp.NumField(); i++ {
+
+		rsfield := rtyp.Field(i)
+		rvfield := rval.Field(i)
+
+		if !rsfield.IsExported() {
+			continue
+		}
+
+		columnName := rsfield.Tag.Get("db")
+		if columnName == "" || columnName == "-" {
+			continue
+		}
+
+		if rsfield.Type.Kind() == reflect.Ptr && rvfield.IsNil() {
+
+			fields = append(fields, columnName)
+			values = append(values, "NULL")
+
+		} else {
+
+			val, err := convertValueToDB(q, rvfield.Interface())
+			if err != nil {
+				return "", nil, err
+			}
+
+			fields = append(fields, columnName)
+			values = append(values, ":"+params.Add(val))
+
+		}
+	}
+
+	if len(fields) == 0 {
+		return "", nil, exerr.New(exerr.TypeSQLBuild, "no fields found in object").Build()
+	}
+
+	//goland:noinspection SqlNoDataSourceInspection
+	return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, strings.Join(fields, ", "), strings.Join(values, ", ")), params, nil
+}
+
+func BuildInsertMultipleStatement[TData any](q Queryable, tableName string, vArr []TData) (string, PP, error) {
+
+	if len(vArr) == 0 {
+		return "", nil, errors.New("no data supplied")
+	}
+
+	rtyp := reflect.ValueOf(vArr[0]).Type()
+
+	sqlPrefix := ""
+	{
+		columns := make([]string, 0)
+
+		for i := 0; i < rtyp.NumField(); i++ {
+			rsfield := rtyp.Field(i)
+
+			if !rsfield.IsExported() {
+				continue
+			}
+
+			columnName := rsfield.Tag.Get("db")
+			if columnName == "" || columnName == "-" {
+				continue
+			}
+
+			columns = append(columns, "\""+columnName+"\"")
+		}
+
+		sqlPrefix = fmt.Sprintf("INSERT"+" INTO \"%s\" (%s) VALUES", tableName, strings.Join(columns, ", "))
+	}
+
+	pp := PP{}
+
+	sqlValuesArr := make([]string, 0)
+
+	for _, v := range vArr {
+
+		rval := reflect.ValueOf(v)
+
+		params := make([]string, 0)
+
+		for i := 0; i < rtyp.NumField(); i++ {
+
+			rsfield := rtyp.Field(i)
+			rvfield := rval.Field(i)
+
+			if !rsfield.IsExported() {
+				continue
+			}
+
+			columnName := rsfield.Tag.Get("db")
+			if columnName == "" || columnName == "-" {
+				continue
+			}
+
+			if rsfield.Type.Kind() == reflect.Ptr && rvfield.IsNil() {
+
+				params = append(params, "NULL")
+
+			} else {
+
+				val, err := convertValueToDB(q, rvfield.Interface())
+				if err != nil {
+					return "", nil, err
+				}
+
+				params = append(params, ":"+pp.Add(val))
+
+			}
+		}
+
+		sqlValuesArr = append(sqlValuesArr, fmt.Sprintf("(%s)", strings.Join(params, ", ")))
+	}
+
+	sqlstr := fmt.Sprintf("%s %s", sqlPrefix, strings.Join(sqlValuesArr, ", "))
+
+	return sqlstr, pp, nil
+}

sq/builder_test.go

@@ -52,8 +52,7 @@ func TestCreateUpdateStatement(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)

sq/commentTrimmer.go

@@ -0,0 +1,32 @@
+package sq
+
+import (
+	"context"
+	"strings"
+)
+
+var CommentTrimmer = NewPreListener(fnTrimComments)
+
+func fnTrimComments(ctx context.Context, cmdtype string, id *uint16, sql *string, params *PP) error {
+
+	res := make([]string, 0)
+
+	for _, s := range strings.Split(*sql, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(s), "--") {
+			continue
+		}
+
+		idx := strings.Index(s, "--")
+		if idx != -1 {
+			s = s[:idx]
+		}
+
+		s = strings.TrimRight(s, " \t\r\n")
+
+		res = append(res, s)
+	}
+
+	*sql = strings.Join(res, "\n")
+
+	return nil
+}

sq/converter.go

@@ -1,14 +1,10 @@
 package sq
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
-	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
-	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
 	"reflect"
-	"time"
 )
 
 type DBTypeConverter interface {
@@ -18,129 +14,16 @@ type DBTypeConverter interface {
 	DBToModel(v any) (any, error)
 }
 
-var ConverterBoolToBit = NewDBTypeConverter[bool, int](func(v bool) (int, error) {
-	return langext.Conditional(v, 1, 0), nil
-}, func(v int) (bool, error) {
-	if v == 0 {
-		return false, nil
-	}
-	if v == 1 {
-		return true, nil
-	}
-	return false, errors.New(fmt.Sprintf("invalid valud for boolean: '%d'", v))
-})
+type DBDataConstraint interface {
+	string | langext.NumberConstraint | []byte
+}
 
-var ConverterTimeToUnixMillis = NewDBTypeConverter[time.Time, int64](func(v time.Time) (int64, error) {
-	return v.UnixMilli(), nil
-}, func(v int64) (time.Time, error) {
-	return time.UnixMilli(v), nil
-})
+type DatabaseConvertible[TModelData any, TDBData DBDataConstraint] interface {
+	MarshalToDB(v TModelData) (TDBData, error)
+	UnmarshalToModel(v TDBData) (TModelData, error)
+}
 
-var ConverterRFCUnixMilliTimeToUnixMillis = NewDBTypeConverter[rfctime.UnixMilliTime, int64](func(v rfctime.UnixMilliTime) (int64, error) {
-	return v.UnixMilli(), nil
-}, func(v int64) (rfctime.UnixMilliTime, error) {
-	return rfctime.NewUnixMilli(time.UnixMilli(v)), nil
-})
-
-var ConverterRFCUnixNanoTimeToUnixNanos = NewDBTypeConverter[rfctime.UnixNanoTime, int64](func(v rfctime.UnixNanoTime) (int64, error) {
-	return v.UnixNano(), nil
-}, func(v int64) (rfctime.UnixNanoTime, error) {
-	return rfctime.NewUnixNano(time.Unix(0, v)), nil
-})
-
-var ConverterRFCUnixTimeToUnixSeconds = NewDBTypeConverter[rfctime.UnixTime, int64](func(v rfctime.UnixTime) (int64, error) {
-	return v.Unix(), nil
-}, func(v int64) (rfctime.UnixTime, error) {
-	return rfctime.NewUnix(time.Unix(v, 0)), nil
-})
-
-// ConverterRFC339TimeToString
-// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
-var ConverterRFC339TimeToString = NewDBTypeConverter[rfctime.RFC3339Time, string](func(v rfctime.RFC3339Time) (string, error) {
-	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05"), nil
-}, func(v string) (rfctime.RFC3339Time, error) {
-	t, err := time.Parse("2006-01-02 15:04:05", v)
-	if err != nil {
-		return rfctime.RFC3339Time{}, err
-	}
-	return rfctime.NewRFC3339(t), nil
-})
-
-// ConverterRFC339NanoTimeToString
-// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
-var ConverterRFC339NanoTimeToString = NewDBTypeConverter[rfctime.RFC3339NanoTime, string](func(v rfctime.RFC3339NanoTime) (string, error) {
-	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05.999999999"), nil
-}, func(v string) (rfctime.RFC3339NanoTime, error) {
-	t, err := time.ParseInLocation("2006-01-02 15:04:05.999999999", v, time.UTC)
-	if err != nil {
-		return rfctime.RFC3339NanoTime{}, err
-	}
-	return rfctime.NewRFC3339Nano(t), nil
-})
-
-var ConverterJsonObjToString = NewDBTypeConverter[JsonObj, string](func(v JsonObj) (string, error) {
-	mrsh, err := json.Marshal(v)
-	if err != nil {
-		return "", err
-	}
-	return string(mrsh), nil
-}, func(v string) (JsonObj, error) {
-	var mrsh JsonObj
-	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
-		return JsonObj{}, err
-	}
-	return mrsh, nil
-})
-
-var ConverterJsonArrToString = NewDBTypeConverter[JsonArr, string](func(v JsonArr) (string, error) {
-	mrsh, err := json.Marshal(v)
-	if err != nil {
-		return "", err
-	}
-	return string(mrsh), nil
-}, func(v string) (JsonArr, error) {
-	var mrsh JsonArr
-	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
-		return JsonArr{}, err
-	}
-	return mrsh, nil
-})
-
-var ConverterExErrCategoryToString = NewDBTypeConverter[exerr.ErrorCategory, string](func(v exerr.ErrorCategory) (string, error) {
-	return v.Category, nil
-}, func(v string) (exerr.ErrorCategory, error) {
-	for _, cat := range exerr.AllCategories {
-		if cat.Category == v {
-			return cat, nil
-		}
-	}
-	return exerr.CatUser, errors.New("failed to convert '" + v + "' to exerr.ErrorCategory")
-})
-
-var ConverterExErrSeverityToString = NewDBTypeConverter[exerr.ErrorSeverity, string](func(v exerr.ErrorSeverity) (string, error) {
-	return v.Severity, nil
-}, func(v string) (exerr.ErrorSeverity, error) {
-	for _, sev := range exerr.AllSeverities {
-		if sev.Severity == v {
-			return sev, nil
-		}
-	}
-	return exerr.SevErr, errors.New("failed to convert '" + v + "' to exerr.ErrorSeverity")
-})
-
-var ConverterExErrTypeToString = NewDBTypeConverter[exerr.ErrorType, string](func(v exerr.ErrorType) (string, error) {
-	return v.Key, nil
-}, func(v string) (exerr.ErrorType, error) {
-	for _, etp := range exerr.ListRegisteredTypes() {
-		if etp.Key == v {
-			return etp, nil
-		}
-	}
-
-	return exerr.NewType(v, nil), nil
-})
-
-type dbTypeConverterImpl[TModelData any, TDBData any] struct {
+type dbTypeConverterImpl[TModelData any, TDBData DBDataConstraint] struct {
 	dbTypeString    string
 	modelTypeString string
 	todb            func(v TModelData) (TDBData, error)
@@ -169,7 +52,7 @@ func (t *dbTypeConverterImpl[TModelData, TDBData]) DBToModel(v any) (any, error)
 	return nil, errors.New(fmt.Sprintf("Unexpected value in DBTypeConverter, expected '%s', found '%T'", t.dbTypeString, v))
 }
 
-func NewDBTypeConverter[TModelData any, TDBData any](todb func(v TModelData) (TDBData, error), tomodel func(v TDBData) (TModelData, error)) DBTypeConverter {
+func NewDBTypeConverter[TModelData any, TDBData DBDataConstraint](todb func(v TModelData) (TDBData, error), tomodel func(v TDBData) (TModelData, error)) DBTypeConverter {
 	return &dbTypeConverterImpl[TModelData, TDBData]{
 		dbTypeString:    fmt.Sprintf("%T", *new(TDBData)),
 		modelTypeString: fmt.Sprintf("%T", *new(TModelData)),
@@ -178,6 +61,15 @@ func NewDBTypeConverter[TModelData any, TDBData any](todb func(v TModelData) (TD
 	}
 }
 
+func NewAutoDBTypeConverter[TDBData DBDataConstraint, TModelData DatabaseConvertible[TModelData, TDBData]](obj TModelData) DBTypeConverter {
+	return &dbTypeConverterImpl[TModelData, TDBData]{
+		dbTypeString:    fmt.Sprintf("%T", *new(TDBData)),
+		modelTypeString: fmt.Sprintf("%T", *new(TModelData)),
+		todb:            obj.MarshalToDB,
+		tomodel:         obj.UnmarshalToModel,
+	}
+}
+
 func convertValueToDB(q Queryable, value any) (any, error) {
 	modelTypeStr := fmt.Sprintf("%T", value)
 

sq/converterDefault.go

@@ -0,0 +1,161 @@
+package sq
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
+	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ========================== COMMON DATATYPES ==========================
+
+var ConverterBoolToBit = NewDBTypeConverter[bool, int64](func(v bool) (int64, error) {
+	return langext.Conditional(v, int64(1), int64(0)), nil
+}, func(v int64) (bool, error) {
+	if v == 0 {
+		return false, nil
+	}
+	if v == 1 {
+		return true, nil
+	}
+	return false, errors.New(fmt.Sprintf("invalid valud for boolean: '%d'", v))
+})
+
+var ConverterTimeToUnixMillis = NewDBTypeConverter[time.Time, int64](func(v time.Time) (int64, error) {
+	return v.UnixMilli(), nil
+}, func(v int64) (time.Time, error) {
+	return time.UnixMilli(v), nil
+})
+
+// ========================== RFCTIME ==========================
+
+var ConverterRFCUnixMilliTimeToUnixMillis = NewDBTypeConverter[rfctime.UnixMilliTime, int64](func(v rfctime.UnixMilliTime) (int64, error) {
+	return v.UnixMilli(), nil
+}, func(v int64) (rfctime.UnixMilliTime, error) {
+	return rfctime.NewUnixMilli(time.UnixMilli(v)), nil
+})
+
+var ConverterRFCUnixNanoTimeToUnixNanos = NewDBTypeConverter[rfctime.UnixNanoTime, int64](func(v rfctime.UnixNanoTime) (int64, error) {
+	return v.UnixNano(), nil
+}, func(v int64) (rfctime.UnixNanoTime, error) {
+	return rfctime.NewUnixNano(time.Unix(0, v)), nil
+})
+
+var ConverterRFCUnixTimeToUnixSeconds = NewDBTypeConverter[rfctime.UnixTime, int64](func(v rfctime.UnixTime) (int64, error) {
+	return v.Unix(), nil
+}, func(v int64) (rfctime.UnixTime, error) {
+	return rfctime.NewUnix(time.Unix(v, 0)), nil
+})
+
+// ConverterRFC339TimeToString
+// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
+var ConverterRFC339TimeToString = NewDBTypeConverter[rfctime.RFC3339Time, string](func(v rfctime.RFC3339Time) (string, error) {
+	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05"), nil
+}, func(v string) (rfctime.RFC3339Time, error) {
+	t, err := time.Parse("2006-01-02 15:04:05", v)
+	if err != nil {
+		return rfctime.RFC3339Time{}, err
+	}
+	return rfctime.NewRFC3339(t), nil
+})
+
+// ConverterRFC339NanoTimeToString
+// Does not really use RFC339 - but sqlite does not understand timezones and the `T` delimiter
+var ConverterRFC339NanoTimeToString = NewDBTypeConverter[rfctime.RFC3339NanoTime, string](func(v rfctime.RFC3339NanoTime) (string, error) {
+	return v.Time().In(time.UTC).Format("2006-01-02 15:04:05.999999999"), nil
+}, func(v string) (rfctime.RFC3339NanoTime, error) {
+	t, err := time.ParseInLocation("2006-01-02 15:04:05.999999999", v, time.UTC)
+	if err != nil {
+		return rfctime.RFC3339NanoTime{}, err
+	}
+	return rfctime.NewRFC3339Nano(t), nil
+})
+
+var ConverterRFCDateToString = NewDBTypeConverter[rfctime.Date, string](func(v rfctime.Date) (string, error) {
+	return fmt.Sprintf("%04d-%02d-%02d", v.Year, v.Month, v.Day), nil
+}, func(v string) (rfctime.Date, error) {
+	split := strings.Split(v, "-")
+	if len(split) != 3 {
+		return rfctime.Date{}, errors.New("invalid date format: " + v)
+	}
+	year, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	month, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	day, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+
+	return rfctime.Date{Year: int(year), Month: int(month), Day: int(day)}, nil
+})
+
+var ConverterRFCTimeToString = NewDBTypeConverter[rfctime.Time, string](func(v rfctime.Time) (string, error) {
+	return v.SerializeShort(), nil
+}, func(v string) (rfctime.Time, error) {
+	res := rfctime.Time{}
+	err := res.Deserialize(v)
+	if err != nil {
+		return rfctime.Time{}, err
+	}
+	return res, nil
+})
+
+var ConverterRFCSecondsF64ToString = NewDBTypeConverter[rfctime.SecondsF64, float64](func(v rfctime.SecondsF64) (float64, error) {
+	return v.Seconds(), nil
+}, func(v float64) (rfctime.SecondsF64, error) {
+	return rfctime.NewSecondsF64(timeext.FromSeconds(v)), nil
+})
+
+// ========================== JSON ==========================
+
+var ConverterJsonObjToString = NewAutoDBTypeConverter(JsonObj{})
+
+var ConverterJsonArrToString = NewAutoDBTypeConverter(JsonArr{})
+
+// Json[T] must be registered manually for each gen-type
+
+// ========================== EXERR ==========================
+
+var ConverterExErrCategoryToString = NewDBTypeConverter[exerr.ErrorCategory, string](func(v exerr.ErrorCategory) (string, error) {
+	return v.Category, nil
+}, func(v string) (exerr.ErrorCategory, error) {
+	for _, cat := range exerr.AllCategories {
+		if cat.Category == v {
+			return cat, nil
+		}
+	}
+	return exerr.CatUser, errors.New("failed to convert '" + v + "' to exerr.ErrorCategory")
+})
+
+var ConverterExErrSeverityToString = NewDBTypeConverter[exerr.ErrorSeverity, string](func(v exerr.ErrorSeverity) (string, error) {
+	return v.Severity, nil
+}, func(v string) (exerr.ErrorSeverity, error) {
+	for _, sev := range exerr.AllSeverities {
+		if sev.Severity == v {
+			return sev, nil
+		}
+	}
+	return exerr.SevErr, errors.New("failed to convert '" + v + "' to exerr.ErrorSeverity")
+})
+
+var ConverterExErrTypeToString = NewDBTypeConverter[exerr.ErrorType, string](func(v exerr.ErrorType) (string, error) {
+	return v.Key, nil
+}, func(v string) (exerr.ErrorType, error) {
+	for _, etp := range exerr.ListRegisteredTypes() {
+		if etp.Key == v {
+			return etp, nil
+		}
+	}
+
+	return exerr.NewType(v, nil), nil
+})

sq/database.go

@@ -17,7 +17,11 @@ type DB interface {
 	AddListener(listener Listener)
 	Exit() error
 	RegisterConverter(DBTypeConverter)
-	RegisterDefaultConverter()
+}
+
+type DBOptions struct {
+	RegisterDefaultConverter *bool
+	RegisterCommentTrimmer   *bool
 }
 
 type database struct {
@@ -28,13 +32,23 @@ type database struct {
 	conv  []DBTypeConverter
 }
 
-func NewDB(db *sqlx.DB) DB {
-	return &database{
+func NewDB(db *sqlx.DB, opt DBOptions) DB {
+	sqdb := &database{
 		db:    db,
 		txctr: 0,
 		lock:  sync.Mutex{},
 		lstr:  make([]Listener, 0),
 	}
+
+	if langext.Coalesce(opt.RegisterDefaultConverter, true) {
+		sqdb.registerDefaultConverter()
+	}
+
+	if langext.Coalesce(opt.RegisterCommentTrimmer, true) {
+		sqdb.AddListener(CommentTrimmer)
+	}
+
+	return sqdb
 }
 
 func (db *database) AddListener(listener Listener) {
@@ -46,7 +60,7 @@ func (db *database) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Resul
 	for _, v := range db.lstr {
 		err := v.PreExec(ctx, nil, &sqlstr, &prep)
 		if err != nil {
-			return nil, err
+			return nil, exerr.Wrap(err, "failed to call SQL pre-exec listener").Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 		}
 	}
 
@@ -67,7 +81,7 @@ func (db *database) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx.Ro
 	for _, v := range db.lstr {
 		err := v.PreQuery(ctx, nil, &sqlstr, &prep)
 		if err != nil {
-			return nil, err
+			return nil, exerr.Wrap(err, "failed to call SQL pre-query listener").Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 		}
 	}
 
@@ -141,16 +155,23 @@ func (db *database) RegisterConverter(conv DBTypeConverter) {
 	db.conv = append(db.conv, conv)
 }
 
-func (db *database) RegisterDefaultConverter() {
+func (db *database) registerDefaultConverter() {
 	db.RegisterConverter(ConverterBoolToBit)
+
 	db.RegisterConverter(ConverterTimeToUnixMillis)
+
 	db.RegisterConverter(ConverterRFCUnixMilliTimeToUnixMillis)
 	db.RegisterConverter(ConverterRFCUnixNanoTimeToUnixNanos)
 	db.RegisterConverter(ConverterRFCUnixTimeToUnixSeconds)
 	db.RegisterConverter(ConverterRFC339TimeToString)
 	db.RegisterConverter(ConverterRFC339NanoTimeToString)
+	db.RegisterConverter(ConverterRFCDateToString)
+	db.RegisterConverter(ConverterRFCTimeToString)
+	db.RegisterConverter(ConverterRFCSecondsF64ToString)
+
 	db.RegisterConverter(ConverterJsonObjToString)
 	db.RegisterConverter(ConverterJsonArrToString)
+
 	db.RegisterConverter(ConverterExErrCategoryToString)
 	db.RegisterConverter(ConverterExErrSeverityToString)
 	db.RegisterConverter(ConverterExErrTypeToString)

sq/filter.go

@@ -0,0 +1,49 @@
+package sq
+
+import ct "gogs.mikescher.com/BlackForestBytes/goext/cursortoken"
+
+type FilterSort struct {
+	Field     string
+	Direction ct.SortDirection
+}
+
+type PaginateFilter interface {
+	SQL(params PP) (filterClause string, joinClause string, joinTables []string)
+	Sort() []FilterSort
+}
+
+type genericPaginateFilter struct {
+	sql  func(params PP) (filterClause string, joinClause string, joinTables []string)
+	sort func() []FilterSort
+}
+
+func (g genericPaginateFilter) SQL(params PP) (filterClause string, joinClause string, joinTables []string) {
+	return g.sql(params)
+}
+
+func (g genericPaginateFilter) Sort() []FilterSort {
+	return g.sort()
+}
+
+func NewPaginateFilter(sql func(params PP) (filterClause string, joinClause string, joinTables []string), sort []FilterSort) PaginateFilter {
+	return genericPaginateFilter{
+		sql: func(params PP) (filterClause string, joinClause string, joinTables []string) {
+			return sql(params)
+		},
+		sort: func() []FilterSort {
+			return sort
+		},
+	}
+}
+
+func NewSimplePaginateFilter(filterClause string, filterParams PP, sort []FilterSort) PaginateFilter {
+	return genericPaginateFilter{
+		sql: func(params PP) (string, string, []string) {
+			params.AddAll(filterParams)
+			return filterClause, "", nil
+		},
+		sort: func() []FilterSort {
+			return sort
+		},
+	}
+}

sq/hasher.go

@@ -31,7 +31,7 @@ func HashMattnSqliteSchema(ctx context.Context, schemaStr string) (string, error
 		return "", err
 	}
 
-	db := NewDB(xdb)
+	db := NewDB(xdb, DBOptions{})
 
 	_, err = db.Exec(ctx, schemaStr, PP{})
 	if err != nil {
@@ -59,7 +59,7 @@ func HashGoSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
 		return "", err
 	}
 
-	db := NewDB(xdb)
+	db := NewDB(xdb, DBOptions{})
 
 	_, err = db.Exec(ctx, schemaStr, PP{})
 	if err != nil {

sq/json.go

@@ -1,5 +1,59 @@
 package sq
 
+import "encoding/json"
+
 type JsonObj map[string]any
 
+func (j JsonObj) MarshalToDB(v JsonObj) (string, error) {
+	mrsh, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}
+
+func (j JsonObj) UnmarshalToModel(v string) (JsonObj, error) {
+	var mrsh JsonObj
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return JsonObj{}, err
+	}
+	return mrsh, nil
+}
+
 type JsonArr []any
+
+func (j JsonArr) MarshalToDB(v JsonArr) (string, error) {
+	mrsh, err := json.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}
+
+func (j JsonArr) UnmarshalToModel(v string) (JsonArr, error) {
+	var mrsh JsonArr
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return JsonArr{}, err
+	}
+	return mrsh, nil
+}
+
+type AutoJson[T any] struct {
+	Value T
+}
+
+func (j AutoJson[T]) MarshalToDB(v AutoJson[T]) (string, error) {
+	mrsh, err := json.Marshal(v.Value)
+	if err != nil {
+		return "", err
+	}
+	return string(mrsh), nil
+}
+
+func (j AutoJson[T]) UnmarshalToModel(v string) (AutoJson[T], error) {
+	mrsh := *new(T)
+	if err := json.Unmarshal([]byte(v), &mrsh); err != nil {
+		return AutoJson[T]{}, err
+	}
+	return AutoJson[T]{Value: mrsh}, nil
+}

sq/listener.go

@@ -17,3 +17,172 @@ type Listener interface {
 	PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 	PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 }
+
+type genListener struct {
+	prePing        func(ctx context.Context) error
+	preTxBegin     func(ctx context.Context, txid uint16) error
+	preTxCommit    func(txid uint16) error
+	preTxRollback  func(txid uint16) error
+	preQuery       func(ctx context.Context, txID *uint16, sql *string, params *PP) error
+	preExec        func(ctx context.Context, txID *uint16, sql *string, params *PP) error
+	postPing       func(result error)
+	postTxBegin    func(txid uint16, result error)
+	postTxCommit   func(txid uint16, result error)
+	postTxRollback func(txid uint16, result error)
+	postQuery      func(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+	postExec       func(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+}
+
+func (g genListener) PrePing(ctx context.Context) error {
+	if g.prePing != nil {
+		return g.prePing(ctx)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxBegin(ctx context.Context, txid uint16) error {
+	if g.preTxBegin != nil {
+		return g.preTxBegin(ctx, txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxCommit(txid uint16) error {
+	if g.preTxCommit != nil {
+		return g.preTxCommit(txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxRollback(txid uint16) error {
+	if g.preTxRollback != nil {
+		return g.preTxRollback(txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreQuery(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+	if g.preQuery != nil {
+		return g.preQuery(ctx, txID, sql, params)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreExec(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+	if g.preExec != nil {
+		return g.preExec(ctx, txID, sql, params)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PostPing(result error) {
+	if g.postPing != nil {
+		g.postPing(result)
+	}
+}
+
+func (g genListener) PostTxBegin(txid uint16, result error) {
+	if g.postTxBegin != nil {
+		g.postTxBegin(txid, result)
+	}
+}
+
+func (g genListener) PostTxCommit(txid uint16, result error) {
+	if g.postTxCommit != nil {
+		g.postTxCommit(txid, result)
+	}
+}
+
+func (g genListener) PostTxRollback(txid uint16, result error) {
+	if g.postTxRollback != nil {
+		g.postTxRollback(txid, result)
+	}
+}
+
+func (g genListener) PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+	if g.postQuery != nil {
+		g.postQuery(txID, sqlOriginal, sqlReal, params)
+	}
+}
+
+func (g genListener) PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+	if g.postExec != nil {
+		g.postExec(txID, sqlOriginal, sqlReal, params)
+	}
+}
+
+func NewPrePingListener(f func(ctx context.Context) error) Listener {
+	return genListener{prePing: f}
+}
+
+func NewPreTxBeginListener(f func(ctx context.Context, txid uint16) error) Listener {
+	return genListener{preTxBegin: f}
+}
+
+func NewPreTxCommitListener(f func(txid uint16) error) Listener {
+	return genListener{preTxCommit: f}
+}
+
+func NewPreTxRollbackListener(f func(txid uint16) error) Listener {
+	return genListener{preTxRollback: f}
+}
+
+func NewPreQueryListener(f func(ctx context.Context, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{preQuery: f}
+}
+
+func NewPreExecListener(f func(ctx context.Context, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{preExec: f}
+}
+
+func NewPreListener(f func(ctx context.Context, cmdtype string, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{
+		preExec: func(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+			return f(ctx, "EXEC", txID, sql, params)
+		},
+		preQuery: func(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+			return f(ctx, "QUERY", txID, sql, params)
+		},
+	}
+}
+
+func NewPostPingListener(f func(result error)) Listener {
+	return genListener{postPing: f}
+}
+
+func NewPostTxBeginListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxBegin: f}
+}
+
+func NewPostTxCommitListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxCommit: f}
+}
+
+func NewPostTxRollbackListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxRollback: f}
+}
+
+func NewPostQueryListener(f func(txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{postQuery: f}
+}
+
+func NewPostExecListener(f func(txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{postExec: f}
+}
+
+func NewPostListener(f func(cmdtype string, txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{
+		postExec: func(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+			f("EXEC", txID, sqlOriginal, sqlReal, params)
+		},
+		postQuery: func(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+			f("QUERY", txID, sqlOriginal, sqlReal, params)
+		},
+	}
+}

sq/main_test.go

@@ -0,0 +1,15 @@
+package sq
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	if !exerr.Initialized() {
+		exerr.Init(exerr.ErrorPackageConfigInit{ZeroLogErrTraces: langext.PFalse, ZeroLogAllTraces: langext.PFalse})
+	}
+	os.Exit(m.Run())
+}

sq/paginate.go

@@ -0,0 +1,115 @@
+package sq
+
+import (
+	"context"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	pag "gogs.mikescher.com/BlackForestBytes/goext/pagination"
+)
+
+func Paginate[TData any](ctx context.Context, q Queryable, table string, filter PaginateFilter, scanMode StructScanMode, scanSec StructScanSafety, page int, limit *int) ([]TData, pag.Pagination, error) {
+	prepParams := PP{}
+
+	sortOrder := filter.Sort()
+	sortCond := ""
+	if len(sortOrder) > 0 {
+		sortCond = "ORDER BY "
+		for i, v := range sortOrder {
+			if i > 0 {
+				sortCond += ", "
+			}
+			sortCond += v.Field + " " + string(v.Direction)
+		}
+	}
+
+	pageCond := ""
+	if limit != nil {
+		pageCond += fmt.Sprintf("LIMIT :%s OFFSET :%s", prepParams.Add(*limit+1), prepParams.Add(*limit*(page-1)))
+	}
+
+	filterCond, joinCond, joinTables := filter.SQL(prepParams)
+
+	selectCond := table + ".*"
+	for _, v := range joinTables {
+		selectCond += ", " + v + ".*"
+	}
+
+	sqlQueryData := "SELECT " + selectCond + " FROM " + table + " " + joinCond + " WHERE ( " + filterCond + " ) " + sortCond + " " + pageCond
+	sqlQueryCount := "SELECT " + "COUNT(*)" + " FROM " + table + " " + joinCond + " WHERE ( " + filterCond + " ) "
+
+	rows, err := q.Query(ctx, sqlQueryData, prepParams)
+	if err != nil {
+		return nil, pag.Pagination{}, exerr.Wrap(err, "failed to list paginated entries from DB").Str("table", table).Any("filter", filter).Int("page", page).Any("limit", limit).Build()
+	}
+
+	entities, err := ScanAll[TData](ctx, q, rows, scanMode, scanSec, true)
+	if err != nil {
+		return nil, pag.Pagination{}, exerr.Wrap(err, "failed to decode paginated entries from DB").Str("table", table).Int("page", page).Any("limit", limit).Str("scanMode", string(scanMode)).Str("scanSec", string(scanSec)).Build()
+	}
+
+	if page == 1 && (limit == nil || len(entities) <= *limit) {
+		return entities, pag.Pagination{
+			Page:             1,
+			Limit:            langext.Coalesce(limit, len(entities)),
+			TotalPages:       1,
+			TotalItems:       len(entities),
+			CurrentPageCount: 1,
+		}, nil
+	} else {
+
+		countRows, err := q.Query(ctx, sqlQueryCount, prepParams)
+		if err != nil {
+			return nil, pag.Pagination{}, exerr.Wrap(err, "failed to query total-count of paginated entries from DB").Str("table", table).Build()
+		}
+
+		if !countRows.Next() {
+			return nil, pag.Pagination{}, exerr.New(exerr.TypeSQLDecode, "SQL COUNT(*) query returned no rows").Str("table", table).Any("filter", filter).Build()
+		}
+
+		var countRes int
+		err = countRows.Scan(&countRes)
+		if err != nil {
+			return nil, pag.Pagination{}, exerr.Wrap(err, "failed to decode total-count of paginated entries from DB").Str("table", table).Build()
+		}
+
+		if len(entities) > *limit {
+			entities = entities[:*limit]
+		}
+
+		paginationObj := pag.Pagination{
+			Page:             page,
+			Limit:            langext.Coalesce(limit, countRes),
+			TotalPages:       pag.CalcPaginationTotalPages(countRes, langext.Coalesce(limit, countRes)),
+			TotalItems:       countRes,
+			CurrentPageCount: len(entities),
+		}
+
+		return entities, paginationObj, nil
+	}
+}
+
+func Count(ctx context.Context, q Queryable, table string, filter PaginateFilter) (int, error) {
+	prepParams := PP{}
+
+	filterCond, joinCond, _ := filter.SQL(prepParams)
+
+	sqlQueryCount := "SELECT " + "COUNT(*)" + " FROM " + table + " " + joinCond + " WHERE ( " + filterCond + " )"
+
+	countRows, err := q.Query(ctx, sqlQueryCount, prepParams)
+	if err != nil {
+		return 0, exerr.Wrap(err, "failed to query count of entries from DB").Str("table", table).Build()
+	}
+
+	if !countRows.Next() {
+		return 0, exerr.New(exerr.TypeSQLDecode, "SQL COUNT(*) query returned no rows").Str("table", table).Any("filter", filter).Build()
+	}
+
+	var countRes int
+	err = countRows.Scan(&countRes)
+	if err != nil {
+		return 0, exerr.Wrap(err, "failed to decode count of entries from DB").Str("table", table).Build()
+	}
+
+	return countRes, nil
+}

sq/params.go

@@ -20,6 +20,12 @@ func (pp *PP) Add(v any) string {
 	return id
 }
 
+func (pp *PP) AddAll(other PP) {
+	for id, v := range other {
+		(*pp)[id] = v
+	}
+}
+
 func PPID() string {
 	return "p_" + langext.RandBase62(8)
 }

sq/scanner.go

@@ -6,8 +6,9 @@ import (
 	"errors"
 	"fmt"
 	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"reflect"
-	"strings"
 )
 
 type StructScanMode string
@@ -26,43 +27,11 @@ const (
 
 func InsertSingle[TData any](ctx context.Context, q Queryable, tableName string, v TData) (sql.Result, error) {
 
-	rval := reflect.ValueOf(v)
-	rtyp := rval.Type()
-
-	columns := make([]string, 0)
-	params := make([]string, 0)
-	pp := PP{}
-
-	for i := 0; i < rtyp.NumField(); i++ {
-
-		rsfield := rtyp.Field(i)
-		rvfield := rval.Field(i)
-
-		if !rsfield.IsExported() {
-			continue
-		}
-
-		columnName := rsfield.Tag.Get("db")
-		if columnName == "" || columnName == "-" {
-			continue
-		}
-
-		paramkey := fmt.Sprintf("_%s", columnName)
-
-		columns = append(columns, "\""+columnName+"\"")
-		params = append(params, ":"+paramkey)
-
-		val, err := convertValueToDB(q, rvfield.Interface())
-		if err != nil {
-			return nil, err
-		}
-
-		pp[paramkey] = val
-
+	sqlstr, pp, err := BuildInsertStatement(q, tableName, v)
+	if err != nil {
+		return nil, err
 	}
 
-	sqlstr := fmt.Sprintf("INSERT"+" INTO \"%s\" (%s) VALUES (%s)", tableName, strings.Join(columns, ", "), strings.Join(params, ", "))
-
 	sqlr, err := q.Exec(ctx, sqlstr, pp)
 	if err != nil {
 		return nil, err
@@ -71,6 +40,127 @@ func InsertSingle[TData any](ctx context.Context, q Queryable, tableName string,
 	return sqlr, nil
 }
 
+func InsertAndQuerySingle[TData any](ctx context.Context, q Queryable, tableName string, v TData, idColumn string, mode StructScanMode, sec StructScanSafety) (TData, error) {
+
+	rval := reflect.ValueOf(v)
+
+	idRVal := fieldByTag(rval, "db", idColumn)
+	if !idRVal.IsValid() || idRVal.IsZero() {
+		return *new(TData), fmt.Errorf("failed to find idColumn '%s' in %T", idColumn, v)
+	}
+
+	idValue, err := convertValueToDB(q, idRVal.Interface())
+	if err != nil {
+		return *new(TData), err
+	}
+
+	_, err = InsertSingle[TData](ctx, q, tableName, v)
+	if err != nil {
+		return *new(TData), err
+	}
+
+	pp := PP{}
+
+	//goland:noinspection ALL
+	sqlstr := fmt.Sprintf("SELECT * FROM %s WHERE %s = :%s", tableName, idColumn, pp.Add(idValue))
+
+	return QuerySingle[TData](ctx, q, sqlstr, pp, mode, sec)
+}
+
+func fieldByTag(rval reflect.Value, tagkey string, tagval string) reflect.Value {
+	rtyp := rval.Type()
+	for i := 0; i < rtyp.NumField(); i++ {
+		rsfield := rtyp.Field(i)
+
+		if !rsfield.IsExported() {
+			continue
+		}
+
+		if rsfield.Tag.Get(tagkey) == tagval {
+			return rval.Field(i)
+		}
+	}
+	panic(fmt.Sprintf("tag %s = '%s' not found in %s", tagkey, tagval, rtyp.Name()))
+}
+
+func InsertMultiple[TData any](ctx context.Context, q Queryable, tableName string, vArr []TData, maxBatch int) ([]sql.Result, error) {
+
+	if len(vArr) == 0 {
+		return make([]sql.Result, 0), nil
+	}
+
+	chunks := langext.ArrChunk(vArr, maxBatch)
+
+	sqlstrArr := make([]string, 0)
+	ppArr := make([]PP, 0)
+
+	for _, chunk := range chunks {
+
+		sqlstr, pp, err := BuildInsertMultipleStatement(q, tableName, chunk)
+		if err != nil {
+			return nil, exerr.Wrap(err, "").Build()
+		}
+
+		sqlstrArr = append(sqlstrArr, sqlstr)
+		ppArr = append(ppArr, pp)
+	}
+
+	res := make([]sql.Result, 0, len(sqlstrArr))
+
+	for i := 0; i < len(sqlstrArr); i++ {
+		sqlr, err := q.Exec(ctx, sqlstrArr[i], ppArr[i])
+		if err != nil {
+			return nil, err
+		}
+
+		res = append(res, sqlr)
+	}
+
+	return res, nil
+}
+
+func UpdateSingle[TData any](ctx context.Context, q Queryable, tableName string, v TData, idColumn string) (sql.Result, error) {
+
+	sqlstr, pp, err := BuildUpdateStatement(q, tableName, v, idColumn)
+	if err != nil {
+		return nil, err
+	}
+
+	sqlr, err := q.Exec(ctx, sqlstr, pp)
+	if err != nil {
+		return nil, err
+	}
+
+	return sqlr, nil
+}
+
+func UpdateAndQuerySingle[TData any](ctx context.Context, q Queryable, tableName string, v TData, idColumn string, mode StructScanMode, sec StructScanSafety) (TData, error) {
+
+	rval := reflect.ValueOf(v)
+
+	idRVal := fieldByTag(rval, "db", idColumn)
+	if !idRVal.IsValid() || idRVal.IsZero() {
+		return *new(TData), fmt.Errorf("failed to find idColumn '%s' in %T", idColumn, v)
+	}
+
+	idValue, err := convertValueToDB(q, idRVal.Interface())
+	if err != nil {
+		return *new(TData), err
+	}
+
+	_, err = UpdateSingle[TData](ctx, q, tableName, v, idColumn)
+	if err != nil {
+		return *new(TData), err
+	}
+
+	pp := PP{}
+
+	//goland:noinspection ALL
+	sqlstr := fmt.Sprintf("SELECT * FROM %s WHERE %s = :%s", tableName, idColumn, pp.Add(idValue))
+
+	return QuerySingle[TData](ctx, q, sqlstr, pp, mode, sec)
+}
+
 func QuerySingle[TData any](ctx context.Context, q Queryable, sql string, pp PP, mode StructScanMode, sec StructScanSafety) (TData, error) {
 	rows, err := q.Query(ctx, sql, pp)
 	if err != nil {
@@ -85,6 +175,23 @@ func QuerySingle[TData any](ctx context.Context, q Queryable, sql string, pp PP,
 	return data, nil
 }
 
+func QuerySingleOpt[TData any](ctx context.Context, q Queryable, sqlstr string, pp PP, mode StructScanMode, sec StructScanSafety) (*TData, error) {
+	rows, err := q.Query(ctx, sqlstr, pp)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ScanSingle[TData](ctx, q, rows, mode, sec, true)
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &data, nil
+}
+
 func QueryAll[TData any](ctx context.Context, q Queryable, sql string, pp PP, mode StructScanMode, sec StructScanSafety) ([]TData, error) {
 	rows, err := q.Query(ctx, sql, pp)
 	if err != nil {

sq/scanner_test.go

@@ -0,0 +1,234 @@
+package sq
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/glebarez/go-sqlite"
+	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/tst"
+	"path/filepath"
+	"testing"
+)
+
+func TestInsertSingle(t *testing.T) {
+
+	type request struct {
+		ID        string  `json:"id"        db:"id"`
+		Timestamp int     `json:"timestamp" db:"timestamp"`
+		StrVal    string  `json:"strVal"    db:"str_val"`
+		FloatVal  float64 `json:"floatVal"    db:"float_val"`
+		Dummy     bool    `json:"dummyBool" db:"dummy_bool"`
+		JsonVal   JsonObj `json:"jsonVal" db:"json_val"`
+	}
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
+
+	_, err := db.Exec(ctx, `
+		CREATE TABLE requests ( 
+		    id TEXT NOT NULL, 
+		    timestamp INTEGER NOT NULL, 
+		    str_val TEXT NOT NULL, 
+		    float_val REAL NOT NULL, 
+		    dummy_bool INTEGER NOT NULL  CHECK(dummy_bool IN (0, 1)), 
+		    json_val TEXT NOT NULL, 
+		    PRIMARY KEY (id) 
+		) STRICT
+`, PP{})
+	tst.AssertNoErr(t, err)
+
+	_, err = InsertSingle(ctx, db, "requests", request{
+		ID:        "9927",
+		Timestamp: 12321,
+		StrVal:    "hello world",
+		Dummy:     true,
+		FloatVal:  3.14159,
+		JsonVal: JsonObj{
+			"firs":   1,
+			"second": true,
+		},
+	})
+	tst.AssertNoErr(t, err)
+}
+
+func TestUpdateSingle(t *testing.T) {
+
+	type request struct {
+		ID        string  `json:"id"        db:"id"`
+		Timestamp int     `json:"timestamp" db:"timestamp"`
+		StrVal    string  `json:"strVal"    db:"str_val"`
+		FloatVal  float64 `json:"floatVal"    db:"float_val"`
+		Dummy     bool    `json:"dummyBool" db:"dummy_bool"`
+		JsonVal   JsonObj `json:"jsonVal" db:"json_val"`
+	}
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
+
+	_, err := db.Exec(ctx, `
+		CREATE TABLE requests ( 
+		    id TEXT NOT NULL, 
+		    timestamp INTEGER NOT NULL, 
+		    str_val TEXT NOT NULL, 
+		    float_val REAL NOT NULL, 
+		    dummy_bool INTEGER NOT NULL  CHECK(dummy_bool IN (0, 1)), 
+		    json_val TEXT NOT NULL, 
+		    PRIMARY KEY (id) 
+		) STRICT
+`, PP{})
+	tst.AssertNoErr(t, err)
+
+	_, err = InsertSingle(ctx, db, "requests", request{
+		ID:        "9927",
+		Timestamp: 12321,
+		StrVal:    "hello world",
+		Dummy:     true,
+		FloatVal:  3.14159,
+		JsonVal: JsonObj{
+			"first":  1,
+			"second": true,
+		},
+	})
+	tst.AssertNoErr(t, err)
+
+	v, err := QuerySingle[request](ctx, db, "SELECT * FROM requests WHERE id = '9927' LIMIT 1", PP{}, SModeExtended, Safe)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, v.Timestamp, 12321)
+	tst.AssertEqual(t, v.StrVal, "hello world")
+	tst.AssertEqual(t, v.Dummy, true)
+	tst.AssertEqual(t, v.FloatVal, 3.14159)
+	tst.AssertStrRepEqual(t, v.JsonVal["first"], 1)
+	tst.AssertStrRepEqual(t, v.JsonVal["second"], true)
+
+	_, err = UpdateSingle(ctx, db, "requests", request{
+		ID:        "9927",
+		Timestamp: 9999,
+		StrVal:    "9999 hello world",
+		Dummy:     false,
+		FloatVal:  123.222,
+		JsonVal: JsonObj{
+			"first":  2,
+			"second": false,
+		},
+	}, "id")
+
+	v, err = QuerySingle[request](ctx, db, "SELECT * FROM requests WHERE id = '9927' LIMIT 1", PP{}, SModeExtended, Safe)
+	tst.AssertNoErr(t, err)
+
+	tst.AssertEqual(t, v.Timestamp, 9999)
+	tst.AssertEqual(t, v.StrVal, "9999 hello world")
+	tst.AssertEqual(t, v.Dummy, false)
+	tst.AssertEqual(t, v.FloatVal, 123.222)
+	tst.AssertStrRepEqual(t, v.JsonVal["first"], 2)
+	tst.AssertStrRepEqual(t, v.JsonVal["second"], false)
+}
+
+func TestInsertMultiple(t *testing.T) {
+
+	type request struct {
+		ID        string  `json:"id"        db:"id"`
+		Timestamp int     `json:"timestamp" db:"timestamp"`
+		StrVal    string  `json:"strVal"    db:"str_val"`
+		FloatVal  float64 `json:"floatVal"    db:"float_val"`
+		Dummy     bool    `json:"dummyBool" db:"dummy_bool"`
+		JsonVal   JsonObj `json:"jsonVal" db:"json_val"`
+	}
+
+	if !langext.InArray("sqlite3", sql.Drivers()) {
+		sqlite.RegisterAsSQLITE3()
+	}
+
+	ctx := context.Background()
+
+	dbdir := t.TempDir()
+	dbfile1 := filepath.Join(dbdir, langext.MustHexUUID()+".sqlite3")
+
+	url := fmt.Sprintf("file:%s?_pragma=journal_mode(%s)&_pragma=timeout(%d)&_pragma=foreign_keys(%s)&_pragma=busy_timeout(%d)", dbfile1, "DELETE", 1000, "true", 1000)
+
+	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
+
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
+
+	_, err := db.Exec(ctx, `
+		CREATE TABLE requests ( 
+		    id TEXT NOT NULL, 
+		    timestamp INTEGER NOT NULL, 
+		    str_val TEXT NOT NULL, 
+		    float_val REAL NOT NULL, 
+		    dummy_bool INTEGER NOT NULL  CHECK(dummy_bool IN (0, 1)), 
+		    json_val TEXT NOT NULL, 
+		    PRIMARY KEY (id) 
+		) STRICT
+`, PP{})
+	tst.AssertNoErr(t, err)
+
+	_, err = InsertMultiple(ctx, db, "requests", []request{
+		{
+			ID:        "1",
+			Timestamp: 1000,
+			StrVal:    "one",
+			Dummy:     true,
+			FloatVal:  0.1,
+			JsonVal: JsonObj{
+				"arr": []int{0},
+			},
+		},
+		{
+			ID:        "2",
+			Timestamp: 2000,
+			StrVal:    "two",
+			Dummy:     true,
+			FloatVal:  0.2,
+			JsonVal: JsonObj{
+				"arr": []int{0, 0},
+			},
+		},
+		{
+			ID:        "3",
+			Timestamp: 3000,
+			StrVal:    "three",
+			Dummy:     true,
+			FloatVal:  0.3,
+			JsonVal: JsonObj{
+				"arr": []int{0, 0, 0},
+			},
+		},
+	}, -1)
+	tst.AssertNoErr(t, err)
+
+	_, err = QuerySingle[request](ctx, db, "SELECT * FROM requests WHERE id = '1' LIMIT 1", PP{}, SModeExtended, Safe)
+	tst.AssertNoErr(t, err)
+
+	_, err = QuerySingle[request](ctx, db, "SELECT * FROM requests WHERE id = '2' LIMIT 1", PP{}, SModeExtended, Safe)
+	tst.AssertNoErr(t, err)
+
+	_, err = QuerySingle[request](ctx, db, "SELECT * FROM requests WHERE id = '3' LIMIT 1", PP{}, SModeExtended, Safe)
+	tst.AssertNoErr(t, err)
+}

sq/sq_test.go

@@ -36,8 +36,7 @@ func TestTypeConverter1(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)
@@ -71,8 +70,7 @@ func TestTypeConverter2(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)
@@ -116,8 +114,7 @@ func TestTypeConverter3(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)

sq/transaction.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"github.com/jmoiron/sqlx"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 )
 
@@ -48,7 +49,7 @@ func (tx *transaction) Rollback() error {
 	for _, v := range tx.db.lstr {
 		err := v.PreTxRollback(tx.id)
 		if err != nil {
-			return err
+			return exerr.Wrap(err, "failed to call SQL pre-rollback listener").Int("tx.id", int(tx.id)).Build()
 		}
 	}
 
@@ -69,7 +70,7 @@ func (tx *transaction) Commit() error {
 	for _, v := range tx.db.lstr {
 		err := v.PreTxCommit(tx.id)
 		if err != nil {
-			return err
+			return exerr.Wrap(err, "failed to call SQL pre-commit listener").Int("tx.id", int(tx.id)).Build()
 		}
 	}
 
@@ -91,7 +92,7 @@ func (tx *transaction) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Re
 	for _, v := range tx.db.lstr {
 		err := v.PreExec(ctx, langext.Ptr(tx.id), &sqlstr, &prep)
 		if err != nil {
-			return nil, err
+			return nil, exerr.Wrap(err, "failed to call SQL pre-exec listener").Int("tx.id", int(tx.id)).Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 		}
 	}
 
@@ -106,7 +107,7 @@ func (tx *transaction) Exec(ctx context.Context, sqlstr string, prep PP) (sql.Re
 	}
 
 	if err != nil {
-		return nil, err
+		return nil, exerr.Wrap(err, "Failed to [exec] sql statement").Int("tx.id", int(tx.id)).Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 	}
 	return res, nil
 }
@@ -116,7 +117,7 @@ func (tx *transaction) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx
 	for _, v := range tx.db.lstr {
 		err := v.PreQuery(ctx, langext.Ptr(tx.id), &sqlstr, &prep)
 		if err != nil {
-			return nil, err
+			return nil, exerr.Wrap(err, "failed to call SQL pre-query listener").Int("tx.id", int(tx.id)).Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 		}
 	}
 
@@ -131,7 +132,7 @@ func (tx *transaction) Query(ctx context.Context, sqlstr string, prep PP) (*sqlx
 	}
 
 	if err != nil {
-		return nil, err
+		return nil, exerr.Wrap(err, "Failed to [query] sql statement").Int("tx.id", int(tx.id)).Str("original_sql", origsql).Str("sql", sqlstr).Any("sql_params", prep).Build()
 	}
 	return rows, nil
 }

tst/assertions.go

@@ -2,6 +2,7 @@ package tst
 
 import (
 	"encoding/hex"
+	"fmt"
 	"reflect"
 	"runtime/debug"
 	"testing"
@@ -125,3 +126,17 @@ func AssertNoErr(t *testing.T, anerr error) {
 		t.Error("Function returned an error: " + anerr.Error() + "\n" + string(debug.Stack()))
 	}
 }
+
+func AssertStrRepEqual(t *testing.T, actual any, expected any) {
+	t.Helper()
+	if fmt.Sprintf("%v", actual) != fmt.Sprintf("%v", expected) {
+		t.Errorf("values differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}
+
+func AssertStrRepNotEqual(t *testing.T, actual any, expected any) {
+	t.Helper()
+	if fmt.Sprintf("%v", actual) == fmt.Sprintf("%v", expected) {
+		t.Errorf("values do not differ: Actual: '%v', Expected: '%v'", actual, expected)
+	}
+}

wmo/collection.go

@@ -38,6 +38,13 @@ type fullTypeRef struct {
 	Index          []int
 }
 
+type IColl interface {
+	Collection() *mongo.Collection
+	Name() string
+	Indexes() mongo.IndexView
+	Drop(ctx context.Context) error
+}
+
 type Coll[TData any] struct {
 	coll                *mongo.Collection                                        // internal mongo collection, access via Collection()
 	dataTypeMap         map[string]fullTypeRef                                   // list of TData fields (only if TData is not an interface)

wmo/queryPaginate.go

@@ -9,7 +9,7 @@ import (
 	pag "gogs.mikescher.com/BlackForestBytes/goext/pagination"
 )
 
-func (c *Coll[TData]) Paginate(ctx context.Context, filter pag.Filter, page int, limit *int) ([]TData, pag.Pagination, error) {
+func (c *Coll[TData]) Paginate(ctx context.Context, filter pag.MongoFilter, page int, limit *int) ([]TData, pag.Pagination, error) {
 	type totalCountResult struct {
 		Count int `bson:"count"`
 	}