v0.0.413 fix tests

.idea/.gitignore

@@ -6,3 +6,5 @@
 # Datasource local storage ignored files
 /dataSources/
 /dataSources.local.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

TODO.md

@@ -2,6 +2,8 @@
 
  - cronext
 
- - rfctime.DateOnly
  - rfctime.HMSTimeOnly
- - rfctime.NanoTimeOnly
+ - rfctime.NanoTimeOnly
+
+ - remove sqlx dependency from sq  (unmaintained, and mostly superseeded by our own stuff?)
+ - Move DBLogger and DBPreprocessor to sq

bfcodegen/csid-generate.go

@@ -26,6 +26,10 @@ type CSIDDef struct {
 	Prefix       string
 }
 
+type CSIDGenOptions struct {
+	DebugOutput *bool
+}
+
 var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
@@ -35,7 +39,9 @@ var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGen
 //go:embed csid-generate.template
 var templateCSIDGenerateText string
 
-func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
+func GenerateCharsetIDSpecs(sourceDir string, destFile string, opt CSIDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -81,13 +87,18 @@ func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allIDs = append(allIDs, fileIDs...)
 
@@ -113,7 +124,7 @@ func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
+func processCSIDFile(basedir string, fn string, debugOutput bool) ([]CSIDDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -155,7 +166,11 @@ func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
 				Name:         match.GroupByName("name").Value(),
 				Prefix:       match.GroupByName("prefix").Value(),
 			}
-			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
 			ids = append(ids, def)
 		}
 	}

bfcodegen/csid-generate_test.go

@@ -34,10 +34,10 @@ func TestGenerateCSIDSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
-	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go")
+	err = GenerateCharsetIDSpecs(tmpDir, tmpDir+"/csid_gen.go", CSIDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

bfcodegen/enum-generate.go

@@ -37,6 +37,11 @@ type EnumDef struct {
 	Values       []EnumDefVal
 }
 
+type EnumGenOptions struct {
+	DebugOutput *bool
+	GoFormat    *bool
+}
+
 var rexEnumPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexEnumDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+(?P<type>[A-Za-z0-9_]+)\s*//\s*(@enum:type).*$`))
@@ -48,7 +53,7 @@ var rexEnumChecksumConst = rext.W(regexp.MustCompile(`const ChecksumEnumGenerato
 //go:embed enum-generate.template
 var templateEnumGenerateText string
 
-func GenerateEnumSpecs(sourceDir string, destFile string) error {
+func GenerateEnumSpecs(sourceDir string, destFile string, opt EnumGenOptions) error {
 
 	oldChecksum := "N/A"
 	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
@@ -61,7 +66,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 		}
 	}
 
-	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, true)
+	gocode, _, changed, err := _generateEnumSpecs(sourceDir, destFile, oldChecksum, langext.Coalesce(opt.GoFormat, true), langext.Coalesce(opt.DebugOutput, false))
 	if err != nil {
 		return err
 	}
@@ -78,7 +83,7 @@ func GenerateEnumSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool) (string, string, bool, error) {
+func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, gofmt bool, debugOutput bool) (string, string, bool, error) {
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -113,13 +118,18 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileEnums, pn, err := processEnumFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return "", "", false, err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allEnums = append(allEnums, fileEnums...)
 
@@ -146,7 +156,7 @@ func _generateEnumSpecs(sourceDir string, destFile string, oldChecksum string, g
 	return string(fdata), newChecksum, true, nil
 }
 
-func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
+func processEnumFile(basedir string, fn string, debugOutput bool) ([]EnumDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -190,7 +200,10 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 				Values:       make([]EnumDefVal, 0),
 			}
 			enums = append(enums, def)
-			fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+
+			if debugOutput {
+				fmt.Printf("Found enum definition { '%s' -> '%s' }\n", def.EnumTypeName, def.Type)
+			}
 		}
 
 		if match, ok := rexEnumValueDef.MatchFirst(line); ok {
@@ -230,16 +243,21 @@ func processEnumFile(basedir string, fn string) ([]EnumDef, string, error) {
 				if v.EnumTypeName == typename {
 					enums[i].Values = append(enums[i].Values, def)
 					found = true
-					if def.Description != nil {
-						fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
-					} else {
-						fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+
+					if debugOutput {
+						if def.Description != nil {
+							fmt.Printf("Found enum value [%s] for '%s'  ('%s')\n", def.Value, def.VarName, *def.Description)
+						} else {
+							fmt.Printf("Found enum value [%s] for '%s'\n", def.Value, def.VarName)
+						}
 					}
 					break
 				}
 			}
 			if !found {
-				fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				if debugOutput {
+					fmt.Printf("Found non-enum value [%s] for '%s' ( looks like enum value, but no matching @enum:type )\n", def.Value, def.VarName)
+				}
 			}
 		}
 	}

bfcodegen/enum-generate.template

@@ -7,6 +7,8 @@ import "gogs.mikescher.com/BlackForestBytes/goext/enums"
 
 const ChecksumEnumGenerator = "{{.Checksum}}" // GoExtVersion: {{.GoextVersion}}
 
+{{ $pkgname  := .PkgName }}
+
 {{range .Enums}}
 
 {{ $hasStr   := ( . | hasStr   ) }}
@@ -97,6 +99,14 @@ func (e {{.EnumTypeName}}) VarName() string {
 	return ""
 }
 
+func (e {{.EnumTypeName}}) TypeName() string {
+	return "{{$typename}}"
+}
+
+func (e {{.EnumTypeName}}) PackageName() string {
+	return "{{$pkgname }}"
+}
+
 func (e {{.EnumTypeName}}) Meta() enums.EnumMetaValue {
     {{if $hasDescr}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: langext.Ptr(e.Description())} {{else}} return enums.EnumMetaValue{VarName: e.VarName(), Value: e, Description: nil} {{end}}
 }
@@ -134,4 +144,12 @@ func {{.EnumTypeName}}ValuesDescriptionMeta() []enums.EnumDescriptionMetaValue {
 }
 {{end}}
 
-{{end}}
+{{end}}
+
+// ================================ ================= ================================
+
+func AllPackageEnums() []enums.Enum {
+    return []enums.Enum{ {{range .Enums}}
+        {{ if gt (len .Values) 0 }} {{  $v := index .Values 0 }} {{ $v.VarName}}, {{end}} // {{ .EnumTypeName }} {{end}}
+    }
+}

bfcodegen/enum-generate_test.go

@@ -37,10 +37,10 @@ func TestGenerateEnumSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	s1, cs1, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
 	tst.AssertNoErr(t, err)
 
-	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true)
+	s2, cs2, _, err := _generateEnumSpecs(tmpDir, "", "N/A", true, true)
 	tst.AssertNoErr(t, err)
 
 	tst.AssertEqual(t, cs1, cs2)
@@ -76,7 +76,7 @@ func TestGenerateEnumSpecsData(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true)
+	s1, _, _, err := _generateEnumSpecs(tmpDir, "", "", true, true)
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

bfcodegen/id-generate.go

@@ -25,6 +25,10 @@ type IDDef struct {
 	Name         string
 }
 
+type IDGenOptions struct {
+	DebugOutput *bool
+}
+
 var rexIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
 
 var rexIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@id:type).*$`))
@@ -34,7 +38,9 @@ var rexIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumIDGenerator =
 //go:embed id-generate.template
 var templateIDGenerateText string
 
-func GenerateIDSpecs(sourceDir string, destFile string) error {
+func GenerateIDSpecs(sourceDir string, destFile string, opt IDGenOptions) error {
+
+	debugOutput := langext.Coalesce(opt.DebugOutput, false)
 
 	files, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -80,13 +86,18 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 	pkgname := ""
 
 	for _, f := range files {
-		fmt.Printf("========= %s =========\n\n", f.Name())
-		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if debugOutput {
+			fmt.Printf("========= %s =========\n\n", f.Name())
+		}
+
+		fileIDs, pn, err := processIDFile(sourceDir, path.Join(sourceDir, f.Name()), debugOutput)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("\n")
+		if debugOutput {
+			fmt.Printf("\n")
+		}
 
 		allIDs = append(allIDs, fileIDs...)
 
@@ -112,7 +123,7 @@ func GenerateIDSpecs(sourceDir string, destFile string) error {
 	return nil
 }
 
-func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
+func processIDFile(basedir string, fn string, debugOutput bool) ([]IDDef, string, error) {
 	file, err := os.Open(fn)
 	if err != nil {
 		return nil, "", err
@@ -153,7 +164,11 @@ func processIDFile(basedir string, fn string) ([]IDDef, string, error) {
 				FileRelative: rfp,
 				Name:         match.GroupByName("name").Value(),
 			}
-			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+
+			if debugOutput {
+				fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			}
+
 			ids = append(ids, def)
 		}
 	}

bfcodegen/id-generate_test.go

@@ -34,10 +34,10 @@ func TestGenerateIDSpecs(t *testing.T) {
 	_, err = cmdext.Runner("tar").Arg("-xvzf").Arg(tmpFile).Arg("-C").Arg(tmpDir).FailOnExitCode().FailOnTimeout().Timeout(time.Minute).Run()
 	tst.AssertNoErr(t, err)
 
-	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", IDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
-	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go")
+	err = GenerateIDSpecs(tmpDir, tmpDir+"/id_gen.go", IDGenOptions{DebugOutput: langext.PTrue})
 	tst.AssertNoErr(t, err)
 
 	fmt.Println()

enums/enum.go

@@ -5,6 +5,8 @@ type Enum interface {
 	ValuesAny() []any
 	ValuesMeta() []EnumMetaValue
 	VarName() string
+	TypeName() string
+	PackageName() string
 }
 
 type StringEnum interface {

ginext/engine.go

@@ -39,6 +39,7 @@ type ginRouteSpec struct {
 type Options struct {
 	AllowCors             *bool                                     // Add cors handler to allow all CORS requests on the default http methods
 	GinDebug              *bool                                     // Set gin.debug to true (adds more logs)
+	SuppressGinLogs       *bool                                     // Suppress our custom gin logs (even if GinDebug == true)
 	BufferBody            *bool                                     // Buffers the input body stream, this way the ginext error handler can later include the whole request body
 	Timeout               *time.Duration                            // The default handler timeout
 	ListenerBeforeRequest []func(g *gin.Context)                    // Register listener that are called before the handler method
@@ -51,7 +52,7 @@ func NewEngine(opt Options) *GinWrapper {
 
 	wrapper := &GinWrapper{
 		engine:                engine,
-		suppressGinLogs:       false,
+		suppressGinLogs:       langext.Coalesce(opt.SuppressGinLogs, false),
 		allowCors:             langext.Coalesce(opt.AllowCors, false),
 		ginDebug:              langext.Coalesce(opt.GinDebug, true),
 		bufferBody:            langext.Coalesce(opt.BufferBody, false),
@@ -73,12 +74,12 @@ func NewEngine(opt Options) *GinWrapper {
 	if !wrapper.ginDebug {
 		gin.SetMode(gin.ReleaseMode)
 
-		ginlogger := gin.Logger()
-		engine.Use(func(context *gin.Context) {
-			if !wrapper.suppressGinLogs {
+		if !wrapper.suppressGinLogs {
+			ginlogger := gin.Logger()
+			engine.Use(func(context *gin.Context) {
 				ginlogger(context)
-			}
-		})
+			})
+		}
 	} else {
 		gin.SetMode(gin.DebugMode)
 	}

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.399"
+const GoextVersion = "0.0.413"
 
-const GoextVersionTimestamp = "2024-03-09T14:16:35+0100"
+const GoextVersionTimestamp = "2024-03-11T21:00:30+0100"

langext/coalesce.go

@@ -5,12 +5,76 @@ import (
 	"time"
 )
 
-func Coalesce[T any](v *T, def T) T {
-	if v == nil {
-		return def
-	} else {
-		return *v
+func Coalesce[T any](v1 *T, def T) T {
+	if v1 != nil {
+		return *v1
 	}
+
+	return def
+}
+
+func CoalesceOpt[T any](v1 *T, v2 *T) *T {
+	if v1 != nil {
+		return v1
+	}
+
+	return v2
+}
+
+func Coalesce3[T any](v1 *T, v2 *T, def T) T {
+	if v1 != nil {
+		return *v1
+	}
+
+	if v2 != nil {
+		return *v2
+	}
+
+	return def
+}
+
+func Coalesce3Opt[T any](v1 *T, v2 *T, v3 *T) *T {
+	if v1 != nil {
+		return v1
+	}
+
+	if v2 != nil {
+		return v2
+	}
+
+	return v3
+}
+
+func Coalesce4[T any](v1 *T, v2 *T, v3 *T, def T) T {
+	if v1 != nil {
+		return *v1
+	}
+
+	if v2 != nil {
+		return *v2
+	}
+
+	if v3 != nil {
+		return *v3
+	}
+
+	return def
+}
+
+func Coalesce4Opt[T any](v1 *T, v2 *T, v3 *T, v4 *T) *T {
+	if v1 != nil {
+		return v1
+	}
+
+	if v2 != nil {
+		return v2
+	}
+
+	if v3 != nil {
+		return v3
+	}
+
+	return v4
 }
 
 func CoalesceString(s *string, def string) string {

rfctime/date.go

@@ -9,6 +9,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson/bsonrw"
 	"go.mongodb.org/mongo-driver/bson/bsontype"
 	"reflect"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -65,36 +67,20 @@ func (t *Date) UnmarshalJSON(data []byte) error {
 	if err := json.Unmarshal(data, &str); err != nil {
 		return err
 	}
-	t0, err := time.Parse(t.FormatStr(), str)
-	if err != nil {
-		return err
-	}
-	t.Year = t0.Year()
-	t.Month = int(t0.Month())
-	t.Day = t0.Day()
-	return nil
+	return t.ParseString(str)
 }
 
 func (t Date) MarshalJSON() ([]byte, error) {
-	str := t.TimeUTC().Format(t.FormatStr())
+	str := t.String()
 	return json.Marshal(str)
 }
 
 func (t Date) MarshalText() ([]byte, error) {
-	b := make([]byte, 0, len(t.FormatStr()))
-	return t.TimeUTC().AppendFormat(b, t.FormatStr()), nil
+	return []byte(t.String()), nil
 }
 
 func (t *Date) UnmarshalText(data []byte) error {
-	var err error
-	v, err := time.Parse(t.FormatStr(), string(data))
-	if err != nil {
-		return err
-	}
-	t.Year = v.Year()
-	t.Month = int(v.Month())
-	t.Day = v.Day()
-	return nil
+	return t.ParseString(string(data))
 }
 
 func (t *Date) UnmarshalBSONValue(bt bsontype.Type, data []byte) error {
@@ -164,7 +150,7 @@ func (t Date) DecodeValue(dc bsoncodec.DecodeContext, vr bsonrw.ValueReader, val
 }
 
 func (t Date) Serialize() string {
-	return t.TimeUTC().Format(t.FormatStr())
+	return t.String()
 }
 
 func (t Date) FormatStr() string {
@@ -212,11 +198,48 @@ func (t Date) Format(layout string) string {
 }
 
 func (t Date) GoString() string {
-	return t.TimeUTC().GoString()
+	return fmt.Sprintf("rfctime.Date{Year: %d, Month: %d, Day: %d}", t.Year, t.Month, t.Day)
 }
 
 func (t Date) String() string {
-	return t.TimeUTC().String()
+	return fmt.Sprintf("%04d-%02d-%02d", t.Year, t.Month, t.Day)
+}
+
+func (t *Date) ParseString(v string) error {
+	split := strings.Split(v, "-")
+	if len(split) != 3 {
+		return errors.New("invalid date format: " + v)
+	}
+	year, err := strconv.ParseInt(split[0], 10, 32)
+	if err != nil {
+		return errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	month, err := strconv.ParseInt(split[1], 10, 32)
+	if err != nil {
+		return errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+	day, err := strconv.ParseInt(split[2], 10, 32)
+	if err != nil {
+		return errors.New("invalid date format: " + v + ": " + err.Error())
+	}
+
+	if year < 0 {
+		return errors.New("invalid date format: " + v + ": year is negative")
+	}
+
+	if month < 1 || month > 12 {
+		return errors.New("invalid date format: " + v + ": month is out of range")
+	}
+
+	if day < 1 || day > 31 {
+		return errors.New("invalid date format: " + v + ": day is out of range")
+	}
+
+	t.Year = int(year)
+	t.Month = int(month)
+	t.Day = int(day)
+
+	return nil
 }
 
 func NewDate(t time.Time) Date {

sq/builder_test.go

@@ -52,8 +52,7 @@ func TestCreateUpdateStatement(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)

sq/commentTrimmer.go

@@ -0,0 +1,32 @@
+package sq
+
+import (
+	"context"
+	"strings"
+)
+
+var CommentTrimmer = NewPreListener(fnTrimComments)
+
+func fnTrimComments(ctx context.Context, cmdtype string, id *uint16, sql *string, params *PP) error {
+
+	res := make([]string, 0)
+
+	for _, s := range strings.Split(*sql, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(s), "--") {
+			continue
+		}
+
+		idx := strings.Index(s, "--")
+		if idx != -1 {
+			s = s[:idx]
+		}
+
+		s = strings.TrimRight(s, " \t\r\n")
+
+		res = append(res, s)
+	}
+
+	*sql = strings.Join(res, "\n")
+
+	return nil
+}

sq/converterDefault.go

@@ -7,8 +7,6 @@ import (
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/rfctime"
 	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
-	"strconv"
-	"strings"
 	"time"
 )
 
@@ -79,24 +77,12 @@ var ConverterRFC339NanoTimeToString = NewDBTypeConverter[rfctime.RFC3339NanoTime
 var ConverterRFCDateToString = NewDBTypeConverter[rfctime.Date, string](func(v rfctime.Date) (string, error) {
 	return fmt.Sprintf("%04d-%02d-%02d", v.Year, v.Month, v.Day), nil
 }, func(v string) (rfctime.Date, error) {
-	split := strings.Split(v, "-")
-	if len(split) != 3 {
-		return rfctime.Date{}, errors.New("invalid date format: " + v)
+	d := rfctime.Date{}
+	if err := d.ParseString(v); err != nil {
+		return rfctime.Date{}, err
+	} else {
+		return d, nil
 	}
-	year, err := strconv.ParseInt(split[0], 10, 32)
-	if err != nil {
-		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
-	}
-	month, err := strconv.ParseInt(split[0], 10, 32)
-	if err != nil {
-		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
-	}
-	day, err := strconv.ParseInt(split[0], 10, 32)
-	if err != nil {
-		return rfctime.Date{}, errors.New("invalid date format: " + v + ": " + err.Error())
-	}
-
-	return rfctime.Date{Year: int(year), Month: int(month), Day: int(day)}, nil
 })
 
 var ConverterRFCTimeToString = NewDBTypeConverter[rfctime.Time, string](func(v rfctime.Time) (string, error) {

sq/database.go

@@ -17,7 +17,11 @@ type DB interface {
 	AddListener(listener Listener)
 	Exit() error
 	RegisterConverter(DBTypeConverter)
-	RegisterDefaultConverter()
+}
+
+type DBOptions struct {
+	RegisterDefaultConverter *bool
+	RegisterCommentTrimmer   *bool
 }
 
 type database struct {
@@ -28,13 +32,23 @@ type database struct {
 	conv  []DBTypeConverter
 }
 
-func NewDB(db *sqlx.DB) DB {
-	return &database{
+func NewDB(db *sqlx.DB, opt DBOptions) DB {
+	sqdb := &database{
 		db:    db,
 		txctr: 0,
 		lock:  sync.Mutex{},
 		lstr:  make([]Listener, 0),
 	}
+
+	if langext.Coalesce(opt.RegisterDefaultConverter, true) {
+		sqdb.registerDefaultConverter()
+	}
+
+	if langext.Coalesce(opt.RegisterCommentTrimmer, true) {
+		sqdb.AddListener(CommentTrimmer)
+	}
+
+	return sqdb
 }
 
 func (db *database) AddListener(listener Listener) {
@@ -141,7 +155,7 @@ func (db *database) RegisterConverter(conv DBTypeConverter) {
 	db.conv = append(db.conv, conv)
 }
 
-func (db *database) RegisterDefaultConverter() {
+func (db *database) registerDefaultConverter() {
 	db.RegisterConverter(ConverterBoolToBit)
 
 	db.RegisterConverter(ConverterTimeToUnixMillis)

sq/filter.go

@@ -47,3 +47,10 @@ func NewSimplePaginateFilter(filterClause string, filterParams PP, sort []Filter
 		},
 	}
 }
+
+func NewEmptyPaginateFilter() PaginateFilter {
+	return genericPaginateFilter{
+		sql:  func(params PP) (string, string, []string) { return "1=1", "", nil },
+		sort: func() []FilterSort { return make([]FilterSort, 0) },
+	}
+}

sq/hasher.go

@@ -31,7 +31,7 @@ func HashMattnSqliteSchema(ctx context.Context, schemaStr string) (string, error
 		return "", err
 	}
 
-	db := NewDB(xdb)
+	db := NewDB(xdb, DBOptions{})
 
 	_, err = db.Exec(ctx, schemaStr, PP{})
 	if err != nil {
@@ -59,7 +59,7 @@ func HashGoSqliteSchema(ctx context.Context, schemaStr string) (string, error) {
 		return "", err
 	}
 
-	db := NewDB(xdb)
+	db := NewDB(xdb, DBOptions{})
 
 	_, err = db.Exec(ctx, schemaStr, PP{})
 	if err != nil {

sq/list.go

@@ -0,0 +1,48 @@
+package sq
+
+import (
+	"context"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
+)
+
+func Iterate[TData any](ctx context.Context, q Queryable, table string, filter PaginateFilter, scanMode StructScanMode, scanSec StructScanSafety, page int, limit *int, consumer func(ctx context.Context, v TData) error) (int, error) {
+	if filter == nil {
+		filter = NewEmptyPaginateFilter()
+	}
+
+	prepParams := PP{}
+
+	sortOrder := filter.Sort()
+	sortCond := ""
+	if len(sortOrder) > 0 {
+		sortCond = "ORDER BY "
+		for i, v := range sortOrder {
+			if i > 0 {
+				sortCond += ", "
+			}
+			sortCond += v.Field + " " + string(v.Direction)
+		}
+	}
+
+	pageCond := ""
+	if limit != nil {
+		pageCond += fmt.Sprintf("LIMIT :%s OFFSET :%s", prepParams.Add(*limit+1), prepParams.Add(*limit*(page-1)))
+	}
+
+	filterCond, joinCond, joinTables := filter.SQL(prepParams)
+
+	selectCond := table + ".*"
+	for _, v := range joinTables {
+		selectCond += ", " + v + ".*"
+	}
+
+	sqlQueryData := "SELECT " + selectCond + " FROM " + table + " " + joinCond + " WHERE ( " + filterCond + " ) " + sortCond + " " + pageCond
+
+	rows, err := q.Query(ctx, sqlQueryData, prepParams)
+	if err != nil {
+		return 0, exerr.Wrap(err, "failed to list paginated entries from DB").Str("table", table).Any("filter", filter).Int("page", page).Any("limit", limit).Build()
+	}
+
+	return IterateAll[TData](ctx, q, rows, scanMode, scanSec, true, consumer)
+}

sq/listener.go

@@ -17,3 +17,172 @@ type Listener interface {
 	PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 	PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP)
 }
+
+type genListener struct {
+	prePing        func(ctx context.Context) error
+	preTxBegin     func(ctx context.Context, txid uint16) error
+	preTxCommit    func(txid uint16) error
+	preTxRollback  func(txid uint16) error
+	preQuery       func(ctx context.Context, txID *uint16, sql *string, params *PP) error
+	preExec        func(ctx context.Context, txID *uint16, sql *string, params *PP) error
+	postPing       func(result error)
+	postTxBegin    func(txid uint16, result error)
+	postTxCommit   func(txid uint16, result error)
+	postTxRollback func(txid uint16, result error)
+	postQuery      func(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+	postExec       func(txID *uint16, sqlOriginal string, sqlReal string, params PP)
+}
+
+func (g genListener) PrePing(ctx context.Context) error {
+	if g.prePing != nil {
+		return g.prePing(ctx)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxBegin(ctx context.Context, txid uint16) error {
+	if g.preTxBegin != nil {
+		return g.preTxBegin(ctx, txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxCommit(txid uint16) error {
+	if g.preTxCommit != nil {
+		return g.preTxCommit(txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreTxRollback(txid uint16) error {
+	if g.preTxRollback != nil {
+		return g.preTxRollback(txid)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreQuery(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+	if g.preQuery != nil {
+		return g.preQuery(ctx, txID, sql, params)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PreExec(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+	if g.preExec != nil {
+		return g.preExec(ctx, txID, sql, params)
+	} else {
+		return nil
+	}
+}
+
+func (g genListener) PostPing(result error) {
+	if g.postPing != nil {
+		g.postPing(result)
+	}
+}
+
+func (g genListener) PostTxBegin(txid uint16, result error) {
+	if g.postTxBegin != nil {
+		g.postTxBegin(txid, result)
+	}
+}
+
+func (g genListener) PostTxCommit(txid uint16, result error) {
+	if g.postTxCommit != nil {
+		g.postTxCommit(txid, result)
+	}
+}
+
+func (g genListener) PostTxRollback(txid uint16, result error) {
+	if g.postTxRollback != nil {
+		g.postTxRollback(txid, result)
+	}
+}
+
+func (g genListener) PostQuery(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+	if g.postQuery != nil {
+		g.postQuery(txID, sqlOriginal, sqlReal, params)
+	}
+}
+
+func (g genListener) PostExec(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+	if g.postExec != nil {
+		g.postExec(txID, sqlOriginal, sqlReal, params)
+	}
+}
+
+func NewPrePingListener(f func(ctx context.Context) error) Listener {
+	return genListener{prePing: f}
+}
+
+func NewPreTxBeginListener(f func(ctx context.Context, txid uint16) error) Listener {
+	return genListener{preTxBegin: f}
+}
+
+func NewPreTxCommitListener(f func(txid uint16) error) Listener {
+	return genListener{preTxCommit: f}
+}
+
+func NewPreTxRollbackListener(f func(txid uint16) error) Listener {
+	return genListener{preTxRollback: f}
+}
+
+func NewPreQueryListener(f func(ctx context.Context, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{preQuery: f}
+}
+
+func NewPreExecListener(f func(ctx context.Context, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{preExec: f}
+}
+
+func NewPreListener(f func(ctx context.Context, cmdtype string, txID *uint16, sql *string, params *PP) error) Listener {
+	return genListener{
+		preExec: func(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+			return f(ctx, "EXEC", txID, sql, params)
+		},
+		preQuery: func(ctx context.Context, txID *uint16, sql *string, params *PP) error {
+			return f(ctx, "QUERY", txID, sql, params)
+		},
+	}
+}
+
+func NewPostPingListener(f func(result error)) Listener {
+	return genListener{postPing: f}
+}
+
+func NewPostTxBeginListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxBegin: f}
+}
+
+func NewPostTxCommitListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxCommit: f}
+}
+
+func NewPostTxRollbackListener(f func(txid uint16, result error)) Listener {
+	return genListener{postTxRollback: f}
+}
+
+func NewPostQueryListener(f func(txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{postQuery: f}
+}
+
+func NewPostExecListener(f func(txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{postExec: f}
+}
+
+func NewPostListener(f func(cmdtype string, txID *uint16, sqlOriginal string, sqlReal string, params PP)) Listener {
+	return genListener{
+		postExec: func(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+			f("EXEC", txID, sqlOriginal, sqlReal, params)
+		},
+		postQuery: func(txID *uint16, sqlOriginal string, sqlReal string, params PP) {
+			f("QUERY", txID, sqlOriginal, sqlReal, params)
+		},
+	}
+}

sq/paginate.go

@@ -9,6 +9,10 @@ import (
 )
 
 func Paginate[TData any](ctx context.Context, q Queryable, table string, filter PaginateFilter, scanMode StructScanMode, scanSec StructScanSafety, page int, limit *int) ([]TData, pag.Pagination, error) {
+	if filter == nil {
+		filter = NewEmptyPaginateFilter()
+	}
+
 	prepParams := PP{}
 
 	sortOrder := filter.Sort()
@@ -90,6 +94,10 @@ func Paginate[TData any](ctx context.Context, q Queryable, table string, filter
 }
 
 func Count(ctx context.Context, q Queryable, table string, filter PaginateFilter) (int, error) {
+	if filter == nil {
+		filter = NewEmptyPaginateFilter()
+	}
+
 	prepParams := PP{}
 
 	filterCond, joinCond, _ := filter.SQL(prepParams)

sq/scanner.go

@@ -333,3 +333,79 @@ func ScanAll[TData any](ctx context.Context, q Queryable, rows *sqlx.Rows, mode
 	}
 	return res, nil
 }
+
+func IterateAll[TData any](ctx context.Context, q Queryable, rows *sqlx.Rows, mode StructScanMode, sec StructScanSafety, close bool, consumer func(ctx context.Context, v TData) error) (int, error) {
+	var strscan *StructScanner
+
+	if sec == Safe {
+		strscan = NewStructScanner(rows, false)
+		var data TData
+		err := strscan.Start(&data)
+		if err != nil {
+			return 0, err
+		}
+	} else if sec == Unsafe {
+		strscan = NewStructScanner(rows, true)
+		var data TData
+		err := strscan.Start(&data)
+		if err != nil {
+			return 0, err
+		}
+	} else {
+		return 0, errors.New("unknown value for <sec>")
+	}
+
+	rcount := 0
+
+	for rows.Next() {
+
+		if err := ctx.Err(); err != nil {
+			return rcount, err
+		}
+
+		if mode == SModeFast {
+			var data TData
+			err := strscan.StructScanBase(&data)
+			if err != nil {
+				return rcount, err
+			}
+
+			err = consumer(ctx, data)
+			if err != nil {
+				return rcount, exerr.Wrap(err, "").Build()
+			}
+
+			rcount++
+
+		} else if mode == SModeExtended {
+			var data TData
+			err := strscan.StructScanExt(q, &data)
+			if err != nil {
+				return rcount, err
+			}
+
+			err = consumer(ctx, data)
+			if err != nil {
+				return rcount, exerr.Wrap(err, "").Build()
+			}
+
+			rcount++
+
+		} else {
+			return rcount, errors.New("unknown value for <mode>")
+		}
+	}
+
+	if close {
+		err := strscan.rows.Close()
+		if err != nil {
+			return rcount, err
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return rcount, err
+	}
+
+	return rcount, nil
+}

sq/scanner_test.go

@@ -36,8 +36,7 @@ func TestInsertSingle(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, `
 		CREATE TABLE requests ( 
@@ -90,8 +89,7 @@ func TestUpdateSingle(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, `
 		CREATE TABLE requests ( 
@@ -176,8 +174,7 @@ func TestInsertMultiple(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, `
 		CREATE TABLE requests ( 

sq/sq_test.go

@@ -36,8 +36,7 @@ func TestTypeConverter1(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)
@@ -71,8 +70,7 @@ func TestTypeConverter2(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)
@@ -116,8 +114,7 @@ func TestTypeConverter3(t *testing.T) {
 
 	xdb := tst.Must(sqlx.Open("sqlite", url))(t)
 
-	db := NewDB(xdb)
-	db.RegisterDefaultConverter()
+	db := NewDB(xdb, DBOptions{RegisterDefaultConverter: langext.PTrue})
 
 	_, err := db.Exec(ctx, "CREATE TABLE `requests` ( id TEXT NOT NULL, timestamp INTEGER NULL, PRIMARY KEY (id) ) STRICT", PP{})
 	tst.AssertNoErr(t, err)