migrate to multiple sqlite db files ( primary + requests + logs )

scnserver/db/impl/logs/database.go

@@ -0,0 +1,111 @@
+package logs
+
+import (
+	server "blackforestbytes.com/simplecloudnotifier"
+	"blackforestbytes.com/simplecloudnotifier/db/dbtools"
+	"blackforestbytes.com/simplecloudnotifier/db/impl/logs/schema"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/mattn/go-sqlite3"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/sq"
+	"time"
+)
+
+type Database struct {
+	db  sq.DB
+	pp  *dbtools.DBPreprocessor
+	wal bool
+}
+
+func NewLogsDatabase(cfg server.Config) (*Database, error) {
+	conf := cfg.DBLogs
+
+	url := fmt.Sprintf("file:%s?_journal=%s&_timeout=%d&_fk=%s", conf.File, conf.Journal, conf.Timeout.Milliseconds(), langext.FormatBool(conf.CheckForeignKeys, "true", "false"))
+
+	xdb, err := sqlx.Open("sqlite3", url)
+	if err != nil {
+		return nil, err
+	}
+
+	if conf.SingleConn {
+		xdb.SetMaxOpenConns(1)
+	} else {
+		xdb.SetMaxOpenConns(5)
+		xdb.SetMaxIdleConns(5)
+		xdb.SetConnMaxLifetime(60 * time.Minute)
+		xdb.SetConnMaxIdleTime(60 * time.Minute)
+	}
+
+	qqdb := sq.NewDB(xdb)
+
+	qqdb.AddListener(dbtools.DBLogger{})
+
+	pp, err := dbtools.NewDBPreprocessor(qqdb)
+	if err != nil {
+		return nil, err
+	}
+
+	qqdb.AddListener(pp)
+
+	scndb := &Database{db: qqdb, pp: pp, wal: conf.Journal == "WAL"}
+
+	return scndb, nil
+}
+
+func (db *Database) Migrate(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 24*time.Second)
+	defer cancel()
+
+	currschema, err := db.ReadSchema(ctx)
+	if currschema == 0 {
+
+		_, err = db.db.Exec(ctx, schema.LogsSchema1, sq.PP{})
+		if err != nil {
+			return err
+		}
+
+		err = db.WriteMetaInt(ctx, "schema", 1)
+		if err != nil {
+			return err
+		}
+
+		err = db.pp.Init(ctx)
+		if err != nil {
+			return err
+		}
+
+		return nil
+
+	} else if currschema == 1 {
+		return nil // current
+	} else {
+		return errors.New(fmt.Sprintf("Unknown DB schema: %d", currschema))
+	}
+
+}
+
+func (db *Database) Ping(ctx context.Context) error {
+	return db.db.Ping(ctx)
+}
+
+func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
+	return db.db.BeginTransaction(ctx, sql.LevelDefault)
+}
+
+func (db *Database) Stop(ctx context.Context) error {
+	if db.wal {
+		_, err := db.db.Exec(ctx, "PRAGMA wal_checkpoint;", sq.PP{})
+		if err != nil {
+			return err
+		}
+	}
+	err := db.db.Exit()
+	if err != nil {
+		return err
+	}
+	return nil
+}

scnserver/db/impl/logs/meta.go

@@ -0,0 +1,242 @@
+package logs
+
+import (
+	"context"
+	"errors"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/sq"
+)
+
+func (db *Database) ReadSchema(ctx context.Context) (retval int, reterr error) {
+
+	r1, err := db.db.Query(ctx, "SELECT name FROM sqlite_master WHERE type = :typ AND name = :name", sq.PP{"typ": "table", "name": "meta"})
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		err = r1.Close()
+		if err != nil {
+			// overwrite return values
+			retval = 0
+			reterr = err
+		}
+	}()
+
+	if !r1.Next() {
+		return 0, nil
+	}
+
+	err = r1.Close()
+	if err != nil {
+		return 0, err
+	}
+
+	r2, err := db.db.Query(ctx, "SELECT value_int FROM meta WHERE meta_key = :key", sq.PP{"key": "schema"})
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		err = r2.Close()
+		if err != nil {
+			// overwrite return values
+			retval = 0
+			reterr = err
+		}
+	}()
+
+	if !r2.Next() {
+		return 0, errors.New("no schema entry in meta table")
+	}
+
+	var dbschema int
+	err = r2.Scan(&dbschema)
+	if err != nil {
+		return 0, err
+	}
+
+	err = r2.Close()
+	if err != nil {
+		return 0, err
+	}
+
+	return dbschema, nil
+}
+
+func (db *Database) WriteMetaString(ctx context.Context, key string, value string) error {
+	_, err := db.db.Exec(ctx, "INSERT INTO meta (meta_key, value_txt) VALUES (:key, :val) ON CONFLICT(meta_key) DO UPDATE SET value_txt = :val", sq.PP{
+		"key": key,
+		"val": value,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (db *Database) WriteMetaInt(ctx context.Context, key string, value int64) error {
+	_, err := db.db.Exec(ctx, "INSERT INTO meta (meta_key, value_int) VALUES (:key, :val) ON CONFLICT(meta_key) DO UPDATE SET value_int = :val", sq.PP{
+		"key": key,
+		"val": value,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (db *Database) WriteMetaReal(ctx context.Context, key string, value float64) error {
+	_, err := db.db.Exec(ctx, "INSERT INTO meta (meta_key, value_real) VALUES (:key, :val) ON CONFLICT(meta_key) DO UPDATE SET value_real = :val", sq.PP{
+		"key": key,
+		"val": value,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (db *Database) WriteMetaBlob(ctx context.Context, key string, value []byte) error {
+	_, err := db.db.Exec(ctx, "INSERT INTO meta (meta_key, value_blob) VALUES (:key, :val) ON CONFLICT(meta_key) DO UPDATE SET value_blob = :val", sq.PP{
+		"key": key,
+		"val": value,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (db *Database) ReadMetaString(ctx context.Context, key string) (retval *string, reterr error) {
+	r2, err := db.db.Query(ctx, "SELECT value_txt FROM meta WHERE meta_key = :key", sq.PP{"key": key})
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err = r2.Close()
+		if err != nil {
+			// overwrite return values
+			retval = nil
+			reterr = err
+		}
+	}()
+	if !r2.Next() {
+		return nil, errors.New("no matching entry in meta table")
+	}
+
+	var value string
+	err = r2.Scan(&value)
+	if err != nil {
+		return nil, err
+	}
+
+	err = r2.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return langext.Ptr(value), nil
+}
+
+func (db *Database) ReadMetaInt(ctx context.Context, key string) (retval *int64, reterr error) {
+	r2, err := db.db.Query(ctx, "SELECT value_int FROM meta WHERE meta_key = :key", sq.PP{"key": key})
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err = r2.Close()
+		if err != nil {
+			// overwrite return values
+			retval = nil
+			reterr = err
+		}
+	}()
+
+	if !r2.Next() {
+		return nil, errors.New("no matching entry in meta table")
+	}
+
+	var value int64
+	err = r2.Scan(&value)
+	if err != nil {
+		return nil, err
+	}
+
+	err = r2.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return langext.Ptr(value), nil
+}
+
+func (db *Database) ReadMetaReal(ctx context.Context, key string) (retval *float64, reterr error) {
+	r2, err := db.db.Query(ctx, "SELECT value_real FROM meta WHERE meta_key = :key", sq.PP{"key": key})
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err = r2.Close()
+		if err != nil {
+			// overwrite return values
+			retval = nil
+			reterr = err
+		}
+	}()
+
+	if !r2.Next() {
+		return nil, errors.New("no matching entry in meta table")
+	}
+
+	var value float64
+	err = r2.Scan(&value)
+	if err != nil {
+		return nil, err
+	}
+
+	err = r2.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return langext.Ptr(value), nil
+}
+
+func (db *Database) ReadMetaBlob(ctx context.Context, key string) (retval *[]byte, reterr error) {
+	r2, err := db.db.Query(ctx, "SELECT value_blob FROM meta WHERE meta_key = :key", sq.PP{"key": key})
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		err = r2.Close()
+		if err != nil {
+			// overwrite return values
+			retval = nil
+			reterr = err
+		}
+	}()
+
+	if !r2.Next() {
+		return nil, errors.New("no matching entry in meta table")
+	}
+
+	var value []byte
+	err = r2.Scan(&value)
+	if err != nil {
+		return nil, err
+	}
+
+	err = r2.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return langext.Ptr(value), nil
+}
+
+func (db *Database) DeleteMeta(ctx context.Context, key string) error {
+	_, err := db.db.Exec(ctx, "DELETE FROM meta WHERE meta_key = :key", sq.PP{"key": key})
+	if err != nil {
+		return err
+	}
+	return nil
+}

scnserver/db/impl/logs/schema/assets.go

@@ -0,0 +1,6 @@
+package schema
+
+import _ "embed"
+
+//go:embed schema_1.ddl
+var LogsSchema1 string

scnserver/db/impl/logs/schema/schema_1.ddl

@@ -0,0 +1,22 @@
+
+CREATE TABLE `logs`
+(
+    log_id               INTEGER        PRIMARY KEY,
+    timestamp_created    INTEGER        NOT NULL
+
+) STRICT;
+
+
+CREATE TABLE `meta`
+(
+    meta_key     TEXT       NOT NULL,
+    value_int    INTEGER        NULL,
+    value_txt    TEXT           NULL,
+    value_real   REAL           NULL,
+    value_blob   BLOB           NULL,
+
+    PRIMARY KEY (meta_key)
+) STRICT;
+
+
+INSERT INTO meta (meta_key, value_int) VALUES ('schema', 1)

scnserver/db/impl/logs/schema/schema_sqlite.ddl

@@ -0,0 +1,7 @@
+CREATE TABLE sqlite_master (
+    type text,
+    name text,
+    tbl_name text,
+    rootpage integer,
+    sql text
+);

scnserver/db/impl/logs/utils.go

@@ -0,0 +1,25 @@
+package logs
+
+import (
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"time"
+)
+
+func bool2DB(b bool) int {
+	if b {
+		return 1
+	} else {
+		return 0
+	}
+}
+
+func time2DB(t time.Time) int64 {
+	return t.UnixMilli()
+}
+
+func time2DBOpt(t *time.Time) *int64 {
+	if t == nil {
+		return nil
+	}
+	return langext.Ptr(t.UnixMilli())
+}