v0.0.293 fix NPE

README.md

@@ -11,14 +11,15 @@ Potentially needs `export GOPRIVATE="gogs.mikescher.com"`
 ### Packages:
 
 | Name        | Maintainer | Description                                                                                                   |
-|--------------|------------|---------------------------------------------------------------------------------------------------------------|
+|-------------|------------|---------------------------------------------------------------------------------------------------------------|
 | langext     | Mike       | General uttility/helper functions, (everything thats missing from go standard library)                        |
 | mathext     | Mike       | Utility/Helper functions for math                                                                             |
 | cryptext    | Mike       | Utility/Helper functions for encryption                                                                       |
 | syncext     | Mike       | Utility/Helper funtions for multi-threading / mutex / channels                                                |
 | dataext     | Mike       | Various useful data structures                                                                                |
 | zipext      | Mike       | Utility for zip/gzip/tar etc                                                                                  |
-| reflectext   | Mike       | Utility for golagn reflection                                                                                 |
+| reflectext  | Mike       | Utility for golang reflection                                                                                 |
+| fsext       | Mike       | Utility for filesytem access                                                                                  |
 |             |            |                                                                                                               |
 | mongoext    | Mike       | Utility/Helper functions for mongodb                                                                          |
 | cursortoken | Mike       | MongoDB cursortoken implementation                                                                            |

bfcodegen/csid-generate.go

@@ -0,0 +1,365 @@
+package bfcodegen
+
+import (
+	"errors"
+	"fmt"
+	"gogs.mikescher.com/BlackForestBytes/goext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cmdext"
+	"gogs.mikescher.com/BlackForestBytes/goext/cryptext"
+	"gogs.mikescher.com/BlackForestBytes/goext/langext"
+	"gogs.mikescher.com/BlackForestBytes/goext/rext"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type CSIDDef struct {
+	File         string
+	FileRelative string
+	Name         string
+	Prefix       string
+}
+
+var rexCSIDPackage = rext.W(regexp.MustCompile(`^package\s+(?P<name>[A-Za-z0-9_]+)\s*$`))
+
+var rexCSIDDef = rext.W(regexp.MustCompile(`^\s*type\s+(?P<name>[A-Za-z0-9_]+)\s+string\s*//\s*(@csid:type)\s+\[(?P<prefix>[A-Z0-9]{3})].*$`))
+
+var rexCSIDChecksumConst = rext.W(regexp.MustCompile(`const ChecksumCharsetIDGenerator = "(?P<cs>[A-Za-z0-9_]*)"`))
+
+func GenerateCharsetIDSpecs(sourceDir string, destFile string) error {
+
+	files, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	oldChecksum := "N/A"
+	if _, err := os.Stat(destFile); !os.IsNotExist(err) {
+		content, err := os.ReadFile(destFile)
+		if err != nil {
+			return err
+		}
+		if m, ok := rexCSIDChecksumConst.MatchFirst(string(content)); ok {
+			oldChecksum = m.GroupByName("cs").Value()
+		}
+	}
+
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return v.Name() != path.Base(destFile) })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return strings.HasSuffix(v.Name(), ".go") })
+	files = langext.ArrFilter(files, func(v os.DirEntry) bool { return !strings.HasSuffix(v.Name(), "_gen.go") })
+	langext.SortBy(files, func(v os.DirEntry) string { return v.Name() })
+
+	newChecksumStr := goext.GoextVersion
+	for _, f := range files {
+		content, err := os.ReadFile(path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+		newChecksumStr += "\n" + f.Name() + "\t" + cryptext.BytesSha256(content)
+	}
+
+	newChecksum := cryptext.BytesSha256([]byte(newChecksumStr))
+
+	if newChecksum != oldChecksum {
+		fmt.Printf("[IDGenerate] Checksum has changed ( %s -> %s ), will generate new file\n\n", oldChecksum, newChecksum)
+	} else {
+		fmt.Printf("[IDGenerate] Checksum unchanged ( %s ), nothing to do\n", oldChecksum)
+		return nil
+	}
+
+	allIDs := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for _, f := range files {
+		fmt.Printf("========= %s =========\n\n", f.Name())
+		fileIDs, pn, err := processCSIDFile(sourceDir, path.Join(sourceDir, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\n")
+
+		allIDs = append(allIDs, fileIDs...)
+
+		if pn != "" {
+			pkgname = pn
+		}
+	}
+
+	if pkgname == "" {
+		return errors.New("no package name found in any file")
+	}
+
+	err = os.WriteFile(destFile, []byte(fmtCSIDOutput(newChecksum, allIDs, pkgname)), 0o755)
+	if err != nil {
+		return err
+	}
+
+	res, err := cmdext.RunCommand("go", []string{"fmt", destFile}, langext.Ptr(2*time.Second))
+	if err != nil {
+		return err
+	}
+
+	if res.CommandTimedOut {
+		fmt.Println(res.StdCombined)
+		return errors.New("go fmt timed out")
+	}
+	if res.ExitCode != 0 {
+		fmt.Println(res.StdCombined)
+		return errors.New("go fmt did not succeed")
+	}
+
+	return nil
+}
+
+func processCSIDFile(basedir string, fn string) ([]CSIDDef, string, error) {
+	file, err := os.Open(fn)
+	if err != nil {
+		return nil, "", err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	bin, err := io.ReadAll(file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	lines := strings.Split(string(bin), "\n")
+
+	ids := make([]CSIDDef, 0)
+
+	pkgname := ""
+
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "// Code generated by") {
+			break
+		}
+
+		if match, ok := rexCSIDPackage.MatchFirst(line); i == 0 && ok {
+			pkgname = match.GroupByName("name").Value()
+			continue
+		}
+
+		if match, ok := rexCSIDDef.MatchFirst(line); ok {
+
+			rfp, err := filepath.Rel(basedir, fn)
+			if err != nil {
+				return nil, "", err
+			}
+
+			def := CSIDDef{
+				File:         fn,
+				FileRelative: rfp,
+				Name:         match.GroupByName("name").Value(),
+				Prefix:       match.GroupByName("prefix").Value(),
+			}
+			fmt.Printf("Found ID definition { '%s' }\n", def.Name)
+			ids = append(ids, def)
+		}
+	}
+
+	return ids, pkgname, nil
+}
+
+func fmtCSIDOutput(cs string, ids []CSIDDef, pkgname string) string {
+	str := "// Code generated by id-generate.go DO NOT EDIT.\n"
+	str += "\n"
+	str += "package " + pkgname + "\n"
+	str += "\n"
+
+	str += `import "crypto/rand"` + "\n"
+	str += `import "fmt"` + "\n"
+	str += `import "github.com/go-playground/validator/v10"` + "\n"
+	str += `import "github.com/rs/zerolog/log"` + "\n"
+	str += `import "gogs.mikescher.com/BlackForestBytes/goext/exerr"` + "\n"
+	str += `import "gogs.mikescher.com/BlackForestBytes/goext/langext"` + "\n"
+	str += `import "gogs.mikescher.com/BlackForestBytes/goext/rext"` + "\n"
+	str += `import "math/big"` + "\n"
+	str += `import "reflect"` + "\n"
+	str += `import "regexp"` + "\n"
+	str += `import "strings"` + "\n"
+	str += "\n"
+
+	str += "const ChecksumCharsetIDGenerator = \"" + cs + "\" // GoExtVersion: " + goext.GoextVersion + "\n"
+	str += "\n"
+
+	str += "const idlen = 24\n"
+	str += "\n"
+	str += "const checklen = 1\n"
+	str += "\n"
+	str += `const idCharset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"` + "\n"
+	str += "const idCharsetLen = len(idCharset)\n"
+	str += "\n"
+	str += "var charSetReverseMap = generateCharsetMap()\n"
+	str += "\n"
+	str += "const (\n"
+	for _, iddef := range ids {
+		str += "	prefix" + iddef.Name + " = \"" + iddef.Prefix + "\"" + "\n"
+	}
+	str += ")\n"
+	str += "\n"
+	str += "var (\n"
+	for _, iddef := range ids {
+		str += "	regex" + iddef.Name + " = generateRegex(prefix" + iddef.Name + ")" + "\n"
+	}
+	str += ")\n"
+	str += "\n"
+	str += "func generateRegex(prefix string) rext.Regex {\n"
+	str += "	return rext.W(regexp.MustCompile(fmt.Sprintf(\"^%s[%s]{%d}[%s]{%d}$\", prefix, idCharset, idlen-len(prefix)-checklen, idCharset, checklen)))\n"
+	str += "}\n"
+	str += "\n"
+
+	str += `func generateCharsetMap() []int {` + "\n"
+	str += `	result := make([]int, 128)` + "\n"
+	str += `	for i := 0; i < len(result); i++ {` + "\n"
+	str += `		result[i] = -1` + "\n"
+	str += `	}` + "\n"
+	str += `	for idx, chr := range idCharset {` + "\n"
+	str += `		result[int(chr)] = idx` + "\n"
+	str += `	}` + "\n"
+	str += `	return result` + "\n"
+	str += `}` + "\n"
+	str += "\n"
+	str += `func generateID(prefix string) string {` + "\n"
+	str += `	k := ""` + "\n"
+	str += `	max := big.NewInt(int64(idCharsetLen))` + "\n"
+	str += `	checksum := 0` + "\n"
+	str += `	for i := 0; i < idlen-len(prefix)-checklen; i++ {` + "\n"
+	str += `		v, err := rand.Int(rand.Reader, max)` + "\n"
+	str += `		if err != nil {` + "\n"
+	str += `			panic(err)` + "\n"
+	str += `		}` + "\n"
+	str += `		v64 := v.Int64()` + "\n"
+	str += `		k += string(idCharset[v64])` + "\n"
+	str += `		checksum = (checksum + int(v64)) % (idCharsetLen)` + "\n"
+	str += `	}` + "\n"
+	str += `	checkstr := string(idCharset[checksum%idCharsetLen])` + "\n"
+	str += `	return prefix + k + checkstr` + "\n"
+	str += `}` + "\n"
+	str += "\n"
+	str += `func validateID(prefix string, value string) error {` + "\n"
+	str += `	if len(value) != idlen {` + "\n"
+	str += `		return exerr.New(exerr.TypeInvalidCSID, "id has the wrong length").Str("value", value).Build()` + "\n"
+	str += `	}` + "\n"
+	str += "\n"
+	str += `	if !strings.HasPrefix(value, prefix) {` + "\n"
+	str += `		return exerr.New(exerr.TypeInvalidCSID, "id is missing the correct prefix").Str("value", value).Str("prefix", prefix).Build()` + "\n"
+	str += `	}` + "\n"
+	str += "\n"
+	str += `	checksum := 0` + "\n"
+	str += `	for i := len(prefix); i < len(value)-checklen; i++ {` + "\n"
+	str += `		ichr := int(value[i])` + "\n"
+	str += `		if ichr < 0 || ichr >= len(charSetReverseMap) || charSetReverseMap[ichr] == -1 {` + "\n"
+	str += `			return exerr.New(exerr.TypeInvalidCSID, "id contains invalid characters").Str("value", value).Build()` + "\n"
+	str += `		}` + "\n"
+	str += `		checksum = (checksum + charSetReverseMap[ichr]) % (idCharsetLen)` + "\n"
+	str += `	}` + "\n"
+	str += "\n"
+	str += `	checkstr := string(idCharset[checksum%idCharsetLen])` + "\n"
+	str += "\n"
+	str += `	if !strings.HasSuffix(value, checkstr) {` + "\n"
+	str += `		return exerr.New(exerr.TypeInvalidCSID, "id checkstring is invalid").Str("value", value).Str("checkstr", checkstr).Build()` + "\n"
+	str += `	}` + "\n"
+	str += "\n"
+	str += `	return nil` + "\n"
+	str += `}` + "\n"
+	str += "\n"
+	str += `func getRawData(prefix string, value string) string {` + "\n"
+	str += `	if len(value) != idlen {` + "\n"
+	str += `		return ""` + "\n"
+	str += `	}` + "\n"
+	str += `	return value[len(prefix) : idlen-checklen]` + "\n"
+	str += `}` + "\n"
+	str += "\n"
+	str += `func getCheckString(prefix string, value string) string {` + "\n"
+	str += `	if len(value) != idlen {` + "\n"
+	str += `		return ""` + "\n"
+	str += `	}` + "\n"
+	str += `	return value[idlen-checklen:]` + "\n"
+	str += `}` + "\n"
+	str += "\n"
+	str += `func ValidateEntityID(vfl validator.FieldLevel) bool {` + "\n"
+	str += `	if !vfl.Field().CanInterface() {` + "\n"
+	str += `		log.Error().Msgf("Failed to validate EntityID (cannot interface ?!?)")` + "\n"
+	str += `		return false` + "\n"
+	str += `	}` + "\n"
+	str += "\n"
+	str += `	ifvalue := vfl.Field().Interface()` + "\n"
+	str += "\n"
+	str += `	if value1, ok := ifvalue.(EntityID); ok {` + "\n"
+	str += "\n"
+	str += `		if vfl.Field().Type().Kind() == reflect.Pointer && langext.IsNil(value1) {` + "\n"
+	str += `			return true` + "\n"
+	str += `		}` + "\n"
+	str += "\n"
+	str += `		if err := value1.Valid(); err != nil {` + "\n"
+	str += `			log.Debug().Msgf("Failed to validate EntityID '%s' (%s)", value1.String(), err.Error())` + "\n"
+	str += `			return false` + "\n"
+	str += `		} else {` + "\n"
+	str += `			return true` + "\n"
+	str += `		}` + "\n"
+	str += "\n"
+	str += `	} else {` + "\n"
+	str += `		log.Error().Msgf("Failed to validate EntityID (wrong type: %T)", ifvalue)` + "\n"
+	str += `		return false` + "\n"
+	str += `	}` + "\n"
+	str += `}` + "\n"
+	str += "\n"
+
+	for _, iddef := range ids {
+
+		str += "// ================================ " + iddef.Name + " (" + iddef.FileRelative + ") ================================" + "\n"
+		str += "" + "\n"
+
+		str += "func New" + iddef.Name + "() " + iddef.Name + " {" + "\n"
+		str += "	return " + iddef.Name + "(generateID(prefix" + iddef.Name + "))" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (id " + iddef.Name + ") Valid() error {" + "\n"
+		str += "	return validateID(prefix" + iddef.Name + ", string(id))" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (i " + iddef.Name + ") String() string {" + "\n"
+		str += "	return string(i)" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (i " + iddef.Name + ") Prefix() string {" + "\n"
+		str += "	return prefix" + iddef.Name + "" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (id " + iddef.Name + ") Raw() string {" + "\n"
+		str += "	return getRawData(prefix" + iddef.Name + ", string(id))" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (id " + iddef.Name + ") CheckString() string {" + "\n"
+		str += "	return getCheckString(prefix" + iddef.Name + ", string(id))" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+		str += "func (id " + iddef.Name + ") Regex() rext.Regex {" + "\n"
+		str += "	return regex" + iddef.Name + "" + "\n"
+		str += "}" + "\n"
+
+		str += "" + "\n"
+
+	}
+
+	return str
+}

exerr/builder.go

@@ -275,7 +275,7 @@ func (b *Builder) Any(key string, val any) *Builder {
 }
 
 func (b *Builder) Stringer(key string, val fmt.Stringer) *Builder {
-	if val == nil {
+	if langext.IsNil(val) {
 		return b.addMeta(key, MDTString, "(!nil)")
 	} else {
 		return b.addMeta(key, MDTString, val.String())

exerr/data.go

@@ -56,6 +56,7 @@ var (
 	TypeBindFailHeader   = NewType("BINDFAIL_HEADER", langext.Ptr(400))
 
 	TypeMarshalEntityID = NewType("MARSHAL_ENTITY_ID", langext.Ptr(400))
+	TypeInvalidCSID     = NewType("INVALID_CSID", langext.Ptr(400))
 
 	TypeUnauthorized = NewType("UNAUTHORIZED", langext.Ptr(401))
 	TypeAuthFailed   = NewType("AUTH_FAILED", langext.Ptr(401))

fsext/exists.go

@@ -0,0 +1,36 @@
+package fsext
+
+import "os"
+
+func PathExists(fp string) (bool, error) {
+	_, err := os.Stat(fp)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func FileExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return !stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func DirectoryExists(fp string) (bool, error) {
+	stat, err := os.Stat(fp)
+	if err == nil {
+		return stat.IsDir(), nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.1 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -49,6 +49,8 @@ github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJw
 github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
 github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=

goextVersion.go

@@ -1,5 +1,5 @@
 package goext
 
-const GoextVersion = "0.0.288"
+const GoextVersion = "0.0.293"
 
-const GoextVersionTimestamp = "2023-10-19T14:16:01+0200"
+const GoextVersionTimestamp = "2023-10-30T13:37:31+0100"

tst/must.go

@@ -0,0 +1,21 @@
+package tst
+
+import (
+	"runtime/debug"
+	"testing"
+)
+
+// Must can b used to AssertNoErr of an (T, err) function
+//
+// Usage:
+//
+// input := "123.8"
+// value := tst.Must(strconv.Atoi(input))(t)
+func Must[T any](v T, anerr error) func(t *testing.T) T {
+	return func(t *testing.T) T {
+		if anerr != nil {
+			t.Error("Function returned an error: " + anerr.Error() + "\n" + string(debug.Stack()))
+		}
+		return v
+	}
+}