[Server] Fix FTS search
This commit is contained in:
@@ -119,11 +119,17 @@ func (h APIHandler) ListMessages(pctx ginext.PreContext) ginext.HTTPResponse {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(q.Search) != 0 {
|
if len(q.Search) != 0 {
|
||||||
filter.SearchStringFTS = langext.Ptr(langext.ArrMap(q.Search, func(v string) string { return strings.TrimSpace(v) }))
|
searchTerms := langext.ArrFilter(langext.ArrMap(q.Search, func(v string) string { return strings.TrimSpace(v) }), func(v string) bool { return v != "" })
|
||||||
|
if len(searchTerms) != 0 {
|
||||||
|
filter.SearchStringFTS = langext.Ptr(searchTerms)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(q.StringSearch) != 0 {
|
if len(q.StringSearch) != 0 {
|
||||||
filter.SearchStringPlain = langext.Ptr(langext.ArrMap(q.StringSearch, func(v string) string { return strings.TrimSpace(v) }))
|
searchTerms := langext.ArrFilter(langext.ArrMap(q.StringSearch, func(v string) string { return strings.TrimSpace(v) }), func(v string) bool { return v != "" })
|
||||||
|
if len(searchTerms) != 0 {
|
||||||
|
filter.SearchStringPlain = langext.Ptr(searchTerms)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(q.Channels) != 0 {
|
if len(q.Channels) != 0 {
|
||||||
|
|||||||
@@ -4,13 +4,15 @@ import (
|
|||||||
"crypto/sha512"
|
"crypto/sha512"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"git.blackforestbytes.com/BlackForestBytes/goext/dataext"
|
"git.blackforestbytes.com/BlackForestBytes/goext/dataext"
|
||||||
"git.blackforestbytes.com/BlackForestBytes/goext/langext"
|
"git.blackforestbytes.com/BlackForestBytes/goext/langext"
|
||||||
"git.blackforestbytes.com/BlackForestBytes/goext/mathext"
|
"git.blackforestbytes.com/BlackForestBytes/goext/mathext"
|
||||||
"git.blackforestbytes.com/BlackForestBytes/goext/sq"
|
"git.blackforestbytes.com/BlackForestBytes/goext/sq"
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type MessageFilter struct {
|
type MessageFilter struct {
|
||||||
@@ -219,7 +221,7 @@ func (f MessageFilter) SQL() (string, string, sq.PP, error) {
|
|||||||
if f.SearchStringFTS != nil {
|
if f.SearchStringFTS != nil {
|
||||||
filter := make([]string, 0)
|
filter := make([]string, 0)
|
||||||
for _, v := range *f.SearchStringFTS {
|
for _, v := range *f.SearchStringFTS {
|
||||||
filter = append(filter, fmt.Sprintf("(messages_fts match :%s)", params.Add(v)))
|
filter = append(filter, fmt.Sprintf("(messages_fts match :%s)", params.Add(sanitizeFTSMatchQuery(v))))
|
||||||
}
|
}
|
||||||
sqlClauses = append(sqlClauses, "("+strings.Join(filter, " OR ")+")")
|
sqlClauses = append(sqlClauses, "("+strings.Join(filter, " OR ")+")")
|
||||||
}
|
}
|
||||||
@@ -246,6 +248,43 @@ func (f MessageFilter) SQL() (string, string, sq.PP, error) {
|
|||||||
return sqlClause, joinClause, params, nil
|
return sqlClause, joinClause, params, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sanitizeFTSMatchQuery converts an arbitrary user-provided search string into a valid FTS5 MATCH query.
|
||||||
|
// It preserves explicit "quoted phrases" typed by the user and wraps every other whitespace-separated word
|
||||||
|
// in double-quotes, so that FTS5 special characters ( - : * ^ ( ) AND OR NOT NEAR ... ) inside plain search
|
||||||
|
// terms are treated as literal text instead of query operators.
|
||||||
|
func sanitizeFTSMatchQuery(s string) string {
|
||||||
|
out := make([]string, 0)
|
||||||
|
buf := strings.Builder{}
|
||||||
|
inQuote := false
|
||||||
|
|
||||||
|
flush := func() {
|
||||||
|
if buf.Len() > 0 {
|
||||||
|
out = append(out, `"`+strings.ReplaceAll(buf.String(), `"`, `""`)+`"`)
|
||||||
|
buf.Reset()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range s {
|
||||||
|
switch {
|
||||||
|
case r == '"':
|
||||||
|
// a double-quote opens or closes an explicit phrase; flush the pending word either way
|
||||||
|
flush()
|
||||||
|
inQuote = !inQuote
|
||||||
|
case !inQuote && unicode.IsSpace(r):
|
||||||
|
flush()
|
||||||
|
default:
|
||||||
|
buf.WriteRune(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flush() // trailing word or an unterminated phrase
|
||||||
|
|
||||||
|
if len(out) == 0 {
|
||||||
|
return `""` // valid FTS5 query that simply matches nothing
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Join(out, " ")
|
||||||
|
}
|
||||||
|
|
||||||
func (f MessageFilter) Hash() string {
|
func (f MessageFilter) Hash() string {
|
||||||
bh, err := dataext.StructHash(f, dataext.StructHashOptions{HashAlgo: sha512.New()})
|
bh, err := dataext.StructHash(f, dataext.StructHashOptions{HashAlgo: sha512.New()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -830,6 +830,11 @@ func TestListMessagesSearch(t *testing.T) {
|
|||||||
{"search=the(2)", 17, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("THE"))},
|
{"search=the(2)", 17, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("THE"))},
|
||||||
{"search=please", 9, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("please"))},
|
{"search=please", 9, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("please"))},
|
||||||
{"search=11pm", 2, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("\"11:00pm\""))},
|
{"search=11pm", 2, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("\"11:00pm\""))},
|
||||||
|
|
||||||
|
{"search=hyphen", 0, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("yt-backup"))},
|
||||||
|
{"search=colon", 0, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("foo:bar"))},
|
||||||
|
{"search=NOT", 0, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape("NOT"))},
|
||||||
|
{"search=empty", 22, fmt.Sprintf("/api/v2/messages?search=%s", url.QueryEscape(""))},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, testdata := range filterTests {
|
for _, testdata := range filterTests {
|
||||||
|
|||||||
Reference in New Issue
Block a user