remove ginext/mongoext (no-dep lib)

dataext/bufferedReadCloser.go

@@ -0,0 +1,56 @@
+package dataext
+
+import "io"
+
+type BufferedReadCloser interface {
+	io.ReadCloser
+	BufferedAll() ([]byte, error)
+}
+
+type bufferedReadCloser struct {
+	buffer   []byte
+	inner    io.ReadCloser
+	finished bool
+}
+
+func (b *bufferedReadCloser) Read(p []byte) (int, error) {
+
+	n, err := b.inner.Read(p)
+	if n > 0 {
+		b.buffer = append(b.buffer, p[0:n]...)
+	}
+
+	if err == io.EOF {
+		b.finished = true
+	}
+
+	return n, err
+}
+
+func NewBufferedReadCloser(sub io.ReadCloser) BufferedReadCloser {
+	return &bufferedReadCloser{
+		buffer:   make([]byte, 0, 1024),
+		inner:    sub,
+		finished: false,
+	}
+}
+
+func (b *bufferedReadCloser) Close() error {
+	err := b.inner.Close()
+	if err != nil {
+		b.finished = true
+	}
+	return err
+}
+
+func (b *bufferedReadCloser) BufferedAll() ([]byte, error) {
+	arr := make([]byte, 1024)
+	for !b.finished {
+		_, err := b.Read(arr)
+		if err != nil && err != io.EOF {
+			return nil, err
+		}
+	}
+
+	return b.buffer, nil
+}

dataext/lruMap.go

@@ -0,0 +1,151 @@
+package dataext
+
+import (
+	"sync"
+)
+
+//
+// This is an LRU (Least-Recently-Used) cache based on a double linked list
+// All the work we do below is to have a cache where we can easily remove the least-used element
+// (to ensure that the cache is never bigger than maxsize items)
+//
+// The cache algorithm the following properties:
+//   - Memory footprint is O(n), with neglible overhead
+//   - The cache is multi-threading safe (sync.Mutex)
+//   - Inserts     are O(1)
+//   - Gets        are O(1)
+//   - Re-Shuffles are O(1)   (= an element is removed on Insert because teh cache was full)
+//
+// There are also a bunch of unit tests to ensure that the cache is always in a consistent state
+//
+
+type LRUData interface{}
+
+type LRUMap struct {
+	maxsize int
+	lock    sync.Mutex
+
+	cache map[string]*cacheNode
+
+	lfuHead *cacheNode
+	lfuTail *cacheNode
+}
+
+type cacheNode struct {
+	key    string
+	data   LRUData
+	parent *cacheNode
+	child  *cacheNode
+}
+
+func NewLRUMap(size int) *LRUMap {
+	if size <= 2 && size != 0 {
+		panic("Size must be > 2  (or 0)")
+	}
+
+	return &LRUMap{
+		maxsize: size,
+		lock:    sync.Mutex{},
+		cache:   make(map[string]*cacheNode, size+1),
+		lfuHead: nil,
+		lfuTail: nil,
+	}
+}
+
+func (c *LRUMap) Put(key string, value LRUData) {
+	if c.maxsize == 0 {
+		return // cache disabled
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	node, exists := c.cache[key]
+
+	if exists {
+		// key already in data: only update LFU and value
+		c.moveNodeToTop(node)
+		node.data = value
+		return
+	}
+
+	// key does not exist: insert into map and add to top of LFU
+	node = &cacheNode{
+		key:    key,
+		data:   value,
+		parent: nil,
+		child:  c.lfuHead,
+	}
+	if c.lfuHead == nil && c.lfuTail == nil { // special case - previously the cache was empty (head == tail == nil)
+		c.lfuHead = node
+		c.lfuTail = node
+	} else {
+		c.lfuHead = node
+		node.child.parent = node
+	}
+	c.cache[key] = node
+
+	if len(c.cache) > c.maxsize { // maxsize is always > 2
+		tail := c.lfuTail
+		delete(c.cache, tail.key)
+		c.lfuTail = tail.parent
+		c.lfuTail.child = nil
+		tail.parent = nil
+		tail.child = nil
+	}
+}
+
+func (c *LRUMap) TryGet(key string) (LRUData, bool) {
+	if c.maxsize == 0 {
+		return nil, false // cache disabled
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	val, ok := c.cache[key]
+	if !ok {
+		return nil, false
+	}
+	c.moveNodeToTop(val)
+	return val.data, ok
+}
+
+func (c *LRUMap) moveNodeToTop(node *cacheNode) {
+	// (only called in critical section !)
+
+	if c.lfuHead == node { // fast case
+		return
+	}
+
+	// Step 1 unlink
+
+	parent := node.parent
+	child := node.child
+	if parent != nil {
+		parent.child = child
+	}
+	if child != nil {
+		child.parent = parent
+	}
+	if node == c.lfuHead {
+		c.lfuHead = node.child
+	}
+	if node == c.lfuTail {
+		c.lfuTail = node.parent
+	}
+
+	// Step 2 re-insert at top
+	node.parent = nil
+	node.child = c.lfuHead
+	c.lfuHead = node
+	if node.child != nil {
+		node.child.parent = node
+	}
+}
+
+func (c *LRUMap) Size() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return len(c.cache)
+}

dataext/lruMap_test.go

@@ -0,0 +1,269 @@
+package dataext
+
+import (
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"math/rand"
+	"strconv"
+	"testing"
+)
+
+func init() {
+	rand.Seed(0)
+}
+
+func TestResultCache1(t *testing.T) {
+	cache := NewLRUMap(8)
+	verifyLRUList(cache, t)
+
+	key := randomKey()
+	val := randomVal()
+
+	if cache.Size() != 0 {
+		t.Errorf("cache size expected == 0, actual == %v", cache.Size())
+	}
+
+	if _, ok := cache.TryGet(key); ok {
+		t.Errorf("empty cache TryGet returned value")
+	}
+	verifyLRUList(cache, t)
+
+	cache.Put(key, val)
+	verifyLRUList(cache, t)
+
+	if cache.Size() != 1 {
+		t.Errorf("cache size expected == 1, actual == %v", cache.Size())
+	}
+
+	cacheval, ok := cache.TryGet(key)
+	verifyLRUList(cache, t)
+	if !ok {
+		t.Errorf("cache TryGet returned no value")
+	}
+	if !eq(cacheval, val) {
+		t.Errorf("cache TryGet returned different value (%+v <> %+v)", cacheval, val)
+	}
+
+	if _, ok := cache.TryGet(randomKey()); ok {
+		t.Errorf("cache TryGet returned a value for non-existant key")
+	}
+	verifyLRUList(cache, t)
+}
+
+func TestResultCache2(t *testing.T) {
+	cache := NewLRUMap(8)
+	verifyLRUList(cache, t)
+
+	key1 := "key1"
+	val1 := randomVal()
+	cache.Put(key1, val1)
+	verifyLRUList(cache, t)
+
+	key2 := "key2"
+	val2 := randomVal()
+	cache.Put(key2, val2)
+	verifyLRUList(cache, t)
+
+	key3 := "key3"
+	val3 := randomVal()
+	cache.Put(key3, val3)
+	verifyLRUList(cache, t)
+
+	key4 := "key4"
+	val4 := randomVal()
+	cache.Put(key4, val4)
+	verifyLRUList(cache, t)
+
+	if _, ok := cache.TryGet(key1); !ok {
+		t.Errorf("cache TryGet returned no value")
+	}
+	verifyLRUList(cache, t)
+	if _, ok := cache.TryGet(key2); !ok {
+		t.Errorf("cache TryGet returned no value")
+	}
+	verifyLRUList(cache, t)
+	if _, ok := cache.TryGet(key3); !ok {
+		t.Errorf("cache TryGet returned no value")
+	}
+	verifyLRUList(cache, t)
+	if _, ok := cache.TryGet(key4); !ok {
+		t.Errorf("cache TryGet returned no value")
+	}
+	verifyLRUList(cache, t)
+	if _, ok := cache.TryGet(randomKey()); ok {
+		t.Errorf("cache TryGet returned a value for non-existant key")
+	}
+	verifyLRUList(cache, t)
+	if cache.Size() != 4 {
+		t.Errorf("cache size expected == 4, actual == %v", cache.Size())
+	}
+	verifyLRUList(cache, t)
+
+	cache.Put(key4, val4) // same key again
+	verifyLRUList(cache, t)
+
+	if cache.Size() != 4 {
+		t.Errorf("cache size expected == 4, actual == %v", cache.Size())
+	}
+
+	cache.Put(randomKey(), randomVal())
+	verifyLRUList(cache, t)
+	cache.Put(randomKey(), randomVal())
+	verifyLRUList(cache, t)
+	cache.Put(randomKey(), randomVal())
+	verifyLRUList(cache, t)
+	cache.Put(randomKey(), randomVal())
+	verifyLRUList(cache, t)
+
+	if cache.Size() != 8 {
+		t.Errorf("cache size expected == 8, actual == %v", cache.Size())
+	}
+
+	cache.Put(randomKey(), randomVal()) // drops key1
+	verifyLRUList(cache, t)
+
+	if cache.Size() != 8 {
+		t.Errorf("cache size expected == 8, actual == %v", cache.Size())
+	}
+
+	if _, ok := cache.TryGet(key1); ok {
+		t.Errorf("[key1] should be dropped from cache")
+	}
+	verifyLRUList(cache, t)
+	if _, ok := cache.TryGet(key2); !ok { // moves key2 to most-recently used
+		t.Errorf("[key2] should still be in cache")
+	}
+	verifyLRUList(cache, t)
+
+	cache.Put(randomKey(), randomVal()) // drops key3
+	verifyLRUList(cache, t)
+
+	if cache.Size() != 8 {
+		t.Errorf("cache size expected == 8, actual == %v", cache.Size())
+	}
+
+	if _, ok := cache.TryGet(key3); ok {
+		t.Errorf("[key3] should be dropped from cache")
+	}
+	if _, ok := cache.TryGet(key2); !ok {
+		t.Errorf("[key2] should still be in cache")
+	}
+}
+
+func TestResultCache3(t *testing.T) {
+	cache := NewLRUMap(8)
+	verifyLRUList(cache, t)
+
+	key1 := "key1"
+	val1 := randomVal()
+	val2 := randomVal()
+
+	cache.Put(key1, val1)
+	verifyLRUList(cache, t)
+
+	if val, ok := cache.TryGet(key1); !ok || !eq(val, val1) {
+		t.Errorf("Value in cache should be [val1]")
+	}
+
+	cache.Put(key1, val2)
+	verifyLRUList(cache, t)
+
+	if val, ok := cache.TryGet(key1); !ok || !eq(val, val2) {
+		t.Errorf("Value in cache should be [val2]")
+	}
+}
+
+// does a basic consistency check over the internal cache representation
+func verifyLRUList(cache *LRUMap, t *testing.T) {
+	size := 0
+
+	tailFound := false
+	headFound := false
+	curr := cache.lfuHead
+	for curr != nil {
+		size++
+
+		if curr.parent == nil {
+			headFound = true
+			if curr != cache.lfuHead {
+				t.Errorf("head != lfuHead")
+				return
+			}
+		}
+
+		if curr.child == nil {
+			tailFound = true
+			if curr != cache.lfuTail {
+				t.Errorf("tail != lfuTail")
+				return
+			}
+		}
+
+		if curr.child != nil {
+			if curr.child.parent != curr {
+				t.Errorf("error in child <-> parent link")
+				return
+			}
+		}
+
+		if curr.parent != nil {
+			if curr.parent.child != curr {
+				t.Errorf("error in parent <-> child link")
+				return
+			}
+		}
+
+		curr = curr.child
+	}
+
+	if cache.Size() > 0 && cache.lfuHead == nil {
+		t.Errorf("no head in cache")
+	}
+	if cache.Size() > 0 && cache.lfuTail == nil {
+		t.Errorf("no tail in cache")
+	}
+
+	if cache.Size() == 0 && cache.lfuHead != nil {
+		t.Errorf("dangling head in cache")
+	}
+	if cache.Size() == 0 && cache.lfuTail != nil {
+		t.Errorf("dangling tail in cache")
+	}
+
+	if cache.Size() > 0 && !headFound {
+		t.Errorf("head not found")
+	}
+
+	if cache.Size() > 0 && !tailFound {
+		t.Errorf("tail not found")
+	}
+
+	if size != cache.Size() {
+		t.Errorf("error size mismatch (%v <> %v)", size, cache.Size())
+	}
+
+	if cache.Size() > cache.maxsize {
+		t.Errorf("too many items: %v", cache.Size())
+	}
+}
+
+func randomKey() string {
+	return strconv.FormatInt(rand.Int63(), 16)
+}
+
+func randomVal() LRUData {
+	v := primitive.NewObjectID()
+	return &v
+}
+
+func eq(a LRUData, b LRUData) bool {
+	v1, ok1 := a.(*primitive.ObjectID)
+	v2, ok2 := b.(*primitive.ObjectID)
+	if ok1 && ok2 {
+		if v1 == nil || v2 == nil {
+			return false
+		}
+		return v1.Hex() == v2.Hex()
+	}
+
+	return false
+}

dataext/syncSet.go

@@ -0,0 +1,65 @@
+package dataext
+
+import "sync"
+
+type SyncStringSet struct {
+	data map[string]bool
+	lock sync.Mutex
+}
+
+func (s *SyncStringSet) Add(value string) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[string]bool)
+	}
+
+	_, ok := s.data[value]
+	s.data[value] = true
+
+	return !ok
+}
+
+func (s *SyncStringSet) AddAll(values []string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[string]bool)
+	}
+
+	for _, value := range values {
+		s.data[value] = true
+	}
+}
+
+func (s *SyncStringSet) Contains(value string) bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[string]bool)
+	}
+
+	_, ok := s.data[value]
+
+	return ok
+}
+
+func (s *SyncStringSet) Get() []string {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.data == nil {
+		s.data = make(map[string]bool)
+	}
+
+	r := make([]string, 0, len(s.data))
+
+	for k := range s.data {
+		r = append(r, k)
+	}
+
+	return r
+}