Webhook targets refactor and bug fixes (#19275)

- old version was unable to retain messages during config reload
- old version could not go from memory to disk during reload
- new version can batch disk queue entries to single for to reduce I/O load
- error logging has been improved, previous version would miss certain errors.
- logic for spawning/despawning additional workers has been adjusted to trigger when half capacity is reached, instead of when the log queue becomes full.
- old version would json marshall x2 and unmarshal 1x for every log item. Now we only do marshal x1 and then we GetRaw from the store and send it without having to re-marshal.

internal/store/queuestore.go

@@ -28,6 +28,8 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/valyala/bytebufferpool"
 )
 
 const (
@@ -84,6 +86,7 @@ func (store *QueueStore[_]) Open() error {
 	if uint64(len(files)) > store.entryLimit {
 		files = files[:store.entryLimit]
 	}
+
 	for _, file := range files {
 		if file.IsDir() {
 			continue
@@ -97,6 +100,54 @@ func (store *QueueStore[_]) Open() error {
 	return nil
 }
 
+// Delete - Remove the store directory from disk
+func (store *QueueStore[_]) Delete() error {
+	return os.Remove(store.directory)
+}
+
+// PutMultiple - puts an item to the store.
+func (store *QueueStore[I]) PutMultiple(item []I) error {
+	store.Lock()
+	defer store.Unlock()
+	if uint64(len(store.entries)) >= store.entryLimit {
+		return errLimitExceeded
+	}
+	// Generate a new UUID for the key.
+	key, err := uuid.NewRandom()
+	if err != nil {
+		return err
+	}
+	return store.multiWrite(key.String(), item)
+}
+
+// multiWrite - writes an item to the directory.
+func (store *QueueStore[I]) multiWrite(key string, item []I) error {
+	buf := bytebufferpool.Get()
+	defer bytebufferpool.Put(buf)
+
+	enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
+
+	for i := range item {
+		err := enc.Encode(item[i])
+		if err != nil {
+			return err
+		}
+	}
+	b := buf.Bytes()
+
+	path := filepath.Join(store.directory, key+store.fileExt)
+	err := os.WriteFile(path, b, os.FileMode(0o770))
+	buf.Reset()
+	if err != nil {
+		return err
+	}
+
+	// Increment the item count.
+	store.entries[key] = time.Now().UnixNano()
+
+	return nil
+}
+
 // write - writes an item to the directory.
 func (store *QueueStore[I]) write(key string, item I) error {
 	// Marshalls the item.
@@ -131,6 +182,30 @@ func (store *QueueStore[I]) Put(item I) error {
 	return store.write(key.String(), item)
 }
 
+// GetRaw - gets an item from the store.
+func (store *QueueStore[I]) GetRaw(key string) (raw []byte, err error) {
+	store.RLock()
+
+	defer func(store *QueueStore[I]) {
+		store.RUnlock()
+		if err != nil {
+			// Upon error we remove the entry.
+			store.Del(key)
+		}
+	}(store)
+
+	raw, err = os.ReadFile(filepath.Join(store.directory, key+store.fileExt))
+	if err != nil {
+		return
+	}
+
+	if len(raw) == 0 {
+		return raw, os.ErrNotExist
+	}
+
+	return
+}
+
 // Get - gets an item from the store.
 func (store *QueueStore[I]) Get(key string) (item I, err error) {
 	store.RLock()

internal/store/store.go

@@ -25,7 +25,6 @@ import (
 	"time"
 
 	xioutil "github.com/minio/minio/internal/ioutil"
-	xnet "github.com/minio/pkg/v2/net"
 )
 
 const (
@@ -46,12 +45,15 @@ type Target interface {
 // Store - Used to persist items.
 type Store[I any] interface {
 	Put(item I) error
+	PutMultiple(item []I) error
 	Get(key string) (I, error)
+	GetRaw(key string) ([]byte, error)
 	Len() int
 	List() ([]string, error)
 	Del(key string) error
 	DelList(key []string) error
 	Open() error
+	Delete() error
 	Extension() string
 }
 
@@ -110,15 +112,14 @@ func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger l
 				break
 			}
 
-			if err != ErrNotConnected && !xnet.IsConnResetErr(err) {
-				logger(context.Background(),
-					fmt.Errorf("target.SendFromStore() failed with '%w'", err),
-					target.Name())
-			}
-
-			// Retrying after 3secs back-off
+			logger(
+				context.Background(),
+				fmt.Errorf("unable to send webhook log entry to '%s' err '%w'", target.Name(), err),
+				target.Name(),
+			)
 
 			select {
+			// Retrying after 3secs back-off
 			case <-retryTicker.C:
 			case <-doneCh:
 				return false
@@ -131,7 +132,6 @@ func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger l
 		select {
 		case key, ok := <-keyCh:
 			if !ok {
-				// closed channel.
 				return
 			}
 
@@ -147,9 +147,7 @@ func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger l
 // StreamItems reads the keys from the store and replays the corresponding item to the target.
 func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, logger logger) {
 	go func() {
-		// Replays the items from the store.
 		keyCh := replayItems(store, doneCh, logger, target.Name())
-		// Send items from the store.
 		sendItems(target, keyCh, doneCh, logger)
 	}()
 }