Add lock overload protection (#20876)

Reject new lock requests immediately when 1000 goroutines are queued 
for the local lock mutex.

We do not reject unlocking, refreshing, or maintenance; they add to the count.

The limit is set to allow for bursty behavior but prevent requests from 
overloading the server completely.
This commit is contained in:
Klaus Post
2025-01-31 11:54:34 -08:00
committed by GitHub
parent abb385af41
commit bdb3db6dad
4 changed files with 290 additions and 122 deletions

View File

@@ -33,8 +33,12 @@ type lockRESTServer struct {
// RefreshHandler - refresh the current lock
func (l *lockRESTServer) RefreshHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) {
// Add a timeout similar to what we expect upstream.
ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.RefreshCall)
defer cancel()
resp := lockRPCRefresh.NewResponse()
refreshed, err := l.ll.Refresh(context.Background(), *args)
refreshed, err := l.ll.Refresh(ctx, *args)
if err != nil {
return l.makeResp(resp, err)
}
@@ -46,8 +50,11 @@ func (l *lockRESTServer) RefreshHandler(args *dsync.LockArgs) (*dsync.LockResp,
// LockHandler - Acquires a lock.
func (l *lockRESTServer) LockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) {
// Add a timeout similar to what we expect upstream.
ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.Acquire)
defer cancel()
resp := lockRPCLock.NewResponse()
success, err := l.ll.Lock(context.Background(), *args)
success, err := l.ll.Lock(ctx, *args)
if err == nil && !success {
return l.makeResp(resp, errLockConflict)
}
@@ -65,8 +72,11 @@ func (l *lockRESTServer) UnlockHandler(args *dsync.LockArgs) (*dsync.LockResp, *
// RLockHandler - Acquires an RLock.
func (l *lockRESTServer) RLockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) {
// Add a timeout similar to what we expect upstream.
ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.Acquire)
defer cancel()
resp := lockRPCRLock.NewResponse()
success, err := l.ll.RLock(context.Background(), *args)
success, err := l.ll.RLock(ctx, *args)
if err == nil && !success {
err = errLockConflict
}