Reduce big message RPC allocations (#19390)

Use `ODirectPoolSmall` buffers for inline data in PutObject.

Add a separate call for inline data that will fetch a buffer for the inline data before unmarshal.

internal/grid/connection.go

@@ -550,10 +550,9 @@ func (c *Connection) queueMsg(msg message, payload sender) error {
 	// This cannot encode subroute.
 	msg.Flags.Clear(FlagSubroute)
 	if payload != nil {
-		if cap(msg.Payload) < payload.Msgsize() {
-			old := msg.Payload
-			msg.Payload = GetByteBuffer()[:0]
-			PutByteBuffer(old)
+		if sz := payload.Msgsize(); cap(msg.Payload) < sz {
+			PutByteBuffer(msg.Payload)
+			msg.Payload = GetByteBufferCap(sz)
 		}
 		var err error
 		msg.Payload, err = payload.MarshalMsg(msg.Payload[:0])
@@ -563,7 +562,7 @@ func (c *Connection) queueMsg(msg message, payload sender) error {
 		}
 	}
 	defer PutByteBuffer(msg.Payload)
-	dst := GetByteBuffer()[:0]
+	dst := GetByteBufferCap(msg.Msgsize())
 	dst, err := msg.MarshalMsg(dst)
 	if err != nil {
 		return err
@@ -578,9 +577,9 @@ func (c *Connection) queueMsg(msg message, payload sender) error {
 // sendMsg will send
 func (c *Connection) sendMsg(conn net.Conn, msg message, payload msgp.MarshalSizer) error {
 	if payload != nil {
-		if cap(msg.Payload) < payload.Msgsize() {
+		if sz := payload.Msgsize(); cap(msg.Payload) < sz {
 			PutByteBuffer(msg.Payload)
-			msg.Payload = GetByteBuffer()[:0]
+			msg.Payload = GetByteBufferCap(sz)[:0]
 		}
 		var err error
 		msg.Payload, err = payload.MarshalMsg(msg.Payload)
@@ -589,7 +588,7 @@ func (c *Connection) sendMsg(conn net.Conn, msg message, payload msgp.MarshalSiz
 		}
 		defer PutByteBuffer(msg.Payload)
 	}
-	dst := GetByteBuffer()[:0]
+	dst := GetByteBufferCap(msg.Msgsize())
 	dst, err := msg.MarshalMsg(dst)
 	if err != nil {
 		return err

internal/grid/grid.go

@@ -42,7 +42,13 @@ const (
 
 	// maxBufferSize is the maximum buffer size.
 	// Buffers larger than this is not reused.
-	maxBufferSize = 64 << 10
+	maxBufferSize = 96 << 10
+
+	// This is the assumed size of bigger buffers and allocation size.
+	biggerBufMin = 32 << 10
+
+	// This is the maximum size of bigger buffers.
+	biggerBufMax = maxBufferSize
 
 	// If there is a queue, merge up to this many messages.
 	maxMergeMessages = 30
@@ -63,6 +69,13 @@ var internalByteBuffer = sync.Pool{
 	},
 }
 
+var internal32KByteBuffer = sync.Pool{
+	New: func() any {
+		m := make([]byte, 0, biggerBufMin)
+		return &m
+	},
+}
+
 // GetByteBuffer can be replaced with a function that returns a small
 // byte buffer.
 // When replacing PutByteBuffer should also be replaced
@@ -72,10 +85,27 @@ var GetByteBuffer = func() []byte {
 	return b[:0]
 }
 
+// GetByteBufferCap returns a length 0 byte buffer with at least the given capacity.
+func GetByteBufferCap(wantSz int) []byte {
+	switch {
+	case wantSz <= defaultBufferSize:
+		return GetByteBuffer()[:0]
+	case wantSz <= maxBufferSize:
+		b := *internal32KByteBuffer.Get().(*[]byte)
+		return b[:0]
+	}
+	return make([]byte, 0, wantSz)
+}
+
 // PutByteBuffer is for returning byte buffers.
 var PutByteBuffer = func(b []byte) {
-	if cap(b) >= minBufferSize && cap(b) < maxBufferSize {
+	if cap(b) >= biggerBufMin && cap(b) < biggerBufMax {
+		internal32KByteBuffer.Put(&b)
+		return
+	}
+	if cap(b) >= minBufferSize && cap(b) < biggerBufMin {
 		internalByteBuffer.Put(&b)
+		return
 	}
 }
 
@@ -117,11 +147,7 @@ type writerWrapper struct {
 }
 
 func (w *writerWrapper) Write(p []byte) (n int, err error) {
-	buf := GetByteBuffer()
-	if cap(buf) < len(p) {
-		PutByteBuffer(buf)
-		buf = make([]byte, len(p))
-	}
+	buf := GetByteBufferCap(len(p))
 	buf = buf[:len(p)]
 	copy(buf, p)
 	select {

internal/grid/handlers.go

@@ -111,6 +111,7 @@ const (
 	HandlerGetBandwidth
 	HandlerWriteAll
 	HandlerListBuckets
+	HandlerRenameDataInline
 
 	// Add more above here ^^^
 	// If all handlers are used, the type of Handler can be changed.
@@ -546,7 +547,7 @@ func (h *SingleHandler[Req, Resp]) Call(ctx context.Context, c Requester, req Re
 		}
 		return resp, ErrDisconnected
 	}
-	payload, err := req.MarshalMsg(GetByteBuffer()[:0])
+	payload, err := req.MarshalMsg(GetByteBufferCap(req.Msgsize()))
 	if err != nil {
 		return resp, err
 	}
@@ -788,8 +789,7 @@ func (h *StreamTypeHandler[Payload, Req, Resp]) register(m *Manager, handle func
 					if dropOutput {
 						continue
 					}
-					dst := GetByteBuffer()
-					dst, err := v.MarshalMsg(dst[:0])
+					dst, err := v.MarshalMsg(GetByteBufferCap(v.Msgsize()))
 					if err != nil {
 						logger.LogOnceIf(ctx, err, err.Error())
 					}
@@ -853,7 +853,7 @@ func (h *StreamTypeHandler[Payload, Req, Resp]) Call(ctx context.Context, c Stre
 	var payloadB []byte
 	if h.WithPayload {
 		var err error
-		payloadB, err = payload.MarshalMsg(GetByteBuffer()[:0])
+		payloadB, err = payload.MarshalMsg(GetByteBufferCap(payload.Msgsize()))
 		if err != nil {
 			return nil, err
 		}
@@ -875,7 +875,7 @@ func (h *StreamTypeHandler[Payload, Req, Resp]) Call(ctx context.Context, c Stre
 		go func() {
 			defer xioutil.SafeClose(stream.Requests)
 			for req := range reqT {
-				b, err := req.MarshalMsg(GetByteBuffer()[:0])
+				b, err := req.MarshalMsg(GetByteBufferCap(req.Msgsize()))
 				if err != nil {
 					logger.LogOnceIf(ctx, err, err.Error())
 				}

internal/grid/handlers_string.go

@@ -80,14 +80,15 @@ func _() {
 	_ = x[HandlerGetBandwidth-69]
 	_ = x[HandlerWriteAll-70]
 	_ = x[HandlerListBuckets-71]
-	_ = x[handlerTest-72]
-	_ = x[handlerTest2-73]
-	_ = x[handlerLast-74]
+	_ = x[HandlerRenameDataInline-72]
+	_ = x[handlerTest-73]
+	_ = x[handlerTest2-74]
+	_ = x[handlerLast-75]
 }
 
-const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketshandlerTesthandlerTest2handlerLast"
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlinehandlerTesthandlerTest2handlerLast"
 
-var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 899, 911, 922}
+var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 927, 938}
 
 func (i HandlerID) String() string {
 	if i >= HandlerID(len(_HandlerID_index)-1) {

internal/grid/muxclient.go

@@ -145,7 +145,7 @@ func (m *muxClient) send(msg message) error {
 // sendLocked the message. msg.Seq and msg.MuxID will be set.
 // m.respMu must be held.
 func (m *muxClient) sendLocked(msg message) error {
-	dst := GetByteBuffer()[:0]
+	dst := GetByteBufferCap(msg.Msgsize())
 	msg.Seq = m.SendSeq
 	msg.MuxID = m.MuxID
 	msg.Flags |= m.BaseFlags

internal/grid/types.go

@@ -189,6 +189,12 @@ func NewBytes() *Bytes {
 	return &b
 }
 
+// NewBytesCap returns an empty Bytes with the given capacity.
+func NewBytesCap(size int) *Bytes {
+	b := Bytes(GetByteBufferCap(size))
+	return &b
+}
+
 // NewBytesWith returns a new Bytes with the provided content.
 // When sent as a parameter, the caller gives up ownership of the byte slice.
 // When returned as response, the handler also gives up ownership of the byte slice.
@@ -203,14 +209,9 @@ func NewBytesWithCopyOf(b []byte) *Bytes {
 		bb := Bytes(nil)
 		return &bb
 	}
-	if len(b) < maxBufferSize {
-		bb := NewBytes()
-		*bb = append(*bb, b...)
-		return bb
-	}
-	bb := Bytes(make([]byte, len(b)))
-	copy(bb, b)
-	return &bb
+	bb := NewBytesCap(len(b))
+	*bb = append(*bb, b...)
+	return bb
 }
 
 // Bytes provides a byte slice that can be serialized.
@@ -238,7 +239,7 @@ func (b *Bytes) UnmarshalMsg(bytes []byte) ([]byte, error) {
 		copy(*b, val)
 	} else {
 		if cap(*b) == 0 && len(val) <= maxBufferSize {
-			*b = GetByteBuffer()[:0]
+			*b = GetByteBufferCap(len(val))
 		} else {
 			PutByteBuffer(*b)
 			*b = make([]byte, 0, len(val))