Improve expiration of tiered objects (#18926)

- Use a shared worker pool for all ILM expiry tasks
- Free version cleanup executes in a separate goroutine
- Add a free version only if removing the remote object fails
- Add ILM expiry metrics to the node namespace
- Move tier journal tasks to expiryState
- Remove unused on-disk journal for tiered objects pending deletion
- Distribute expiry tasks across workers such that the expiry of versions of
  the same object serialized
- Ability to resize worker pool without server restart
- Make scaling down of expiryState workers' concurrency safe; Thanks
  @klauspost
- Add error logs when expiryState and transition state are not
  initialized (yet)
* metrics: Add missed tier journal entry tasks
* Initialize the ILM worker pool after the object layer

cmd/handler-api.go

@@ -47,6 +47,7 @@ type apiConfig struct {
 	replicationPriority   string
 	replicationMaxWorkers int
 	transitionWorkers     int
+	expiryWorkers         int
 
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
@@ -170,7 +171,9 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	}
 	t.replicationPriority = cfg.ReplicationPriority
 	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
-	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
+
+	// N B api.transition_workers will be deprecated
+	if globalTransitionState != nil {
 		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
 	}
 	t.transitionWorkers = cfg.TransitionWorkers
@@ -365,6 +368,13 @@ func (t *apiConfig) getReplicationOpts() replicationPoolOpts {
 	}
 }
 
+func (t *apiConfig) getExpiryWorkers() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.expiryWorkers
+}
+
 func (t *apiConfig) getTransitionWorkers() int {
 	t.mu.RLock()
 	defer t.mu.RUnlock()