Mirrors/minio
1
0
Fork 1
mirror of https://github.com/pgsty/minio.git synced 2026-03-16 01:26:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

add backups for usage-caches to rely on upon error (#18029)

Browse Source 
This allows scanner to avoid lengthy scans, skip
things appropriately and also not lose metrics in
any manner.

reduce longer deadlines for usage-cache loads/saves
to match the disk timeout which is 2minutes now per
IOP.
This commit is contained in:
Harshavardhana
2023-09-14 11:53:52 -07:00
committed by GitHub
parent 822cbd4b43
commit a2aabfabd9
3 changed files with 87 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/data-usage.go
View File

@@ -42,6 +42,7 @@ const (
// storeDataUsageInBackend will store all objects sent on the gui channel until closed.
func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
attempts := 1
for dataUsageInfo := range dui {
json := jsoniter.ConfigCompatibleWithStandardLibrary
dataUsageJSON, err := json.Marshal(dataUsageInfo)
@@ -49,9 +50,14 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
logger.LogIf(ctx, err)
continue
}
if attempts > 10 {
saveConfig(ctx, objAPI, dataUsageObjNamePath+".bkp", dataUsageJSON) // Save a backup every 10th update.
attempts = 1
}
if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
logger.LogIf(ctx, err)
}
attempts++
}
}
@@ -94,10 +100,13 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath)
if err != nil {
if errors.Is(err, errConfigNotFound) {
return DataUsageInfo{}, nil
buf, err = readConfig(ctx, objAPI, dataUsageObjNamePath+".bkp")
if err != nil {
if errors.Is(err, errConfigNotFound) {
return DataUsageInfo{}, nil
}
return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
}
return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
}
var dataUsageInfo DataUsageInfo