Introduce simpler GetMultipartInfo call for performance (#9722)

Advantages avoids 100's of stats which are needed for each
upload operation in FS/NAS gateway mode when uploading a large
multipart object, dramatically increases performance for
multipart uploads by avoiding recursive calls.

For other gateway's simplifies the approach since
azure, gcs, hdfs gateway's don't capture any specific
metadata during upload which needs handler validation
for encryption/compression.

Erasure coding was already optimized, additionally
just avoids small allocations of large data structure.

Fixes #7206
This commit is contained in:
Harshavardhana
2020-05-28 12:36:20 -07:00
committed by GitHub
parent 231c5cf6de
commit b330c2c57e
14 changed files with 292 additions and 71 deletions

View File

@@ -350,6 +350,50 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
}, nil
}
// GetMultipartInfo returns multipart metadata uploaded during newMultipartUpload, used
// by callers to verify object states
// - encrypted
// - compressed
func (fs *FSObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
minfo := MultipartInfo{
Bucket: bucket,
Object: object,
UploadID: uploadID,
}
if err := checkListPartsArgs(ctx, bucket, object, fs); err != nil {
return minfo, toObjectErr(err)
}
// Check if bucket exists
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
return minfo, toObjectErr(err, bucket)
}
uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
if _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)); err != nil {
if err == errFileNotFound || err == errFileAccessDenied {
return minfo, InvalidUploadID{UploadID: uploadID}
}
return minfo, toObjectErr(err, bucket, object)
}
fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
if err != nil {
logger.LogIf(ctx, err)
return minfo, toObjectErr(err, bucket, object)
}
var fsMeta fsMetaV1
var json = jsoniter.ConfigCompatibleWithStandardLibrary
if err = json.Unmarshal(fsMetaBytes, &fsMeta); err != nil {
return minfo, toObjectErr(err, bucket, object)
}
minfo.UserDefined = fsMeta.Meta
return minfo, nil
}
// ListObjectParts - lists all previously uploaded parts for a given
// object and uploadID. Takes additional input of part-number-marker
// to indicate where the listing should begin from.