From 3f60764bd4abd92bfc34a90d4929e67db8497d80 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 10 Aug 2025 17:39:23 +0100 Subject: [PATCH] azureblob: fix deadlock with --max-connections with InvalidBlockOrBlob errors Before this change the azureblob backend could deadlock when using --max-connections. This is because when it receives InvalidBlockOrBlob error it attempts to clear the condition before retrying. This in turn involved recursively calling the pacer. At this point the pacer can easily have no connections left which causes a deadlock as all the other pacer connections are waiting for the InvalidBlockOrBlob to be resolved. This fixes the problem by using a temporary pacer when resolving the InvalidBlockOrBlob errors. --- backend/azureblob/azureblob.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 0ef8aaa34..2f7c8c927 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -2757,6 +2757,8 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) { blockList blockblob.GetBlockListResponse properties *blob.GetPropertiesResponse options *blockblob.CommitBlockListOptions + // Use temporary pacer as this can be called recursively which can cause a deadlock with --max-connections + pacer = fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))) ) properties, err = o.readMetaDataAlways(ctx) @@ -2768,7 +2770,7 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) { if objectExists { // Get the committed block list - err = o.fs.pacer.Call(func() (bool, error) { + err = pacer.Call(func() (bool, error) { blockList, err = blockBlobSVC.GetBlockList(ctx, blockblob.BlockListTypeAll, nil) return o.fs.shouldRetry(ctx, err) }) @@ -2810,7 +2812,7 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) { // Commit only the committed blocks fs.Debugf(o, "Committing %d blocks to remove uncommitted blocks", len(blockIDs)) - err = o.fs.pacer.Call(func() (bool, error) { + err = pacer.Call(func() (bool, error) { _, err := blockBlobSVC.CommitBlockList(ctx, blockIDs, options) return o.fs.shouldRetry(ctx, err) })