Fix pipeline cancellation status handling and step state synchronization (#6011)

Co-authored-by: pnkcaht <samzoovsk19@gmail.com>
Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com>
Co-authored-by: Lauris B <lauris@nix.lv>

server/queue/fifo.go

@@ -157,6 +157,7 @@ func (q *fifo) finished(ids []string, exitStatus model.StatusValue, err error) e
 }
 
 // Wait waits until the item is done executing.
+// Also signals via error ErrCancel if workflow got canceled.
 func (q *fifo) Wait(ctx context.Context, taskID string) error {
 	q.Lock()
 	state := q.running[taskID]

server/queue/persistent.go

@@ -17,11 +17,14 @@ package queue
 
 import (
 	"context"
+	"errors"
+	"fmt"
 
 	"github.com/rs/zerolog/log"
 
 	"go.woodpecker-ci.org/woodpecker/v3/server/model"
 	"go.woodpecker-ci.org/woodpecker/v3/server/store"
+	"go.woodpecker-ci.org/woodpecker/v3/server/store/types"
 )
 
 // WithTaskStore returns a queue that is backed by the TaskStore. This
@@ -77,7 +80,14 @@ func (q *persistentQueue) Error(c context.Context, id string, err error) error {
 	if err := q.Queue.Error(c, id, err); err != nil {
 		return err
 	}
-	return q.store.TaskDelete(id)
+
+	if deleteErr := q.store.TaskDelete(id); deleteErr != nil {
+		if !errors.Is(deleteErr, types.RecordNotExist) {
+			return deleteErr
+		}
+		log.Debug().Msgf("task %s already removed from store", id)
+	}
+	return nil
 }
 
 // ErrorAtOnce signals multiple tasks are done and complete with an error.
@@ -86,10 +96,16 @@ func (q *persistentQueue) ErrorAtOnce(c context.Context, ids []string, err error
 	if err := q.Queue.ErrorAtOnce(c, ids, err); err != nil {
 		return err
 	}
+
+	var errs []error
 	for _, id := range ids {
-		if err := q.store.TaskDelete(id); err != nil {
-			return err
+		if deleteErr := q.store.TaskDelete(id); deleteErr != nil && !errors.Is(deleteErr, types.RecordNotExist) {
+			errs = append(errs, fmt.Errorf("task id [%s]: %w", id, deleteErr))
 		}
 	}
+
+	if len(errs) != 0 {
+		return fmt.Errorf("failed to delete tasks from persistent store: %w", errors.Join(errs...))
+	}
 	return nil
 }

server/queue/queue.go

@@ -101,6 +101,7 @@ type Queue interface {
 	ErrorAtOnce(c context.Context, ids []string, err error) error
 
 	// Wait waits until the task is complete.
+	// Also signals via error ErrCancel if workflow got canceled.
 	Wait(c context.Context, id string) error
 
 	// Info returns internal queue information.