Process workflow logs in batches (#4045)

server/store/datastore/log.go

@@ -15,16 +15,33 @@
 package datastore
 
 import (
+	"github.com/rs/zerolog/log"
+
 	"go.woodpecker-ci.org/woodpecker/v2/server/model"
 )
 
+// Maximum number of records to store in one PostgreSQL statement.
+// Too large a value results in `pq: got XX parameters but PostgreSQL only supports 65535 parameters`.
+const pgBatchSize = 1000
+
 func (s storage) LogFind(step *model.Step) ([]*model.LogEntry, error) {
 	var logEntries []*model.LogEntry
 	return logEntries, s.engine.Asc("id").Where("step_id = ?", step.ID).Find(&logEntries)
 }
 
-func (s storage) LogAppend(logEntry *model.LogEntry) error {
-	_, err := s.engine.Insert(logEntry)
+func (s storage) LogAppend(_ *model.Step, logEntries []*model.LogEntry) error {
+	var err error
+
+	// TODO: adapted from slices.Chunk(); switch to it in Go 1.23+
+	for i := 0; i < len(logEntries); i += pgBatchSize {
+		end := min(pgBatchSize, len(logEntries[i:]))
+		chunk := logEntries[i : i+end]
+
+		if _, err = s.engine.Insert(chunk); err != nil {
+			log.Error().Err(err).Msg("could not store log entries to db")
+		}
+	}
+
 	return err
 }
 

server/store/datastore/log_test.go

@@ -45,9 +45,7 @@ func TestLogCreateFindDelete(t *testing.T) {
 		},
 	}
 
-	for _, logEntry := range logEntries {
-		assert.NoError(t, store.LogAppend(logEntry))
-	}
+	assert.NoError(t, store.LogAppend(&step, logEntries))
 
 	// we want to find our inserted logs
 	_logEntries, err := store.LogFind(&step)
@@ -83,9 +81,7 @@ func TestLogAppend(t *testing.T) {
 		},
 	}
 
-	for _, logEntry := range logEntries {
-		assert.NoError(t, store.LogAppend(logEntry))
-	}
+	assert.NoError(t, store.LogAppend(&step, logEntries))
 
 	logEntry := &model.LogEntry{
 		StepID: step.ID,
@@ -94,7 +90,7 @@ func TestLogAppend(t *testing.T) {
 		Time:   20,
 	}
 
-	assert.NoError(t, store.LogAppend(logEntry))
+	assert.NoError(t, store.LogAppend(&step, []*model.LogEntry{logEntry}))
 
 	_logEntries, err := store.LogFind(&step)
 	assert.NoError(t, err)

server/store/mocks/store.go

@@ -1340,17 +1340,17 @@ func (_m *Store) HasRedirectionForRepo(_a0 int64, _a1 string) (bool, error) {
 	return r0, r1
 }
 
-// LogAppend provides a mock function with given fields: logEntry
-func (_m *Store) LogAppend(logEntry *model.LogEntry) error {
-	ret := _m.Called(logEntry)
+// LogAppend provides a mock function with given fields: _a0, _a1
+func (_m *Store) LogAppend(_a0 *model.Step, _a1 []*model.LogEntry) error {
+	ret := _m.Called(_a0, _a1)
 
 	if len(ret) == 0 {
 		panic("no return value specified for LogAppend")
 	}
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(*model.LogEntry) error); ok {
-		r0 = rf(logEntry)
+	if rf, ok := ret.Get(0).(func(*model.Step, []*model.LogEntry) error); ok {
+		r0 = rf(_a0, _a1)
 	} else {
 		r0 = ret.Error(0)
 	}

server/store/store.go

@@ -143,7 +143,7 @@ type Store interface {
 
 	// Logs
 	LogFind(*model.Step) ([]*model.LogEntry, error)
-	LogAppend(logEntry *model.LogEntry) error
+	LogAppend(*model.Step, []*model.LogEntry) error
 	LogDelete(*model.Step) error
 
 	// Tasks