Add log service addon (#5507)

Co-authored-by: Robert Kaussow <xoxys@rknet.org>
Co-authored-by: Robert Kaussow <mail@thegeeklab.de>

cmd/server/flags.go

@@ -343,13 +343,13 @@ var flags = append([]cli.Flag{
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_LOG_STORE"),
 		Name:    "log-store",
-		Usage:   "log store to use ('database' or 'file')",
+		Usage:   "log store to use ('database', 'addon' or 'file')",
 		Value:   "database",
 	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_LOG_STORE_FILE_PATH"),
 		Name:    "log-store-file-path",
-		Usage:   "directory used for file based log storage",
+		Usage:   "directory used for file based log storage or addon executable file path",
 	},
 	//
 	// backend options for pipeline compiler

cmd/server/setup.go

@@ -38,6 +38,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v3/server/queue"
 	"go.woodpecker-ci.org/woodpecker/v3/server/services"
 	logService "go.woodpecker-ci.org/woodpecker/v3/server/services/log"
+	"go.woodpecker-ci.org/woodpecker/v3/server/services/log/addon"
 	"go.woodpecker-ci.org/woodpecker/v3/server/services/log/file"
 	"go.woodpecker-ci.org/woodpecker/v3/server/services/permissions"
 	"go.woodpecker-ci.org/woodpecker/v3/server/store"
@@ -125,6 +126,8 @@ func setupLogStore(c *cli.Command, s store.Store) (logService.Service, error) {
 	switch c.String("log-store") {
 	case "file":
 		return file.NewLogStore(c.String("log-store-file-path"))
+	case "addon":
+		return addon.Load(c.String("log-store-file-path"))
 	default:
 		return s, nil
 	}

docs/docs/30-administration/10-configuration/10-server.md

@@ -1121,7 +1121,11 @@ Disable version check in admin web UI.
 - Name: `WOODPECKER_LOG_STORE`
 - Default: `database`
 
-Where to store logs. Possible values: `database` or `file`.
+Where to store logs. Possible values:
+
+- `database`: stores the logs in the database
+- `file`: stores logs in JSON files on the files system
+- `addon`: uses an [addon](./100-addons.md#log) to store logs
 
 ---
 
@@ -1130,7 +1134,10 @@ Where to store logs. Possible values: `database` or `file`.
 - Name: `WOODPECKER_LOG_STORE_FILE_PATH`
 - Default: none
 
-Directory to store logs in if [`WOODPECKER_LOG_STORE`](#log_store) is `file`.
+If [`WOODPECKER_LOG_STORE`](#log_store) is:
+
+- `file`: Directory to store logs in
+- `addon`: The path to the addon executable
 
 ---
 

docs/docs/30-administration/10-configuration/100-addons.md

@@ -0,0 +1,42 @@
+# Addons
+
+Addons can be used to extend the Woodpecker server. Currently, they can be used for forges and the log service.
+
+:::warning
+Addon forges are still experimental. Their implementation can change and break at any time.
+:::
+
+:::danger
+You must trust the author of the addon forge you are using. They may have access to authentication codes and other potentially sensitive information.
+:::
+
+## Usage
+
+To use an addon forge, download the correct addon version.
+
+### Forge
+
+Use this in your `.env`:
+
+```ini
+WOODPECKER_ADDON_FORGE=/path/to/your/addon/forge/file
+```
+
+In case you run Woodpecker as container, you probably want to mount the addon binary to `/opt/addons/`.
+
+#### List of addon forges
+
+- [Radicle](https://radicle.xyz/): Open source, peer-to-peer code collaboration stack built on Git. Radicle addon for Woodpecker CI can be found at [this repo](https://explorer.radicle.gr/nodes/seed.radicle.gr/rad:z39Cf1XzrvCLRZZJRUZnx9D1fj5ws).
+
+### Log
+
+Use this in your `.env`:
+
+```ini
+WOODPECKER_LOG_STORE=addon
+WOODPECKER_LOG_STORE_FILE_PATH=/path/to/your/addon/forge/file
+```
+
+## Developing addon forges
+
+See [Addons](../../92-development/100-addons.md).

docs/docs/30-administration/10-configuration/12-forges/11-overview.md

@@ -14,3 +14,5 @@
 | [when.path filter](../../../20-usage/20-workflow-syntax.md#path)                                                       | :white_check_mark:     | :white_check_mark:   | :white_check_mark:       | :white_check_mark:     | :x:                          | :white_check_mark:                                 |
 
 ¹ The deployment event can be triggered for all forges from Woodpecker directly. However, only GitHub can trigger them using webhooks.
+
+In addition to this, Woodpecker supports [addon forges](../100-addons.md) if the forge you are using does not meet the [Woodpecker requirements](../../../92-development/02-core-ideas.md#forges) or your setup is too specific to be included in the Woodpecker core.

docs/docs/92-development/02-core-ideas.md

@@ -8,7 +8,7 @@
 ## Addons and extensions
 
 If you are wondering whether your contribution will be accepted to be merged in the Woodpecker core, or whether it's better to write an
-[addon forge](../30-administration/10-configuration/12-forges/100-addon.md), [extension](../30-administration/10-configuration/10-server.md#external-configuration-api) or an
+[addon](../30-administration/10-configuration/100-addons.md), [extension](../30-administration/10-configuration/10-server.md#external-configuration-api) or an
 [external custom backend](../30-administration/10-configuration/11-backends/50-custom.md), please check these points:
 
 - Is your change very specific to your setup and unlikely to be used by anyone else?

docs/docs/92-development/100-addons.md

@@ -1,34 +1,16 @@
-# Custom
+# Addons
 
-If the forge you are using does not meet the [Woodpecker requirements](../../../92-development/02-core-ideas.md#forges) or your setup is too specific to be included in the Woodpecker core, you can write an addon forge.
+The Woodpecker server supports addons for forges and the log store.
 
 :::warning
-Addon forges are still experimental. Their implementation can change and break at any time.
+Addons are still experimental. Their implementation can change and break at any time.
 :::
 
-:::danger
-You must trust the author of the addon forge you are using. They may have access to authentication codes and other potentially sensitive information.
-:::
-
-## Usage
-
-To use an addon forge, download the correct addon version. Then, you can add the following to your configuration:
-
-```ini
-WOODPECKER_ADDON_FORGE=/path/to/your/addon/forge/file
-```
-
-In case you run Woodpecker as container, you probably want to mount the addon binary to `/opt/addons/`.
-
-### Bug reports
+## Bug reports
 
 If you experience bugs, please check which component has the issue. If it's the addon, **do not raise an issue in the main repository**, but rather use the separate addon repositories. To check which component is responsible for the bug, look at the logs. Logs from addons are marked with a special field `addon` containing their addon file name.
 
-## List of addon forges
-
-- [Radicle](https://radicle.xyz/): Open source, peer-to-peer code collaboration stack built on Git. Radicle addon for Woodpecker CI can be found at [this repo](https://explorer.radicle.gr/nodes/seed.radicle.gr/rad:z39Cf1XzrvCLRZZJRUZnx9D1fj5ws).
-
-## Creating addon forges
+## Creating addons
 
 Addons use RPC to communicate to the server and are implemented using the [`go-plugin` library](https://github.com/hashicorp/go-plugin).
 
@@ -38,7 +20,7 @@ This example will use the Go language.
 
 Directly import Woodpecker's Go packages (`go.woodpecker-ci.org/woodpecker/v3`) and use the interfaces and types defined there.
 
-In the `main` function, just call `"go.woodpecker-ci.org/woodpecker/v3/server/forge/addon".Serve` with a `"go.woodpecker-ci.org/woodpecker/v3/server/forge".Forge` as argument.
+In the `main` function, just call the `Serve` method in the corresponding [addon package](#addon-types) with the service as argument.
 This will take care of connecting the addon forge to the server.
 
 :::note
@@ -47,6 +29,8 @@ It is not possible to access global variables from Woodpecker, for example the s
 
 ### Example structure
 
+This is an example for a forge addon.
+
 ```go
 package main
 
@@ -68,3 +52,10 @@ type config struct {
 
 // `config` must implement `"go.woodpecker-ci.org/woodpecker/v3/server/forge".Forge`. You must directly use Woodpecker's packages - see imports above.
 ```
+
+### Addon types
+
+| Type      | Addon package                                                 | Service interface                                                 |
+| --------- | ------------------------------------------------------------- | ----------------------------------------------------------------- |
+| Forge     | `go.woodpecker-ci.org/woodpecker/v3/server/forge/addon`       | `"go.woodpecker-ci.org/woodpecker/v3/server/forge".Forge`         |
+| Log store | `go.woodpecker-ci.org/woodpecker/v3/server/service/log/addon` | `"go.woodpecker-ci.org/woodpecker/v3/server/service/log".Service` |

server/forge/addon/client.go

@@ -28,6 +28,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v3/server/forge"
 	"go.woodpecker-ci.org/woodpecker/v3/server/forge/types"
 	"go.woodpecker-ci.org/woodpecker/v3/server/model"
+	"go.woodpecker-ci.org/woodpecker/v3/shared/logger"
 )
 
 // make sure RPC implements forge.Forge.
@@ -40,8 +41,8 @@ func Load(file string) (forge.Forge, error) {
 			pluginKey: &Plugin{},
 		},
 		Cmd: exec.Command(file),
-		Logger: &clientLogger{
-			logger: log.With().Str("addon", file).Logger(),
+		Logger: &logger.AddonClientLogger{
+			Logger: log.With().Str("addon", file).Logger(),
 		},
 	})
 	// TODO: defer client.Kill()

server/services/log/addon/client.go

@@ -0,0 +1,116 @@
+// Copyright 2025 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package addon
+
+import (
+	"encoding/json"
+	"net/rpc"
+	"os/exec"
+
+	"github.com/hashicorp/go-plugin"
+	"github.com/rs/zerolog/log"
+
+	"go.woodpecker-ci.org/woodpecker/v3/server/model"
+	logService "go.woodpecker-ci.org/woodpecker/v3/server/services/log"
+	"go.woodpecker-ci.org/woodpecker/v3/shared/logger"
+)
+
+// make sure RPC implements logService.Service.
+var _ logService.Service = new(RPC)
+
+func Load(file string) (logService.Service, error) {
+	client := plugin.NewClient(&plugin.ClientConfig{
+		HandshakeConfig: HandshakeConfig,
+		Plugins: map[string]plugin.Plugin{
+			pluginKey: &Plugin{},
+		},
+		Cmd: exec.Command(file),
+		Logger: &logger.AddonClientLogger{
+			Logger: log.With().Str("addon", file).Logger(),
+		},
+	})
+	// TODO: defer client.Kill()
+
+	rpcClient, err := client.Client()
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := rpcClient.Dispense(pluginKey)
+	if err != nil {
+		return nil, err
+	}
+
+	extension, _ := raw.(logService.Service)
+	return extension, nil
+}
+
+type RPC struct {
+	client *rpc.Client
+}
+
+func (g *RPC) LogFind(step *model.Step) ([]*model.LogEntry, error) {
+	args, err := json.Marshal(step)
+	if err != nil {
+		return nil, err
+	}
+	var jsonResp []byte
+	err = g.client.Call("Plugin.LogFind", args, &jsonResp)
+	if err != nil {
+		return nil, err
+	}
+
+	var resp []*model.LogEntry
+	err = json.Unmarshal(jsonResp, &resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+func (g *RPC) LogAppend(step *model.Step, logEntries []*model.LogEntry) error {
+	args, err := json.Marshal(&argumentsAppend{
+		Step:       step,
+		LogEntries: logEntries,
+	})
+	if err != nil {
+		return err
+	}
+	var jsonResp []byte
+	return g.client.Call("Plugin.LogAppend", args, &jsonResp)
+}
+
+func (g *RPC) LogDelete(step *model.Step) error {
+	args, err := json.Marshal(step)
+	if err != nil {
+		return err
+	}
+	var jsonResp []byte
+	return g.client.Call("Plugin.LogDelete", args, &jsonResp)
+}
+
+func (g *RPC) StepFinished(step *model.Step) {
+	args, err := json.Marshal(step)
+	if err != nil {
+		log.Error().Err(err).Msg("could not marshal json for log addon")
+		return
+	}
+	var jsonResp []byte
+	err = g.client.Call("Plugin.StepFinished", args, &jsonResp)
+	if err != nil {
+		log.Error().Err(err).Msg("StepFinished via addon failed")
+	}
+}

server/services/log/addon/plugin.go

@@ -0,0 +1,43 @@
+// Copyright 2025 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package addon
+
+import (
+	"net/rpc"
+
+	"github.com/hashicorp/go-plugin"
+
+	"go.woodpecker-ci.org/woodpecker/v3/server/services/log"
+)
+
+const pluginKey = "log"
+
+var HandshakeConfig = plugin.HandshakeConfig{
+	ProtocolVersion:  1,
+	MagicCookieKey:   "WOODPECKER_LOG_ADDON_PLUGIN",
+	MagicCookieValue: "woodpecker-plugin-magic-cookie-value",
+}
+
+type Plugin struct {
+	Impl log.Service
+}
+
+func (p *Plugin) Server(*plugin.MuxBroker) (any, error) {
+	return &RPCServer{Impl: p.Impl}, nil
+}
+
+func (*Plugin) Client(_ *plugin.MuxBroker, c *rpc.Client) (any, error) {
+	return &RPC{client: c}, nil
+}

server/services/log/addon/server.go

@@ -0,0 +1,87 @@
+// Copyright 2025 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package addon
+
+import (
+	"encoding/json"
+
+	"github.com/hashicorp/go-plugin"
+
+	"go.woodpecker-ci.org/woodpecker/v3/server/model"
+	"go.woodpecker-ci.org/woodpecker/v3/server/services/log"
+)
+
+func Serve(impl log.Service) {
+	plugin.Serve(&plugin.ServeConfig{
+		HandshakeConfig: HandshakeConfig,
+		Plugins: map[string]plugin.Plugin{
+			pluginKey: &Plugin{Impl: impl},
+		},
+	})
+}
+
+type RPCServer struct {
+	Impl log.Service
+}
+
+type argumentsAppend struct {
+	Step       *model.Step       `json:"step"`
+	LogEntries []*model.LogEntry `json:"log_entries"`
+}
+
+func (s *RPCServer) LogFind(args []byte, resp *[]byte) error {
+	var a model.Step
+	err := json.Unmarshal(args, &a)
+	if err != nil {
+		return err
+	}
+	log, err := s.Impl.LogFind(&a)
+	if err != nil {
+		return err
+	}
+	*resp, err = json.Marshal(log)
+	return err
+}
+
+func (s *RPCServer) LogAppend(args []byte, resp *[]byte) error {
+	var a argumentsAppend
+	err := json.Unmarshal(args, &a)
+	if err != nil {
+		return err
+	}
+	*resp = []byte{}
+	return s.Impl.LogAppend(a.Step, a.LogEntries)
+}
+
+func (s *RPCServer) LogDelete(args []byte, resp *[]byte) error {
+	var a model.Step
+	err := json.Unmarshal(args, &a)
+	if err != nil {
+		return err
+	}
+	*resp = []byte{}
+	return s.Impl.LogDelete(&a)
+}
+
+func (s *RPCServer) StepFinished(args []byte, resp *[]byte) error {
+	var a model.Step
+	err := json.Unmarshal(args, &a)
+	if err != nil {
+		return err
+	}
+	*resp = []byte{}
+	s.Impl.StepFinished(&a)
+	return nil
+}

shared/logger/addon_logger.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package addon
+package logger
 
 import (
 	"bytes"
@@ -24,8 +24,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-type clientLogger struct {
-	logger   zerolog.Logger
+type AddonClientLogger struct {
+	Logger   zerolog.Logger
 	name     string
 	withArgs []any
 }
@@ -48,9 +48,9 @@ func convertLvl(level hclog.Level) zerolog.Level {
 	return zerolog.NoLevel
 }
 
-func (c *clientLogger) applyArgs(args []any) *zerolog.Logger {
+func (c *AddonClientLogger) applyArgs(args []any) *zerolog.Logger {
 	var key string
-	logger := c.logger.With()
+	logger := c.Logger.With()
 	args = append(args, c.withArgs)
 	for i, arg := range args {
 		switch {
@@ -68,67 +68,67 @@ func (c *clientLogger) applyArgs(args []any) *zerolog.Logger {
 	return &l
 }
 
-func (c *clientLogger) Log(level hclog.Level, msg string, args ...any) {
+func (c *AddonClientLogger) Log(level hclog.Level, msg string, args ...any) {
 	c.applyArgs(args).WithLevel(convertLvl(level)).Msg(msg)
 }
 
-func (c *clientLogger) Trace(msg string, args ...any) {
+func (c *AddonClientLogger) Trace(msg string, args ...any) {
 	c.applyArgs(args).Trace().Msg(msg)
 }
 
-func (c *clientLogger) Debug(msg string, args ...any) {
+func (c *AddonClientLogger) Debug(msg string, args ...any) {
 	c.applyArgs(args).Debug().Msg(msg)
 }
 
-func (c *clientLogger) Info(msg string, args ...any) {
+func (c *AddonClientLogger) Info(msg string, args ...any) {
 	c.applyArgs(args).Info().Msg(msg)
 }
 
-func (c *clientLogger) Warn(msg string, args ...any) {
+func (c *AddonClientLogger) Warn(msg string, args ...any) {
 	c.applyArgs(args).Warn().Msg(msg)
 }
 
-func (c *clientLogger) Error(msg string, args ...any) {
+func (c *AddonClientLogger) Error(msg string, args ...any) {
 	c.applyArgs(args).Error().Msg(msg)
 }
 
-func (c *clientLogger) IsTrace() bool {
+func (c *AddonClientLogger) IsTrace() bool {
 	return log.Logger.GetLevel() >= zerolog.TraceLevel
 }
 
-func (c *clientLogger) IsDebug() bool {
+func (c *AddonClientLogger) IsDebug() bool {
 	return log.Logger.GetLevel() >= zerolog.DebugLevel
 }
 
-func (c *clientLogger) IsInfo() bool {
+func (c *AddonClientLogger) IsInfo() bool {
 	return log.Logger.GetLevel() >= zerolog.InfoLevel
 }
 
-func (c *clientLogger) IsWarn() bool {
+func (c *AddonClientLogger) IsWarn() bool {
 	return log.Logger.GetLevel() >= zerolog.WarnLevel
 }
 
-func (c *clientLogger) IsError() bool {
+func (c *AddonClientLogger) IsError() bool {
 	return log.Logger.GetLevel() >= zerolog.ErrorLevel
 }
 
-func (c *clientLogger) ImpliedArgs() []any {
+func (c *AddonClientLogger) ImpliedArgs() []any {
 	return c.withArgs
 }
 
-func (c *clientLogger) With(args ...any) hclog.Logger {
-	return &clientLogger{
-		logger:   c.logger,
+func (c *AddonClientLogger) With(args ...any) hclog.Logger {
+	return &AddonClientLogger{
+		Logger:   c.Logger,
 		name:     c.name,
 		withArgs: args,
 	}
 }
 
-func (c *clientLogger) Name() string {
+func (c *AddonClientLogger) Name() string {
 	return c.name
 }
 
-func (c *clientLogger) Named(name string) hclog.Logger {
+func (c *AddonClientLogger) Named(name string) hclog.Logger {
 	curr := c.name
 	if curr != "" {
 		curr = c.name + "."
@@ -136,20 +136,20 @@ func (c *clientLogger) Named(name string) hclog.Logger {
 	return c.ResetNamed(curr + name)
 }
 
-func (c *clientLogger) ResetNamed(name string) hclog.Logger {
-	return &clientLogger{
-		logger:   c.logger,
+func (c *AddonClientLogger) ResetNamed(name string) hclog.Logger {
+	return &AddonClientLogger{
+		Logger:   c.Logger,
 		name:     name,
 		withArgs: c.withArgs,
 	}
 }
 
-func (c *clientLogger) SetLevel(level hclog.Level) {
-	c.logger = c.logger.Level(convertLvl(level))
+func (c *AddonClientLogger) SetLevel(level hclog.Level) {
+	c.Logger = c.Logger.Level(convertLvl(level))
 }
 
-func (c *clientLogger) GetLevel() hclog.Level {
-	switch c.logger.GetLevel() {
+func (c *AddonClientLogger) GetLevel() hclog.Level {
+	switch c.Logger.GetLevel() {
 	case zerolog.ErrorLevel:
 		return hclog.Error
 	case zerolog.WarnLevel:
@@ -164,12 +164,12 @@ func (c *clientLogger) GetLevel() hclog.Level {
 	return hclog.NoLevel
 }
 
-func (c *clientLogger) StandardLogger(opts *hclog.StandardLoggerOptions) *std_log.Logger {
+func (c *AddonClientLogger) StandardLogger(opts *hclog.StandardLoggerOptions) *std_log.Logger {
 	return std_log.New(c.StandardWriter(opts), "", 0)
 }
 
-func (c *clientLogger) StandardWriter(*hclog.StandardLoggerOptions) io.Writer {
-	return ioAdapter{logger: c.logger}
+func (c *AddonClientLogger) StandardWriter(*hclog.StandardLoggerOptions) io.Writer {
+	return ioAdapter{logger: c.Logger}
 }
 
 type ioAdapter struct {