Global and organization registries (#1672)

Co-authored-by: Anbraten <6918444+anbraten@users.noreply.github.com>

server/services/registry/combined.go

@@ -15,7 +15,10 @@
 package registry
 
 import (
+	"errors"
+
 	"go.woodpecker-ci.org/woodpecker/v2/server/model"
+	"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
 )
 
 type combined struct {
@@ -31,29 +34,44 @@ func NewCombined(dbRegistry Service, registries ...ReadOnlyService) Service {
 	}
 }
 
-func (c *combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) {
-	for _, registry := range c.registries {
-		res, err := registry.RegistryFind(repo, name)
-		if err != nil {
-			return nil, err
-		}
-		if res != nil {
-			return res, nil
-		}
-	}
-	return nil, nil
+func (c *combined) RegistryFind(repo *model.Repo, addr string) (*model.Registry, error) {
+	return c.dbRegistry.RegistryFind(repo, addr)
 }
 
 func (c *combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) {
-	var registries []*model.Registry
+	return c.dbRegistry.RegistryList(repo, p)
+}
+
+func (c *combined) RegistryListPipeline(repo *model.Repo, pipeline *model.Pipeline) ([]*model.Registry, error) {
+	dbRegistries, err := c.dbRegistry.RegistryListPipeline(repo, pipeline)
+	if err != nil {
+		return nil, err
+	}
+
+	registries := make([]*model.Registry, 0, len(dbRegistries))
+	exists := make(map[string]struct{}, len(dbRegistries))
+
+	// Assign database stored registries to the map to avoid duplicates
+	// from the combined registries so to prioritize ones in database.
+	for _, reg := range dbRegistries {
+		exists[reg.Address] = struct{}{}
+	}
+
 	for _, registry := range c.registries {
-		list, err := registry.RegistryList(repo, &model.ListOptions{All: true})
+		list, err := registry.GlobalRegistryList(&model.ListOptions{All: true})
 		if err != nil {
 			return nil, err
 		}
-		registries = append(registries, list...)
+		for _, reg := range list {
+			if _, ok := exists[reg.Address]; ok {
+				continue
+			}
+			exists[reg.Address] = struct{}{}
+			registries = append(registries, reg)
+		}
 	}
-	return model.ApplyPagination(p, registries), nil
+
+	return append(registries, dbRegistries...), nil
 }
 
 func (c *combined) RegistryCreate(repo *model.Repo, registry *model.Registry) error {
@@ -64,6 +82,86 @@ func (c *combined) RegistryUpdate(repo *model.Repo, registry *model.Registry) er
 	return c.dbRegistry.RegistryUpdate(repo, registry)
 }
 
-func (c *combined) RegistryDelete(repo *model.Repo, name string) error {
-	return c.dbRegistry.RegistryDelete(repo, name)
+func (c *combined) RegistryDelete(repo *model.Repo, addr string) error {
+	return c.dbRegistry.RegistryDelete(repo, addr)
+}
+
+func (c *combined) OrgRegistryFind(owner int64, addr string) (*model.Registry, error) {
+	return c.dbRegistry.OrgRegistryFind(owner, addr)
+}
+
+func (c *combined) OrgRegistryList(owner int64, p *model.ListOptions) ([]*model.Registry, error) {
+	return c.dbRegistry.OrgRegistryList(owner, p)
+}
+
+func (c *combined) OrgRegistryCreate(owner int64, registry *model.Registry) error {
+	return c.dbRegistry.OrgRegistryCreate(owner, registry)
+}
+
+func (c *combined) OrgRegistryUpdate(owner int64, registry *model.Registry) error {
+	return c.dbRegistry.OrgRegistryUpdate(owner, registry)
+}
+
+func (c *combined) OrgRegistryDelete(owner int64, addr string) error {
+	return c.dbRegistry.OrgRegistryDelete(owner, addr)
+}
+
+func (c *combined) GlobalRegistryFind(addr string) (*model.Registry, error) {
+	registry, err := c.dbRegistry.GlobalRegistryFind(addr)
+	if err != nil && !errors.Is(err, types.RecordNotExist) {
+		return nil, err
+	}
+	if registry != nil {
+		return registry, nil
+	}
+	for _, reg := range c.registries {
+		if registry, err := reg.GlobalRegistryFind(addr); err == nil {
+			return registry, nil
+		}
+	}
+	return nil, types.RecordNotExist
+}
+
+func (c *combined) GlobalRegistryList(p *model.ListOptions) ([]*model.Registry, error) {
+	dbRegistries, err := c.dbRegistry.GlobalRegistryList(&model.ListOptions{All: true})
+	if err != nil {
+		return nil, err
+	}
+
+	registries := make([]*model.Registry, 0, len(dbRegistries))
+	exists := make(map[string]struct{}, len(dbRegistries))
+
+	// Assign database stored registries to the map to avoid duplicates
+	// from the combined registries so to prioritize ones in database.
+	for _, reg := range dbRegistries {
+		exists[reg.Address] = struct{}{}
+	}
+
+	for _, registry := range c.registries {
+		list, err := registry.GlobalRegistryList(&model.ListOptions{All: true})
+		if err != nil {
+			return nil, err
+		}
+		for _, reg := range list {
+			if _, ok := exists[reg.Address]; ok {
+				continue
+			}
+			exists[reg.Address] = struct{}{}
+			registries = append(registries, reg)
+		}
+	}
+
+	return model.ApplyPagination(p, append(registries, dbRegistries...)), nil
+}
+
+func (c *combined) GlobalRegistryCreate(registry *model.Registry) error {
+	return c.dbRegistry.GlobalRegistryCreate(registry)
+}
+
+func (c *combined) GlobalRegistryUpdate(registry *model.Registry) error {
+	return c.dbRegistry.GlobalRegistryUpdate(registry)
+}
+
+func (c *combined) GlobalRegistryDelete(addr string) error {
+	return c.dbRegistry.GlobalRegistryDelete(addr)
 }

server/services/registry/db.go

@@ -28,12 +28,45 @@ func NewDB(store store.Store) Service {
 	return &db{store}
 }
 
-func (d *db) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) {
-	return d.store.RegistryFind(repo, name)
+func (d *db) RegistryFind(repo *model.Repo, addr string) (*model.Registry, error) {
+	return d.store.RegistryFind(repo, addr)
 }
 
 func (d *db) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) {
-	return d.store.RegistryList(repo, p)
+	return d.store.RegistryList(repo, false, p)
+}
+
+func (d *db) RegistryListPipeline(repo *model.Repo, _ *model.Pipeline) ([]*model.Registry, error) {
+	r, err := d.store.RegistryList(repo, true, &model.ListOptions{All: true})
+	if err != nil {
+		return nil, err
+	}
+
+	// Return only registries with unique address
+	// Priority order in case of duplicate addresses are repository, user/organization, global
+	registries := make([]*model.Registry, 0, len(r))
+	uniq := make(map[string]struct{})
+	for _, condition := range []struct {
+		IsRepository   bool
+		IsOrganization bool
+		IsGlobal       bool
+	}{
+		{IsRepository: true},
+		{IsOrganization: true},
+		{IsGlobal: true},
+	} {
+		for _, registry := range r {
+			if registry.IsRepository() != condition.IsRepository || registry.IsOrganization() != condition.IsOrganization || registry.IsGlobal() != condition.IsGlobal {
+				continue
+			}
+			if _, ok := uniq[registry.Address]; ok {
+				continue
+			}
+			uniq[registry.Address] = struct{}{}
+			registries = append(registries, registry)
+		}
+	}
+	return registries, nil
 }
 
 func (d *db) RegistryCreate(_ *model.Repo, in *model.Registry) error {
@@ -45,5 +78,57 @@ func (d *db) RegistryUpdate(_ *model.Repo, in *model.Registry) error {
 }
 
 func (d *db) RegistryDelete(repo *model.Repo, addr string) error {
-	return d.store.RegistryDelete(repo, addr)
+	registry, err := d.store.RegistryFind(repo, addr)
+	if err != nil {
+		return err
+	}
+	return d.store.RegistryDelete(registry)
+}
+
+func (d *db) OrgRegistryFind(owner int64, name string) (*model.Registry, error) {
+	return d.store.OrgRegistryFind(owner, name)
+}
+
+func (d *db) OrgRegistryList(owner int64, p *model.ListOptions) ([]*model.Registry, error) {
+	return d.store.OrgRegistryList(owner, p)
+}
+
+func (d *db) OrgRegistryCreate(_ int64, in *model.Registry) error {
+	return d.store.RegistryCreate(in)
+}
+
+func (d *db) OrgRegistryUpdate(_ int64, in *model.Registry) error {
+	return d.store.RegistryUpdate(in)
+}
+
+func (d *db) OrgRegistryDelete(owner int64, addr string) error {
+	registry, err := d.store.OrgRegistryFind(owner, addr)
+	if err != nil {
+		return err
+	}
+	return d.store.RegistryDelete(registry)
+}
+
+func (d *db) GlobalRegistryFind(addr string) (*model.Registry, error) {
+	return d.store.GlobalRegistryFind(addr)
+}
+
+func (d *db) GlobalRegistryList(p *model.ListOptions) ([]*model.Registry, error) {
+	return d.store.GlobalRegistryList(p)
+}
+
+func (d *db) GlobalRegistryCreate(in *model.Registry) error {
+	return d.store.RegistryCreate(in)
+}
+
+func (d *db) GlobalRegistryUpdate(in *model.Registry) error {
+	return d.store.RegistryUpdate(in)
+}
+
+func (d *db) GlobalRegistryDelete(addr string) error {
+	registry, err := d.store.GlobalRegistryFind(addr)
+	if err != nil {
+		return err
+	}
+	return d.store.RegistryDelete(registry)
 }

server/services/registry/filesystem.go

@@ -25,6 +25,7 @@ import (
 	"github.com/docker/cli/cli/config/types"
 
 	"go.woodpecker-ci.org/woodpecker/v2/server/model"
+	model_types "go.woodpecker-ci.org/woodpecker/v2/server/store/types"
 )
 
 type filesystem struct {
@@ -79,17 +80,29 @@ func parseDockerConfig(path string) ([]*model.Registry, error) {
 			Address:  key,
 			Username: auth.Username,
 			Password: auth.Password,
+			ReadOnly: true,
 		})
 	}
 
 	return registries, nil
 }
 
-func (f *filesystem) RegistryFind(*model.Repo, string) (*model.Registry, error) {
-	return nil, nil
+func (f *filesystem) GlobalRegistryFind(addr string) (*model.Registry, error) {
+	registries, err := f.GlobalRegistryList(&model.ListOptions{All: true})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, reg := range registries {
+		if reg.Address == addr {
+			return reg, nil
+		}
+	}
+
+	return nil, model_types.RecordNotExist
 }
 
-func (f *filesystem) RegistryList(_ *model.Repo, p *model.ListOptions) ([]*model.Registry, error) {
+func (f *filesystem) GlobalRegistryList(p *model.ListOptions) ([]*model.Registry, error) {
 	regs, err := parseDockerConfig(f.path)
 	if err != nil {
 		return nil, err

server/services/registry/service.go

@@ -18,15 +18,29 @@ import "go.woodpecker-ci.org/woodpecker/v2/server/model"
 
 // Service defines a service for managing registries.
 type Service interface {
+	RegistryListPipeline(*model.Repo, *model.Pipeline) ([]*model.Registry, error)
+	// Repository registries
 	RegistryFind(*model.Repo, string) (*model.Registry, error)
 	RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error)
 	RegistryCreate(*model.Repo, *model.Registry) error
 	RegistryUpdate(*model.Repo, *model.Registry) error
 	RegistryDelete(*model.Repo, string) error
+	// Organization registries
+	OrgRegistryFind(int64, string) (*model.Registry, error)
+	OrgRegistryList(int64, *model.ListOptions) ([]*model.Registry, error)
+	OrgRegistryCreate(int64, *model.Registry) error
+	OrgRegistryUpdate(int64, *model.Registry) error
+	OrgRegistryDelete(int64, string) error
+	// Global registries
+	GlobalRegistryFind(string) (*model.Registry, error)
+	GlobalRegistryList(*model.ListOptions) ([]*model.Registry, error)
+	GlobalRegistryCreate(*model.Registry) error
+	GlobalRegistryUpdate(*model.Registry) error
+	GlobalRegistryDelete(string) error
 }
 
 // ReadOnlyService defines a service for managing registries.
 type ReadOnlyService interface {
-	RegistryFind(*model.Repo, string) (*model.Registry, error)
-	RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error)
+	GlobalRegistryFind(string) (*model.Registry, error)
+	GlobalRegistryList(*model.ListOptions) ([]*model.Registry, error)
 }