WIP: Plugin RFC

.github/codecov.yaml

@@ -1,6 +1,7 @@
 ignore:
 - "test"  # Our test helpers largely do not have tests themselves.
 - "**/*_string.go"  # Ignore generated string implementations.
+- "toolkit/urn/parser.go"  # Generated file
 
 coverage:
   status:

book.toml

@@ -25,3 +25,6 @@ command = "go run github.com/quay/claircore/internal/cmd/mdbook-injecturls"
 [preprocessor.mermaid]
 command = "go run github.com/quay/claircore/internal/cmd/mdbook-mermaid"
 after = ["links"]
+
+[preprocessor.plugins]
+command = "go run github.com/quay/claircore/internal/cmd/plugintool -mdbook"

docs/reference.md

@@ -16,3 +16,5 @@ Important interfaces and structs are outlined.
 - [Version Filter](./reference/version_filter.md)
 - [Versioned Scanner](./reference/versioned_scanner.md)
 - [Resolver](./reference/resolver.md)
+
+{{# plugintool indexer }}

go.mod

@@ -1,8 +1,9 @@
 module github.com/quay/claircore
 
 go 1.21.8
 
 require (
+	go.opentelemetry.io/otel/trace v1.25.0
 	github.com/Masterminds/semver v1.5.0
 	github.com/doug-martin/goqu/v8 v8.6.0
 	github.com/golang/mock v1.6.0
@@ -12,6 +13,7 @@
 	github.com/jackc/pgtype v1.14.2
 	github.com/jackc/pgx/v4 v4.18.3
 	github.com/klauspost/compress v1.17.8
+	github.com/jackc/puddle/v2 v2.2.1
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
@@ -22,8 +24,10 @@
 	github.com/quay/zlog v1.1.8
 	github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319
 	github.com/rs/zerolog v1.30.0
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/ulikunitz/xz v0.5.11
 	go.opentelemetry.io/otel v1.25.0
+	go.opentelemetry.io/otel/metric v1.25.0
 	go.opentelemetry.io/otel/trace v1.25.0
 	golang.org/x/crypto v0.22.0
 	golang.org/x/sync v0.7.0
@@ -55,7 +59,6 @@
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	go.opentelemetry.io/otel/metric v1.25.0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
@@ -67,3 +70,5 @@
 )
 
 replace github.com/quay/claircore/updater/driver => ./updater/driver
+
+replace github.com/quay/claircore/toolkit => ./toolkit

go.sum

@@ -38,7 +38,6 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
@@ -97,6 +96,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
@@ -147,8 +148,6 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz
 github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/quay/claircore/toolkit v1.1.1 h1:9GFy14ffOkIOpl0fbR+bHr4i19VEwms1pXw8S8up0e4=
-github.com/quay/claircore/toolkit v1.1.1/go.mod h1:ZZHA/b/qpfUcNHFJeYVA0bOp7aL4r3CTFhlBV/ezoFI=
 github.com/quay/goval-parser v0.8.8 h1:Uf+f9iF2GIR5GPUY2pGoa9il2+4cdES44ZlM0mWm4cA=
 github.com/quay/goval-parser v0.8.8/go.mod h1:Y0NTNfPYOC7yxsYKzJOrscTWUPq1+QbtHw4XpPXWPMc=
 github.com/quay/zlog v1.1.8 h1:/cKgHpqKu3g7mB9OlvnfbqrbPIssyxeameDBytGPrNs=
@@ -164,6 +163,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=

indexer/ecosystem.go

@@ -23,6 +23,22 @@ type Ecosystem struct {
 	Name                 string
 }
 
+// EcosystemSpec ties together a set of plugins.
+type EcosystemSpec struct {
+	Name string `json:"name"`
+
+	// Feature plugins.
+	Distribution []string `json:"distribution"`
+	File         []string `json:"file"`
+	Package      []string `json:"package"`
+	Repository   []string `json:"repository"`
+	// Sees all the plugins above at the manifest level.
+	Coalescer []string `json:"coalescer"`
+
+	// Sees the output of all Coalescers across all ecosystems.
+	Resolver []string `json:"resolver"`
+}
+
 // EcosystemsToScanners extracts and dedupes multiple ecosystems and returns
 // their discrete scanners.
 func EcosystemsToScanners(ctx context.Context, ecosystems []*Ecosystem) ([]PackageScanner, []DistributionScanner, []RepositoryScanner, []FileScanner, error) {

indexer/ecosystem_loader.go

@@ -0,0 +1,91 @@
+package indexer
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/quay/claircore/toolkit/registry"
+)
+
+// TODO(hank) Find a better home for this.
+
+type ecosystemLoader []loadEntry
+
+type loadEntry struct {
+	Path     string `json:"path"`
+	Contents string `json:"contents"`
+	Default  bool   `json:"default"`
+}
+
+var (
+	//go:embed loader_schema.json
+	loaderSchema string
+
+	// Ecosystem is a DynamicPlugin hook for loading EcosystemSpecs from JSON
+	// objects.
+	//
+	//plugintool:register indexer
+	loaderDescription = registry.Description[DynamicPlugin]{
+		Default:      true,
+		ConfigSchema: loaderSchema,
+		New: func(ctx context.Context, f func(v any) error) (DynamicPlugin, error) {
+			return newLoader(ctx, f)
+		},
+	}
+)
+
+const loaderName = `urn:claircore:indexer:dynamicplugin:ecosystem`
+
+func newLoader(ctx context.Context, f func(v any) error) (*ecosystemLoader, error) {
+	var l ecosystemLoader
+	if err := f(&l); err != nil {
+		return nil, err
+	}
+	return &l, nil
+}
+
+// Run implements [indexer.DynamicPlugin].
+func (l *ecosystemLoader) Run(ctx context.Context) error {
+	var errs []error
+	for n, i := range *l {
+		var b []byte
+		var err error
+		switch {
+		case i.Path != "" && i.Contents != "":
+			err = fmt.Errorf(`element %d contains "path" and "contents" keys`, n)
+		case i.Path != "":
+			b, err = os.ReadFile(i.Path)
+		case i.Contents != "":
+			b = []byte(i.Contents)
+		}
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		var spec EcosystemSpec
+		if err := json.Unmarshal(b, &spec); err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		err = registry.Register(spec.Name, &registry.Description[EcosystemSpec]{
+			Default: i.Default,
+			New: func(_ context.Context, f func(any) error) (EcosystemSpec, error) {
+				return spec, f(new(struct{}))
+			},
+		})
+		switch {
+		case errors.Is(err, nil): // OK
+		case errors.Is(err, registry.ErrAlreadyRegistered): // Skip
+		default:
+			errs = append(errs, err)
+		}
+	}
+	if err := errors.Join(errs...); err != nil {
+		return fmt.Errorf("ecosystem: %w", err)
+	}
+	return nil
+}

indexer/ecosystem_loader_test.go

@@ -0,0 +1,133 @@
+package indexer
+
+import (
+	"context"
+	"encoding/json"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/quay/claircore/toolkit/registry"
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore/internal/plugin"
+)
+
+func init() {
+	err := registry.Register(loaderName, &registry.Description[DynamicPlugin]{
+		Default:      true,
+		ConfigSchema: loaderSchema,
+		New: func(ctx context.Context, f func(v any) error) (DynamicPlugin, error) {
+			return newLoader(ctx, f)
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestEcosystemLoader(t *testing.T) {
+	var todo []string
+	sys := os.DirFS(".")
+	err := fs.WalkDir(sys, "testdata", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if ok, _ := path.Match(`*.config.json`, d.Name()); ok {
+			todo = append(todo, p)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	checkConfig := func(fn string) func(*testing.T) {
+		return func(t *testing.T) {
+			b, err := os.ReadFile(fn)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx := context.Background()
+			ctx = zlog.Test(ctx, t)
+			cfg := plugin.Config{
+				Configs: map[string][]byte{
+					loaderName: b,
+				},
+				PoolSize: 1,
+			}
+			if len(b) == 0 {
+				delete(cfg.Configs, loaderName)
+			}
+			pool, err := plugin.NewPool[DynamicPlugin](ctx, &cfg, loaderName)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer pool.Close()
+			p, done, err := pool.Get(ctx, loaderName)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer done()
+			if err := p.Run(ctx); err != nil {
+				t.Fatal(err)
+			}
+
+			for _, e := range *(p.(*ecosystemLoader)) {
+				var spec EcosystemSpec
+				var b []byte
+				var err error
+				switch {
+				case e.Path != "" && e.Contents != "":
+					t.Error(`cannot have both "path" and "contents"`)
+					continue
+				case e.Path != "":
+					b, err = os.ReadFile(e.Path)
+				case e.Contents != "":
+					b = []byte(e.Contents)
+				default:
+					panic("unreachable")
+				}
+				if err != nil {
+					t.Error(err)
+					continue
+				}
+				if err := json.Unmarshal(b, &spec); err != nil {
+					t.Error(err)
+					continue
+				}
+				m, err := registry.GetDescription[EcosystemSpec](spec.Name)
+				if err != nil {
+					t.Error(err)
+					continue
+				}
+				desc, ok := m[spec.Name]
+				if !ok {
+					t.Errorf("missing description for %q", spec.Name)
+					continue
+				}
+				if got, want := e.Default, desc.Default; got != want {
+					t.Errorf("default: got: %v, want: %v", got, want)
+					continue
+				}
+				got, err := desc.New(ctx, func(_ any) error { return nil })
+				if err != nil {
+					t.Error(err)
+					continue
+				}
+
+				if want := spec; !cmp.Equal(got, want) {
+					t.Error(cmp.Diff(got, want))
+				}
+			}
+		}
+	}
+
+	t.Run("null", checkConfig("/dev/null"))
+	for _, fn := range todo {
+		t.Run(strings.TrimSuffix(path.Base(fn), ".config.json"), checkConfig(fn))
+	}
+}

indexer/indexer.go

@@ -0,0 +1,11 @@
+package indexer
+
+import "context"
+
+// DynamicPlugin is an interface for registering additional plugins at runtime.
+//
+// The indexer attempts to call the [Run] method only once, but the
+// implementation should be idempotent.
+type DynamicPlugin interface {
+	Run(context.Context) error
+}