diff --git a/backend/cmd/generate/main.go b/backend/cmd/generate/main.go index d31a730..61d10d7 100644 --- a/backend/cmd/generate/main.go +++ b/backend/cmd/generate/main.go @@ -24,6 +24,7 @@ func main() { licensesFile := "" skipUpdateProviders := false skipUpdateModules := false + namespacePrefix := "" namespace := "" name := "" targetSystem := "" @@ -55,6 +56,7 @@ func main() { flag.StringVar(&licensesFile, "licenses-file", licensesFile, "JSON file containing a list of approved licenses to include when indexing. (required)") flag.BoolVar(&skipUpdateProviders, "skip-update-providers", skipUpdateProviders, "Skip updating provider indexes.") flag.BoolVar(&skipUpdateModules, "skip-update-modules", skipUpdateModules, "Skip updating module indexes.") + flag.StringVar(&namespacePrefix, "namespace-prefix", namespace, "Limit updates to a namespace prefix.") flag.StringVar(&namespace, "namespace", namespace, "Limit updates to a namespace.") flag.StringVar(&name, "name", name, "Limit updates to a name. Only works in conjunction with -namespace. For providers, this will result in a single provider getting updated. For modules, this will update all target systems under a name.") flag.StringVar(&targetSystem, "target-system", targetSystem, "Limit updates to a target system for module updates only. Only works in conjunction with -namespace and -name.") @@ -124,6 +126,7 @@ func main() { if err := backendInstance.Generate( ctx, append(forceOpts, + backend.WithNamespacePrefix(namespacePrefix), backend.WithNamespace(namespace), backend.WithName(name), backend.WithTargetSystem(targetSystem), diff --git a/backend/internal/backend.go b/backend/internal/backend.go index e3d5913..0bb9f95 100644 --- a/backend/internal/backend.go +++ b/backend/internal/backend.go @@ -77,6 +77,7 @@ type GenerateOpt func(c *GenerateConfig) error type GenerateConfig struct { SkipUpdateProviders bool SkipUpdateModules bool + NamespacePrefix string Namespace string Name string TargetSystem string @@ -206,8 +207,24 @@ var namespaceRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") var nameRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") var targetSystemRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") +func WithNamespacePrefix(namespacePrefix string) GenerateOpt { + return func(c *GenerateConfig) error { + if namespacePrefix != "" && c.Namespace != "" { + return fmt.Errorf("filtering for both namespace and namespace prefix is not supported") + } + if !namespaceRe.MatchString(namespacePrefix) { + return fmt.Errorf("invalid namespace: %s", namespaceRe) + } + c.NamespacePrefix = namespacePrefix + return nil + } +} + func WithNamespace(namespace string) GenerateOpt { return func(c *GenerateConfig) error { + if namespace != "" && c.NamespacePrefix != "" { + return fmt.Errorf("filtering for both namespace and namespace prefix is not supported") + } if !namespaceRe.MatchString(namespace) { return fmt.Errorf("invalid namespace: %s", namespaceRe) } @@ -318,6 +335,10 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error { if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) } + } else if cfg.NamespacePrefix != "" { + if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { + return fmt.Errorf("failed to generate modules (%w)", err) + } } else { if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) @@ -333,6 +354,10 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error { if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) } + } else if cfg.NamespacePrefix != "" { + if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { + return fmt.Errorf("failed to index providers (%w)", err) + } } else { if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) diff --git a/backend/internal/moduleindex/generator.go b/backend/internal/moduleindex/generator.go index 027b170..19cd822 100644 --- a/backend/internal/moduleindex/generator.go +++ b/backend/internal/moduleindex/generator.go @@ -47,6 +47,9 @@ const indexPrefix = "modules" type Generator interface { // Generate generates all module index files incrementally and removes items no longer in the registry. Generate(ctx context.Context, opts ...Opts) error + // GenerateNamespacePrefix generates module index files incrementally for all namespaces matching a given + // prefix and removes modules from the index that no longer exist. + GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the // registry. GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error @@ -127,6 +130,26 @@ func (g generator) GenerateNamespaceAndName(ctx context.Context, namespace strin }, opts) } +func (g generator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error { + namespacePrefix = module.NormalizeNamespace(namespacePrefix) + g.log.Info(ctx, "Listing all modules...") + moduleListFull, err := g.metadataAPI.ListModules(ctx) + if err != nil { + return err + } + + var moduleList []module.Addr + for _, module := range moduleListFull { + if strings.HasPrefix(module.Namespace, namespacePrefix) { + moduleList = append(moduleList, module) + } + } + + return g.generate(ctx, moduleList, func(moduleAddr ModuleAddr) bool { + return !strings.HasPrefix(moduleAddr.Namespace, namespacePrefix) + }, opts) +} + func (g generator) GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error { namespace = module.NormalizeNamespace(namespace) g.log.Info(ctx, "Listing all modules...") diff --git a/backend/internal/providerindex/generator.go b/backend/internal/providerindex/generator.go index c7bf0cd..c40f5de 100644 --- a/backend/internal/providerindex/generator.go +++ b/backend/internal/providerindex/generator.go @@ -28,10 +28,13 @@ type DocumentationGenerator interface { // Generate generates all module index files incrementally and removes items no longer in the registry. Generate(ctx context.Context, opts ...Opts) error - // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the - // registry. + // GenerateNamespace generates provider index files incrementally for one namespace. GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error + // GenerateNamespacePrefix generates provider index files incrementally for multiple namespaces matching the given + // prefix. + GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error + // GenerateSingleProvider generates module index files for a single provider only. GenerateSingleProvider(ctx context.Context, addr provider.Addr, opts ...Opts) error } @@ -117,9 +120,29 @@ func (d *documentationGenerator) Generate(ctx context.Context, opts ...Opts) err func (d *documentationGenerator) GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error { d.log.Info(ctx, "Listing all providers in namespace %s...", namespace) providerList, err := d.metadataAPI.ListProvidersByNamespace(ctx, namespace, true) + + d.log.Info(ctx, "Loaded %d providers", len(providerList)) + + err = d.scrape(ctx, providerList, opts) + if err != nil { + return err + } + + return nil +} + +func (d *documentationGenerator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error { + d.log.Info(ctx, "Listing all providers with the namespace prefix %s...", namespacePrefix) + providerListFull, err := d.metadataAPI.ListProviders(ctx, true) if err != nil { return err } + var providerList []provider.Addr + for _, providerAddr := range providerListFull { + if strings.HasPrefix(providerAddr.Namespace, namespacePrefix) { + providerList = append(providerList, providerAddr) + } + } d.log.Info(ctx, "Loaded %d providers", len(providerList)) err = d.scrape(ctx, providerList, opts)