Add namespacePrefix for more efficient population

backend/cmd/generate/main.go

@@ -24,6 +24,7 @@ func main() {
 	licensesFile := ""
 	skipUpdateProviders := false
 	skipUpdateModules := false
+	namespacePrefix := ""
 	namespace := ""
 	name := ""
 	targetSystem := ""
@@ -55,6 +56,7 @@ func main() {
 	flag.StringVar(&licensesFile, "licenses-file", licensesFile, "JSON file containing a list of approved licenses to include when indexing. (required)")
 	flag.BoolVar(&skipUpdateProviders, "skip-update-providers", skipUpdateProviders, "Skip updating provider indexes.")
 	flag.BoolVar(&skipUpdateModules, "skip-update-modules", skipUpdateModules, "Skip updating module indexes.")
+	flag.StringVar(&namespacePrefix, "namespace-prefix", namespace, "Limit updates to a namespace prefix.")
 	flag.StringVar(&namespace, "namespace", namespace, "Limit updates to a namespace.")
 	flag.StringVar(&name, "name", name, "Limit updates to a name. Only works in conjunction with -namespace. For providers, this will result in a single provider getting updated. For modules, this will update all target systems under a name.")
 	flag.StringVar(&targetSystem, "target-system", targetSystem, "Limit updates to a target system for module updates only. Only works in conjunction with -namespace and -name.")
@@ -124,6 +126,7 @@ func main() {
 	if err := backendInstance.Generate(
 		ctx,
 		append(forceOpts,
+			backend.WithNamespacePrefix(namespacePrefix),
 			backend.WithNamespace(namespace),
 			backend.WithName(name),
 			backend.WithTargetSystem(targetSystem),

backend/internal/backend.go

@@ -77,6 +77,7 @@ type GenerateOpt func(c *GenerateConfig) error
 type GenerateConfig struct {
 	SkipUpdateProviders bool
 	SkipUpdateModules   bool
+	NamespacePrefix     string
 	Namespace           string
 	Name                string
 	TargetSystem        string
@@ -206,8 +207,24 @@ var namespaceRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$")
 var nameRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$")
 var targetSystemRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$")
 
+func WithNamespacePrefix(namespacePrefix string) GenerateOpt {
+	return func(c *GenerateConfig) error {
+		if namespacePrefix != "" && c.Namespace != "" {
+			return fmt.Errorf("filtering for both namespace and namespace prefix is not supported")
+		}
+		if !namespaceRe.MatchString(namespacePrefix) {
+			return fmt.Errorf("invalid namespace: %s", namespaceRe)
+		}
+		c.NamespacePrefix = namespacePrefix
+		return nil
+	}
+}
+
 func WithNamespace(namespace string) GenerateOpt {
 	return func(c *GenerateConfig) error {
+		if namespace != "" && c.NamespacePrefix != "" {
+			return fmt.Errorf("filtering for both namespace and namespace prefix is not supported")
+		}
 		if !namespaceRe.MatchString(namespace) {
 			return fmt.Errorf("invalid namespace: %s", namespaceRe)
 		}
@@ -318,6 +335,10 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error {
 			if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
 				return fmt.Errorf("failed to generate modules (%w)", err)
 			}
+		} else if cfg.NamespacePrefix != "" {
+			if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
+				return fmt.Errorf("failed to generate modules (%w)", err)
+			}
 		} else {
 			if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
 				return fmt.Errorf("failed to generate modules (%w)", err)
@@ -333,6 +354,10 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error {
 			if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
 				return fmt.Errorf("failed to index providers (%w)", err)
 			}
+		} else if cfg.NamespacePrefix != "" {
+			if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
+				return fmt.Errorf("failed to index providers (%w)", err)
+			}
 		} else {
 			if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
 				return fmt.Errorf("failed to index providers (%w)", err)

backend/internal/moduleindex/generator.go

@@ -47,6 +47,9 @@ const indexPrefix = "modules"
 type Generator interface {
 	// Generate generates all module index files incrementally and removes items no longer in the registry.
 	Generate(ctx context.Context, opts ...Opts) error
+	// GenerateNamespacePrefix generates module index files incrementally for all namespaces matching a given
+	// prefix and removes modules from the index that no longer exist.
+	GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error
 	// GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the
 	// registry.
 	GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error
@@ -127,6 +130,26 @@ func (g generator) GenerateNamespaceAndName(ctx context.Context, namespace strin
 	}, opts)
 }
 
+func (g generator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error {
+	namespacePrefix = module.NormalizeNamespace(namespacePrefix)
+	g.log.Info(ctx, "Listing all modules...")
+	moduleListFull, err := g.metadataAPI.ListModules(ctx)
+	if err != nil {
+		return err
+	}
+
+	var moduleList []module.Addr
+	for _, module := range moduleListFull {
+		if strings.HasPrefix(module.Namespace, namespacePrefix) {
+			moduleList = append(moduleList, module)
+		}
+	}
+
+	return g.generate(ctx, moduleList, func(moduleAddr ModuleAddr) bool {
+		return !strings.HasPrefix(moduleAddr.Namespace, namespacePrefix)
+	}, opts)
+}
+
 func (g generator) GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error {
 	namespace = module.NormalizeNamespace(namespace)
 	g.log.Info(ctx, "Listing all modules...")

backend/internal/providerindex/generator.go

@@ -28,10 +28,13 @@ type DocumentationGenerator interface {
 	// Generate generates all module index files incrementally and removes items no longer in the registry.
 	Generate(ctx context.Context, opts ...Opts) error
 
-	// GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the
-	// registry.
+	// GenerateNamespace generates provider index files incrementally for one namespace.
 	GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error
 
+	// GenerateNamespacePrefix generates provider index files incrementally for multiple namespaces matching the given
+	// prefix.
+	GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error
+
 	// GenerateSingleProvider generates module index files for a single provider only.
 	GenerateSingleProvider(ctx context.Context, addr provider.Addr, opts ...Opts) error
 }
@@ -117,9 +120,29 @@ func (d *documentationGenerator) Generate(ctx context.Context, opts ...Opts) err
 func (d *documentationGenerator) GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error {
 	d.log.Info(ctx, "Listing all providers in namespace %s...", namespace)
 	providerList, err := d.metadataAPI.ListProvidersByNamespace(ctx, namespace, true)
+
+	d.log.Info(ctx, "Loaded %d providers", len(providerList))
+
+	err = d.scrape(ctx, providerList, opts)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *documentationGenerator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error {
+	d.log.Info(ctx, "Listing all providers with the namespace prefix %s...", namespacePrefix)
+	providerListFull, err := d.metadataAPI.ListProviders(ctx, true)
 	if err != nil {
 		return err
 	}
+	var providerList []provider.Addr
+	for _, providerAddr := range providerListFull {
+		if strings.HasPrefix(providerAddr.Namespace, namespacePrefix) {
+			providerList = append(providerList, providerAddr)
+		}
+	}
 	d.log.Info(ctx, "Loaded %d providers", len(providerList))
 
 	err = d.scrape(ctx, providerList, opts)