Force repo data regeneration

Signed-off-by: AbstractionFactory <[email protected]>

.github/workflows/index-manual.yml

@@ -22,6 +22,11 @@ on:
         required: false
         default: ""
         description: "Force regenerating a namespace, name, or target system. This parameter is a comma-separate list consisting of either a namespace, a namespace and a name separated by a /, or a namespace, name and target system separated by a /. Example: namespace/name/targetsystem,othernamespace/othername"
+      force_repo_data_update:
+        type: boolean
+        required: false
+        default: false
+        description: Force updating GitHub repository data even if no new version was found. (Use with care!)
 concurrency:
   group: index
   cancel-in-progress: false
@@ -57,8 +62,9 @@ jobs:
           TARGET_SYSTEM: ${{inputs.target_system}}
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
           FORCE_REGENERATE: ${{inputs.force_regenerate}}
+          FORCE_REPO_DATA_UPDATE_ARG: ${{inputs.force_repo_data_update && "--force-repo-data-update" || ""}}
         run: |
-          go run github.com/opentofu/registry-ui/cmd/generate --log-level=trace --s3-bucket=${S3_BUCKET} --blocklist ../blocklist.json --licenses-file ../licenses.json --namespace="${NAMESPACE}" --name="${NAME}" --target-system="${TARGET_SYSTEM}" --force-regenerate="${FORCE_REGENERATE}"
+          go run github.com/opentofu/registry-ui/cmd/generate --log-level=trace --s3-bucket=${S3_BUCKET} --blocklist ../blocklist.json --licenses-file ../licenses.json --namespace="${NAMESPACE}" --name="${NAME}" --target-system="${TARGET_SYSTEM}" --force-regenerate="${FORCE_REGENERATE}" "${FORCE_REPO_DATA_UPDATE_ARG}"
       - name: Update search index
         working-directory: search/pg-indexer
         env:

backend/cmd/generate/main.go

@@ -30,6 +30,7 @@ func main() {
 	targetSystem := ""
 	logLevel := "info"
 	forceRegenerate := ""
+	forceRepoDataUpdate := false
 	registryDir := defaults.RegistryDir
 	workDir := defaults.WorkDir
 	destinationDir := defaults.DestinationDir
@@ -71,6 +72,7 @@ func main() {
 	flag.StringVar(&s3Params.Region, "s3-region", s3Params.Region, "Region to use for S3 uploads.")
 	flag.IntVar(&commitParallelism, "commit-parallelism", commitParallelism, "Parallel uploads to use on commit.")
 	flag.StringVar(&tofuBinaryPath, "tofu-binary-path", tofuBinaryPath, "Temporary: Tofu binary path to use for module schema extraction.")
+	flag.BoolVar(&forceRepoDataUpdate, "force-repodata-update", forceRepoDataUpdate, "Force updating the repository metadata regardless of new versions added.")
 	flag.StringVar(&forceRegenerate, "force-regenerate", forceRegenerate, "Force regenerating a namespace, name, or target system. This parameter is a comma-separate list consisting of either a namespace, a namespace and a name separated by a /, or a namespace, name and target system separated by a /. Example: namespace/name/targetsystem,othernamespace/othername")
 	flag.StringVar(&blockListFile, "blocklist", blockListFile, "File containing the blocklist to use.")
 	flag.Parse()
@@ -114,7 +116,17 @@ func main() {
 		os.Exit(1)
 	}
 
-	backendInstance, err := backendFactory.Create(ctx, registryDir, workDir, destinationDir, blockList, s3Params, commitParallelism, tofuBinaryPath, approvedLicenses)
+	backendInstance, err := backendFactory.Create(
+		ctx,
+		registryDir,
+		workDir,
+		destinationDir,
+		blockList,
+		s3Params,
+		commitParallelism,
+		tofuBinaryPath,
+		approvedLicenses,
+	)
 	if err != nil {
 		mainLogger.Error(ctx, err.Error())
 		os.Exit(1)
@@ -137,6 +149,7 @@ func main() {
 			backend.WithTargetSystem(targetSystem),
 			backend.WithSkipUpdateModules(skipUpdateModules),
 			backend.WithSkipUpdateProviders(skipUpdateProviders),
+			backend.WithForceRepoDataUpdate(forceRepoDataUpdate),
 		)...,
 	); err != nil {
 		mainLogger.Error(ctx, err.Error())

backend/internal/backend.go

@@ -81,6 +81,7 @@ type GenerateConfig struct {
 	Namespace           string
 	Name                string
 	TargetSystem        string
+	ForceRepoDataUpdate bool
 	ForceRegenerate     ForceRegenerateType
 }
 
@@ -196,6 +197,13 @@ func WithSkipUpdateProviders(skip bool) GenerateOpt {
 	}
 }
 
+func WithForceRepoDataUpdate(force bool) GenerateOpt {
+	return func(c *GenerateConfig) error {
+		c.ForceRepoDataUpdate = force
+		return nil
+	}
+}
+
 func WithSkipUpdateModules(skip bool) GenerateOpt {
 	return func(c *GenerateConfig) error {
 		c.SkipUpdateModules = skip
@@ -328,38 +336,38 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error {
 				return fmt.Errorf("failed to generate modules (%w)", err)
 			}
 		} else if cfg.Name != "" {
-			if err := b.moduleIndexGenerator.GenerateNamespaceAndName(ctx, cfg.Namespace, cfg.Name, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.moduleIndexGenerator.GenerateNamespaceAndName(ctx, cfg.Namespace, cfg.Name, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to generate modules (%w)", err)
 			}
 		} else if cfg.Namespace != "" {
-			if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to generate modules (%w)", err)
 			}
 		} else if cfg.NamespacePrefix != "" {
-			if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to generate modules (%w)", err)
 			}
 		} else {
-			if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to generate modules (%w)", err)
 			}
 		}
 	}
 	if !cfg.SkipUpdateProviders && cfg.TargetSystem == "" {
 		if cfg.Name != "" {
-			if err := b.providerIndexGenerator.GenerateSingleProvider(ctx, provider.Addr{Namespace: cfg.Namespace, Name: cfg.Name}, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.providerIndexGenerator.GenerateSingleProvider(ctx, provider.Addr{Namespace: cfg.Namespace, Name: cfg.Name}, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to index providers (%w)", err)
 			}
 		} else if cfg.Namespace != "" {
-			if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to index providers (%w)", err)
 			}
 		} else if cfg.NamespacePrefix != "" {
-			if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to index providers (%w)", err)
 			}
 		} else {
-			if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
+			if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
 				return fmt.Errorf("failed to index providers (%w)", err)
 			}
 		}

backend/internal/moduleindex/generator.go

@@ -61,7 +61,8 @@ type Generator interface {
 }
 
 type GenerateConfig struct {
-	Force ForceRegenerate
+	Force               ForceRegenerate
+	ForceRepoDataUpdate bool
 }
 
 type noForce struct {
@@ -87,6 +88,13 @@ func WithForce(force ForceRegenerate) Opts {
 	}
 }
 
+func WithForceRepoDataUpdate(force bool) Opts {
+	return func(_ context.Context, generateConfig *GenerateConfig) error {
+		generateConfig.ForceRepoDataUpdate = force
+		return nil
+	}
+}
+
 func New(log logger.Logger, metadataAPI metadata.API, vcsClient vcs.Client, licenseDetector license.Detector, storage indexstorage.API, moduleSchemaExtractor moduleschema.Extractor, searchAPI search.API, blocklist blocklist.BlockList) Generator {
 	return &generator{
 		log:                   log.WithName("Module indexer"),
@@ -260,6 +268,11 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block
 
 			repoInfoFetched := false
 
+			if cfg.ForceRepoDataUpdate {
+				g.fetchRepoInfo(ctx, entry)
+				repoInfoFetched = true
+			}
+
 			for _, ver := range metadataVersions {
 				if err := ver.Validate(); err != nil {
 					g.log.Warn(ctx, "Module %s version %s has an invalid version number, skipping...", moduleAddr.String(), ver.Version)

backend/internal/providerindex/generator.go

@@ -40,7 +40,8 @@ type DocumentationGenerator interface {
 }
 
 type GenerateConfig struct {
-	Force ForceRegenerate
+	Force               ForceRegenerate
+	ForceRepoDataUpdate bool
 }
 
 type noForce struct {
@@ -66,6 +67,13 @@ func WithForce(force ForceRegenerate) Opts {
 	}
 }
 
+func WithForceRepoDataUpdate(force bool) Opts {
+	return func(_ context.Context, generateConfig *GenerateConfig) error {
+		generateConfig.ForceRepoDataUpdate = force
+		return nil
+	}
+}
+
 func NewDocumentationGenerator(log logger.Logger, metadataAPI metadata.API, vcsClient vcs.Client, licenseDetector license.Detector, source providerdocsource.API, destination providerindexstorage.API, searchAPI search.API, blocklist blocklist.BlockList) DocumentationGenerator {
 	return &documentationGenerator{
 		log:             log.WithName("Provider indexer"),
@@ -278,6 +286,11 @@ func (d *documentationGenerator) scrapeProvider(ctx context.Context, addr provid
 
 	providerData.Warnings = meta.Warnings
 
+	if cfg.ForceRepoDataUpdate {
+		d.extractRepoInfo(ctx, addr, providerData)
+		repoInfoFetched = true
+	}
+
 	for _, version := range meta.Versions {
 		if err := version.Version.Validate(); err != nil {
 			d.log.Warn(ctx, "Invalid version number for provider %s: %s, skipping... (%v)", addr, version.Version, err)