From 31b960ca3069671be877aa0b7e396f2f5e28c627 Mon Sep 17 00:00:00 2001 From: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:06:22 +0100 Subject: [PATCH] Force repo data regeneration Signed-off-by: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com> --- .github/workflows/index-manual.yml | 8 ++++++- backend/cmd/generate/main.go | 15 ++++++++++++- backend/internal/backend.go | 24 ++++++++++++++------- backend/internal/moduleindex/generator.go | 15 ++++++++++++- backend/internal/providerindex/generator.go | 15 ++++++++++++- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/.github/workflows/index-manual.yml b/.github/workflows/index-manual.yml index ab5e59e..b77e894 100644 --- a/.github/workflows/index-manual.yml +++ b/.github/workflows/index-manual.yml @@ -22,6 +22,11 @@ on: required: false default: "" description: "Force regenerating a namespace, name, or target system. This parameter is a comma-separate list consisting of either a namespace, a namespace and a name separated by a /, or a namespace, name and target system separated by a /. Example: namespace/name/targetsystem,othernamespace/othername" + force_repo_data_update: + type: boolean + required: false + default: false + description: Force updating GitHub repository data even if no new version was found. (Use with care!) concurrency: group: index cancel-in-progress: false @@ -57,8 +62,9 @@ jobs: TARGET_SYSTEM: ${{inputs.target_system}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} FORCE_REGENERATE: ${{inputs.force_regenerate}} + FORCE_REPO_DATA_UPDATE_ARG: ${{inputs.force_repo_data_update && "--force-repo-data-update" || ""}} run: | - go run github.com/opentofu/registry-ui/cmd/generate --log-level=trace --s3-bucket=${S3_BUCKET} --blocklist ../blocklist.json --licenses-file ../licenses.json --namespace="${NAMESPACE}" --name="${NAME}" --target-system="${TARGET_SYSTEM}" --force-regenerate="${FORCE_REGENERATE}" + go run github.com/opentofu/registry-ui/cmd/generate --log-level=trace --s3-bucket=${S3_BUCKET} --blocklist ../blocklist.json --licenses-file ../licenses.json --namespace="${NAMESPACE}" --name="${NAME}" --target-system="${TARGET_SYSTEM}" --force-regenerate="${FORCE_REGENERATE}" "${FORCE_REPO_DATA_UPDATE_ARG}" - name: Update search index working-directory: search/pg-indexer env: diff --git a/backend/cmd/generate/main.go b/backend/cmd/generate/main.go index 3b75826..c71e903 100644 --- a/backend/cmd/generate/main.go +++ b/backend/cmd/generate/main.go @@ -30,6 +30,7 @@ func main() { targetSystem := "" logLevel := "info" forceRegenerate := "" + forceRepoDataUpdate := false registryDir := defaults.RegistryDir workDir := defaults.WorkDir destinationDir := defaults.DestinationDir @@ -71,6 +72,7 @@ func main() { flag.StringVar(&s3Params.Region, "s3-region", s3Params.Region, "Region to use for S3 uploads.") flag.IntVar(&commitParallelism, "commit-parallelism", commitParallelism, "Parallel uploads to use on commit.") flag.StringVar(&tofuBinaryPath, "tofu-binary-path", tofuBinaryPath, "Temporary: Tofu binary path to use for module schema extraction.") + flag.BoolVar(&forceRepoDataUpdate, "force-repodata-update", forceRepoDataUpdate, "Force updating the repository metadata regardless of new versions added.") flag.StringVar(&forceRegenerate, "force-regenerate", forceRegenerate, "Force regenerating a namespace, name, or target system. This parameter is a comma-separate list consisting of either a namespace, a namespace and a name separated by a /, or a namespace, name and target system separated by a /. Example: namespace/name/targetsystem,othernamespace/othername") flag.StringVar(&blockListFile, "blocklist", blockListFile, "File containing the blocklist to use.") flag.Parse() @@ -114,7 +116,17 @@ func main() { os.Exit(1) } - backendInstance, err := backendFactory.Create(ctx, registryDir, workDir, destinationDir, blockList, s3Params, commitParallelism, tofuBinaryPath, approvedLicenses) + backendInstance, err := backendFactory.Create( + ctx, + registryDir, + workDir, + destinationDir, + blockList, + s3Params, + commitParallelism, + tofuBinaryPath, + approvedLicenses, + ) if err != nil { mainLogger.Error(ctx, err.Error()) os.Exit(1) @@ -137,6 +149,7 @@ func main() { backend.WithTargetSystem(targetSystem), backend.WithSkipUpdateModules(skipUpdateModules), backend.WithSkipUpdateProviders(skipUpdateProviders), + backend.WithForceRepoDataUpdate(forceRepoDataUpdate), )..., ); err != nil { mainLogger.Error(ctx, err.Error()) diff --git a/backend/internal/backend.go b/backend/internal/backend.go index 0bb9f95..c3887ad 100644 --- a/backend/internal/backend.go +++ b/backend/internal/backend.go @@ -81,6 +81,7 @@ type GenerateConfig struct { Namespace string Name string TargetSystem string + ForceRepoDataUpdate bool ForceRegenerate ForceRegenerateType } @@ -196,6 +197,13 @@ func WithSkipUpdateProviders(skip bool) GenerateOpt { } } +func WithForceRepoDataUpdate(force bool) GenerateOpt { + return func(c *GenerateConfig) error { + c.ForceRepoDataUpdate = force + return nil + } +} + func WithSkipUpdateModules(skip bool) GenerateOpt { return func(c *GenerateConfig) error { c.SkipUpdateModules = skip @@ -328,38 +336,38 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error { return fmt.Errorf("failed to generate modules (%w)", err) } } else if cfg.Name != "" { - if err := b.moduleIndexGenerator.GenerateNamespaceAndName(ctx, cfg.Namespace, cfg.Name, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.moduleIndexGenerator.GenerateNamespaceAndName(ctx, cfg.Namespace, cfg.Name, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) } } else if cfg.Namespace != "" { - if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) } } else if cfg.NamespacePrefix != "" { - if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) } } else { - if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) } } } if !cfg.SkipUpdateProviders && cfg.TargetSystem == "" { if cfg.Name != "" { - if err := b.providerIndexGenerator.GenerateSingleProvider(ctx, provider.Addr{Namespace: cfg.Namespace, Name: cfg.Name}, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.providerIndexGenerator.GenerateSingleProvider(ctx, provider.Addr{Namespace: cfg.Namespace, Name: cfg.Name}, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) } } else if cfg.Namespace != "" { - if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) } } else if cfg.NamespacePrefix != "" { - if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) } } else { - if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { + if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) } } diff --git a/backend/internal/moduleindex/generator.go b/backend/internal/moduleindex/generator.go index 0a55f64..49580d9 100644 --- a/backend/internal/moduleindex/generator.go +++ b/backend/internal/moduleindex/generator.go @@ -61,7 +61,8 @@ type Generator interface { } type GenerateConfig struct { - Force ForceRegenerate + Force ForceRegenerate + ForceRepoDataUpdate bool } type noForce struct { @@ -87,6 +88,13 @@ func WithForce(force ForceRegenerate) Opts { } } +func WithForceRepoDataUpdate(force bool) Opts { + return func(_ context.Context, generateConfig *GenerateConfig) error { + generateConfig.ForceRepoDataUpdate = force + return nil + } +} + func New(log logger.Logger, metadataAPI metadata.API, vcsClient vcs.Client, licenseDetector license.Detector, storage indexstorage.API, moduleSchemaExtractor moduleschema.Extractor, searchAPI search.API, blocklist blocklist.BlockList) Generator { return &generator{ log: log.WithName("Module indexer"), @@ -260,6 +268,11 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block repoInfoFetched := false + if cfg.ForceRepoDataUpdate { + g.fetchRepoInfo(ctx, entry) + repoInfoFetched = true + } + for _, ver := range metadataVersions { if err := ver.Validate(); err != nil { g.log.Warn(ctx, "Module %s version %s has an invalid version number, skipping...", moduleAddr.String(), ver.Version) diff --git a/backend/internal/providerindex/generator.go b/backend/internal/providerindex/generator.go index 06320ae..ab7fb15 100644 --- a/backend/internal/providerindex/generator.go +++ b/backend/internal/providerindex/generator.go @@ -40,7 +40,8 @@ type DocumentationGenerator interface { } type GenerateConfig struct { - Force ForceRegenerate + Force ForceRegenerate + ForceRepoDataUpdate bool } type noForce struct { @@ -66,6 +67,13 @@ func WithForce(force ForceRegenerate) Opts { } } +func WithForceRepoDataUpdate(force bool) Opts { + return func(_ context.Context, generateConfig *GenerateConfig) error { + generateConfig.ForceRepoDataUpdate = force + return nil + } +} + func NewDocumentationGenerator(log logger.Logger, metadataAPI metadata.API, vcsClient vcs.Client, licenseDetector license.Detector, source providerdocsource.API, destination providerindexstorage.API, searchAPI search.API, blocklist blocklist.BlockList) DocumentationGenerator { return &documentationGenerator{ log: log.WithName("Provider indexer"), @@ -278,6 +286,11 @@ func (d *documentationGenerator) scrapeProvider(ctx context.Context, addr provid providerData.Warnings = meta.Warnings + if cfg.ForceRepoDataUpdate { + d.extractRepoInfo(ctx, addr, providerData) + repoInfoFetched = true + } + for _, version := range meta.Versions { if err := version.Version.Validate(); err != nil { d.log.Warn(ctx, "Invalid version number for provider %s: %s, skipping... (%v)", addr, version.Version, err)