Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Force repo data regeneration #262

Merged
merged 1 commit into from
Dec 2, 2024
Merged

Force repo data regeneration #262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .github/workflows/index-manual.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ on:
required: false
default: ""
description: "Force regenerating a namespace, name, or target system. This parameter is a comma-separate list consisting of either a namespace, a namespace and a name separated by a /, or a namespace, name and target system separated by a /. Example: namespace/name/targetsystem,othernamespace/othername"
force_repo_data_update:
type: boolean
required: false
default: false
description: Force updating GitHub repository data even if no new version was found. (Use with care!)
concurrency:
group: index
cancel-in-progress: false
Expand Down Expand Up @@ -57,8 +62,9 @@ jobs:
TARGET_SYSTEM: ${{inputs.target_system}}
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
FORCE_REGENERATE: ${{inputs.force_regenerate}}
FORCE_REPO_DATA_UPDATE_ARG: ${{inputs.force_repo_data_update && "--force-repo-data-update" || ""}}
run: |
go run github.com/opentofu/registry-ui/cmd/generate --log-level=trace --s3-bucket=${S3_BUCKET} --blocklist ../blocklist.json --licenses-file ../licenses.json --namespace="${NAMESPACE}" --name="${NAME}" --target-system="${TARGET_SYSTEM}" --force-regenerate="${FORCE_REGENERATE}"
go run github.com/opentofu/registry-ui/cmd/generate --log-level=trace --s3-bucket=${S3_BUCKET} --blocklist ../blocklist.json --licenses-file ../licenses.json --namespace="${NAMESPACE}" --name="${NAME}" --target-system="${TARGET_SYSTEM}" --force-regenerate="${FORCE_REGENERATE}" "${FORCE_REPO_DATA_UPDATE_ARG}"
- name: Update search index
working-directory: search/pg-indexer
env:
Expand Down
15 changes: 14 additions & 1 deletion backend/cmd/generate/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func main() {
targetSystem := ""
logLevel := "info"
forceRegenerate := ""
forceRepoDataUpdate := false
registryDir := defaults.RegistryDir
workDir := defaults.WorkDir
destinationDir := defaults.DestinationDir
Expand Down Expand Up @@ -71,6 +72,7 @@ func main() {
flag.StringVar(&s3Params.Region, "s3-region", s3Params.Region, "Region to use for S3 uploads.")
flag.IntVar(&commitParallelism, "commit-parallelism", commitParallelism, "Parallel uploads to use on commit.")
flag.StringVar(&tofuBinaryPath, "tofu-binary-path", tofuBinaryPath, "Temporary: Tofu binary path to use for module schema extraction.")
flag.BoolVar(&forceRepoDataUpdate, "force-repodata-update", forceRepoDataUpdate, "Force updating the repository metadata regardless of new versions added.")
flag.StringVar(&forceRegenerate, "force-regenerate", forceRegenerate, "Force regenerating a namespace, name, or target system. This parameter is a comma-separate list consisting of either a namespace, a namespace and a name separated by a /, or a namespace, name and target system separated by a /. Example: namespace/name/targetsystem,othernamespace/othername")
flag.StringVar(&blockListFile, "blocklist", blockListFile, "File containing the blocklist to use.")
flag.Parse()
Expand Down Expand Up @@ -114,7 +116,17 @@ func main() {
os.Exit(1)
}

backendInstance, err := backendFactory.Create(ctx, registryDir, workDir, destinationDir, blockList, s3Params, commitParallelism, tofuBinaryPath, approvedLicenses)
backendInstance, err := backendFactory.Create(
ctx,
registryDir,
workDir,
destinationDir,
blockList,
s3Params,
commitParallelism,
tofuBinaryPath,
approvedLicenses,
)
if err != nil {
mainLogger.Error(ctx, err.Error())
os.Exit(1)
Expand All @@ -137,6 +149,7 @@ func main() {
backend.WithTargetSystem(targetSystem),
backend.WithSkipUpdateModules(skipUpdateModules),
backend.WithSkipUpdateProviders(skipUpdateProviders),
backend.WithForceRepoDataUpdate(forceRepoDataUpdate),
)...,
); err != nil {
mainLogger.Error(ctx, err.Error())
Expand Down
24 changes: 16 additions & 8 deletions backend/internal/backend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ type GenerateConfig struct {
Namespace string
Name string
TargetSystem string
ForceRepoDataUpdate bool
ForceRegenerate ForceRegenerateType
}

Expand Down Expand Up @@ -196,6 +197,13 @@ func WithSkipUpdateProviders(skip bool) GenerateOpt {
}
}

func WithForceRepoDataUpdate(force bool) GenerateOpt {
return func(c *GenerateConfig) error {
c.ForceRepoDataUpdate = force
return nil
}
}

func WithSkipUpdateModules(skip bool) GenerateOpt {
return func(c *GenerateConfig) error {
c.SkipUpdateModules = skip
Expand Down Expand Up @@ -328,38 +336,38 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error {
return fmt.Errorf("failed to generate modules (%w)", err)
}
} else if cfg.Name != "" {
if err := b.moduleIndexGenerator.GenerateNamespaceAndName(ctx, cfg.Namespace, cfg.Name, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.moduleIndexGenerator.GenerateNamespaceAndName(ctx, cfg.Namespace, cfg.Name, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to generate modules (%w)", err)
}
} else if cfg.Namespace != "" {
if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to generate modules (%w)", err)
}
} else if cfg.NamespacePrefix != "" {
if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to generate modules (%w)", err)
}
} else {
if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate), moduleindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to generate modules (%w)", err)
}
}
}
if !cfg.SkipUpdateProviders && cfg.TargetSystem == "" {
if cfg.Name != "" {
if err := b.providerIndexGenerator.GenerateSingleProvider(ctx, provider.Addr{Namespace: cfg.Namespace, Name: cfg.Name}, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.providerIndexGenerator.GenerateSingleProvider(ctx, provider.Addr{Namespace: cfg.Namespace, Name: cfg.Name}, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to index providers (%w)", err)
}
} else if cfg.Namespace != "" {
if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to index providers (%w)", err)
}
} else if cfg.NamespacePrefix != "" {
if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to index providers (%w)", err)
}
} else {
if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate)); err != nil {
if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate), providerindex.WithForceRepoDataUpdate(cfg.ForceRepoDataUpdate)); err != nil {
return fmt.Errorf("failed to index providers (%w)", err)
}
}
Expand Down
15 changes: 14 additions & 1 deletion backend/internal/moduleindex/generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ type Generator interface {
}

type GenerateConfig struct {
Force ForceRegenerate
Force ForceRegenerate
ForceRepoDataUpdate bool
}

type noForce struct {
Expand All @@ -87,6 +88,13 @@ func WithForce(force ForceRegenerate) Opts {
}
}

func WithForceRepoDataUpdate(force bool) Opts {
return func(_ context.Context, generateConfig *GenerateConfig) error {
generateConfig.ForceRepoDataUpdate = force
return nil
}
}

func New(log logger.Logger, metadataAPI metadata.API, vcsClient vcs.Client, licenseDetector license.Detector, storage indexstorage.API, moduleSchemaExtractor moduleschema.Extractor, searchAPI search.API, blocklist blocklist.BlockList) Generator {
return &generator{
log: log.WithName("Module indexer"),
Expand Down Expand Up @@ -260,6 +268,11 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block

repoInfoFetched := false

if cfg.ForceRepoDataUpdate {
g.fetchRepoInfo(ctx, entry)
repoInfoFetched = true
}

for _, ver := range metadataVersions {
if err := ver.Validate(); err != nil {
g.log.Warn(ctx, "Module %s version %s has an invalid version number, skipping...", moduleAddr.String(), ver.Version)
Expand Down
15 changes: 14 additions & 1 deletion backend/internal/providerindex/generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ type DocumentationGenerator interface {
}

type GenerateConfig struct {
Force ForceRegenerate
Force ForceRegenerate
ForceRepoDataUpdate bool
}

type noForce struct {
Expand All @@ -66,6 +67,13 @@ func WithForce(force ForceRegenerate) Opts {
}
}

func WithForceRepoDataUpdate(force bool) Opts {
return func(_ context.Context, generateConfig *GenerateConfig) error {
generateConfig.ForceRepoDataUpdate = force
return nil
}
}

func NewDocumentationGenerator(log logger.Logger, metadataAPI metadata.API, vcsClient vcs.Client, licenseDetector license.Detector, source providerdocsource.API, destination providerindexstorage.API, searchAPI search.API, blocklist blocklist.BlockList) DocumentationGenerator {
return &documentationGenerator{
log: log.WithName("Provider indexer"),
Expand Down Expand Up @@ -278,6 +286,11 @@ func (d *documentationGenerator) scrapeProvider(ctx context.Context, addr provid

providerData.Warnings = meta.Warnings

if cfg.ForceRepoDataUpdate {
d.extractRepoInfo(ctx, addr, providerData)
repoInfoFetched = true
}

for _, version := range meta.Versions {
if err := version.Version.Validate(); err != nil {
d.log.Warn(ctx, "Invalid version number for provider %s: %s, skipping... (%v)", addr, version.Version, err)
Expand Down
Loading