From 5b2a401a20dd8bac4e474cedacc31cf9d8c4d1a2 Mon Sep 17 00:00:00 2001 From: Christian Mesh Date: Fri, 8 Dec 2023 10:25:58 -0500 Subject: [PATCH 1/2] git ls-remote is faster than checking tags However for providers, we should also check the rss feed for the latest entries as it's a more expensive check. Signed-off-by: Christian Mesh --- src/cmd/bump-versions/main.go | 4 +- src/internal/module/build.go | 10 ----- src/internal/module/update.go | 73 --------------------------------- src/internal/provider/build.go | 14 +++++++ src/internal/provider/update.go | 11 +---- 5 files changed, 17 insertions(+), 95 deletions(-) delete mode 100644 src/internal/module/update.go diff --git a/src/cmd/bump-versions/main.go b/src/cmd/bump-versions/main.go index f258b81083..82a0e35dad 100644 --- a/src/cmd/bump-versions/main.go +++ b/src/cmd/bump-versions/main.go @@ -36,7 +36,7 @@ func main() { os.Exit(1) } - err = modules.Parallel(20, func(m module.Module) error { + err = modules.Parallel(200, func(m module.Module) error { return m.UpdateMetadataFile() }) if err != nil { @@ -49,7 +49,7 @@ func main() { logger.Error("Failed to list providers", slog.Any("err", err)) os.Exit(1) } - err = providers.Parallel(20, func(p provider.Provider) error { + err = providers.Parallel(200, func(p provider.Provider) error { return p.UpdateMetadataFile() }) if err != nil { diff --git a/src/internal/module/build.go b/src/internal/module/build.go index 5ffcec2763..79b879414c 100644 --- a/src/internal/module/build.go +++ b/src/internal/module/build.go @@ -13,16 +13,6 @@ import ( func (m Module) UpdateMetadataFile() error { m.Logger.Info("Beginning version bump process for module", slog.String("module", m.Namespace+"/"+m.Name+"/"+m.TargetSystem)) - shouldUpdate, err := m.shouldUpdateMetadataFile() - if err != nil { - m.Logger.Error("Failed to determine update status", slog.Any("err", err)) - return err - } - if !shouldUpdate { - m.Logger.Info("No version bump required") - return nil - } - meta, err := m.BuildMetadata() if err != nil { return err diff --git a/src/internal/module/update.go b/src/internal/module/update.go deleted file mode 100644 index 77b879f858..0000000000 --- a/src/internal/module/update.go +++ /dev/null @@ -1,73 +0,0 @@ -package module - -import ( - "log/slog" - - "golang.org/x/mod/semver" -) - -func (p Module) shouldUpdateMetadataFile() (bool, error) { - semVerTag, err := p.getLastSemVerTag() - if err != nil { - return false, err - } - - if semVerTag == "" { - // Repo unavailable or tags deleted - return false, nil - } - - fileContent, err := p.ReadMetadata() - if err != nil { - return false, err - } - - for _, v := range fileContent.Versions { - if v.Version == semVerTag { - p.Logger.Info("Found latest tag, nothing to update...", slog.String("tag", semVerTag)) - return false, nil - } - } - - p.Logger.Info("Could not find latest tag, updating...", slog.String("tag", semVerTag)) - return true, nil -} - -// getSemVerTagsFromRSS returns a list of semver tags from the RSS feed -// ignoring all non-valid semver tags -func (p Module) getSemVerTagsFromRSS() ([]string, error) { - releasesRssUrl := p.RSSURL() - tags, err := p.Github.GetTagsFromRSS(releasesRssUrl) - if err != nil { - return nil, err - } - - var semverTags []string - for _, tag := range tags { - if semver.IsValid(tag) || semver.IsValid("v"+tag) { - semverTags = append(semverTags, tag) - } - } - - return semverTags, nil -} - -// getLastSemVerTag returns the most recently created semver tag from the RSS feed -// by sorting the tags by descending creation date -func (p Module) getLastSemVerTag() (string, error) { - semverTags, err := p.getSemVerTagsFromRSS() - if err != nil { - // TODO This is a stopgap, the logs will need to be checked regularly for this. - p.Logger.Error("Unable to fetch tags, skipping", slog.Any("err", err)) - return "", nil - } - - if len(semverTags) < 1 { - // TODO This is a stopgap, the logs will need to be checked regularly for this. - p.Logger.Error("no semver tags found in repository, skipping", slog.String("url", p.RepositoryURL())) - return "", nil - } - - // Tags should be sorted by descending creation date. So, return the first tag - return semverTags[0], nil -} diff --git a/src/internal/provider/build.go b/src/internal/provider/build.go index 5c7ee41cda..76097fb5a2 100644 --- a/src/internal/provider/build.go +++ b/src/internal/provider/build.go @@ -2,6 +2,7 @@ package provider import ( "fmt" + "log/slog" "slices" "github.com/opentofu/registry-stable/internal" @@ -63,6 +64,19 @@ func (p Provider) buildMetadata() (*Metadata, error) { // filter the releases to only include those that do not already exist in the metadata newReleases := meta.filterNewReleases(releases) + if len(newReleases) == 0 { + p.Logger.Info("No version bump required, all versions exist") + return &meta, nil + } + + shouldUpdate, err := p.shouldUpdateMetadataFile() + if err != nil { + p.Logger.Error("Failed to determine update status, forcing update", slog.Any("err", err)) + } else if !shouldUpdate { + p.Logger.Info("No version bump required, latest versions exist") + return &meta, nil + } + type versionResult struct { v *Version err error diff --git a/src/internal/provider/update.go b/src/internal/provider/update.go index be0461ec2a..c79c552dd7 100644 --- a/src/internal/provider/update.go +++ b/src/internal/provider/update.go @@ -11,21 +11,12 @@ import ( func (p Provider) UpdateMetadataFile() error { p.Logger.Info("Beginning version bump process") - shouldUpdate, err := p.shouldUpdateMetadataFile() - if err != nil { - p.Logger.Error("Failed to determine update status", slog.Any("err", err)) - return err - } - if !shouldUpdate { - p.Logger.Info("No version bump required") - return nil - } - meta, err := p.buildMetadata() if err != nil { p.Logger.Error("Failed to version bump provider", slog.Any("err", err)) return err } + p.Logger.Info("Completed provider version bump successfully") return p.WriteMetadata(*meta) } From fbeaa3d5cae25627e47c5a42c4bc510ebbe37575 Mon Sep 17 00:00:00 2001 From: Christian Mesh Date: Fri, 8 Dec 2023 10:52:31 -0500 Subject: [PATCH 2/2] Don't fail on listing tags, warn error instead Signed-off-by: Christian Mesh --- src/internal/module/build.go | 6 +++++- src/internal/provider/build.go | 7 ++++--- src/internal/provider/update.go | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/internal/module/build.go b/src/internal/module/build.go index 79b879414c..abe9a68f0a 100644 --- a/src/internal/module/build.go +++ b/src/internal/module/build.go @@ -17,6 +17,9 @@ func (m Module) UpdateMetadataFile() error { if err != nil { return err } + if meta == nil { + return nil + } return m.WriteMetadata(*meta) } @@ -26,7 +29,8 @@ func (m Module) UpdateMetadataFile() error { func (m Module) BuildMetadata() (*Metadata, error) { tags, err := m.getSemverTags() if err != nil { - return nil, err + m.Logger.Error("Unable to fetch semver tags, skipping", slog.Any("err", err)) + return nil, nil } versions := make([]Version, len(tags)) diff --git a/src/internal/provider/build.go b/src/internal/provider/build.go index 76097fb5a2..ef99a3caec 100644 --- a/src/internal/provider/build.go +++ b/src/internal/provider/build.go @@ -58,7 +58,8 @@ func (p Provider) buildMetadata() (*Metadata, error) { // fetch ALL the releases releases, err := p.getSemverTags() if err != nil { - return nil, err + p.Logger.Error("Unable to fetch semver tags, skipping", slog.Any("err", err)) + return nil, nil } // filter the releases to only include those that do not already exist in the metadata @@ -66,7 +67,7 @@ func (p Provider) buildMetadata() (*Metadata, error) { if len(newReleases) == 0 { p.Logger.Info("No version bump required, all versions exist") - return &meta, nil + return nil, nil } shouldUpdate, err := p.shouldUpdateMetadataFile() @@ -74,7 +75,7 @@ func (p Provider) buildMetadata() (*Metadata, error) { p.Logger.Error("Failed to determine update status, forcing update", slog.Any("err", err)) } else if !shouldUpdate { p.Logger.Info("No version bump required, latest versions exist") - return &meta, nil + return nil, nil } type versionResult struct { diff --git a/src/internal/provider/update.go b/src/internal/provider/update.go index c79c552dd7..5b30ef75c6 100644 --- a/src/internal/provider/update.go +++ b/src/internal/provider/update.go @@ -16,6 +16,9 @@ func (p Provider) UpdateMetadataFile() error { p.Logger.Error("Failed to version bump provider", slog.Any("err", err)) return err } + if meta == nil { + return nil + } p.Logger.Info("Completed provider version bump successfully") return p.WriteMetadata(*meta)