ci: add dependency-checker command

.gitignore

@@ -28,3 +28,6 @@ go.work.sum
 
 # Local values files
 values.local.yaml
+
+# Website cache for the dependency checker
+cache

Makefile

@@ -53,6 +53,10 @@ markdown-links-lint:
 check-configuration-keys:
 	./ci/check_configuration_keys.py --values deploy/helm/sumologic/values.yaml --readme deploy/helm/sumologic/README.md
 
+.PHONY: check-dependencies
+check-dependencies:
+	@python ./ci/check_dependencies/main.py
+
 .PHONY: template-tests-lint
 template-tests-lint:
 	make -C ./tests/helm golint

ci/check_dependencies/.gitignore

@@ -0,0 +1,129 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

ci/check_dependencies/README.md

@@ -0,0 +1,35 @@
+# dependency-checker
+
+Dependency checker uses web scraping to get information about:
+
+- platforms suppported by [Sumologic Kubernetes Helm Chart][helm-chart]
+- subcharts which are use in [Sumologic Kubernetes Helm Chart][helm-chart]
+
+## Usage
+
+### Prepare a virtualenv and activate it
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+```
+
+### Install dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### Run the script
+
+```bash
+python main.py
+```
+
+Web pages used to get information are saved in `cache` directory to clean it please run:
+
+```bash
+rm -r ./cache
+```
+
+[helm-chart]: https://github.com/SumoLogic/sumologic-kubernetes-collection

ci/check_dependencies/aks.py

@@ -0,0 +1,71 @@
+import requests
+import os
+import json
+import lxml.html as lh
+import traceback
+
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+from datetime import datetime, timedelta
+
+import common
+
+
+def get_pd_dataframe_from_html(html):
+    soup = BeautifulSoup(html, "lxml")
+
+    # get information from second table on web page
+    calendar_table = soup.select_one("table:nth-of-type(1)")
+
+    # Obtain every title of columns with tag <th>
+    headers = []
+    for i in calendar_table.find_all("th"):
+        title = i.text
+        headers.append(title)
+
+    data = pd.DataFrame(columns=headers)
+
+    # Create a for loop to fill pandas dataframe
+    for j in calendar_table.find_all("tr")[1:]:
+        row_data = j.find_all("td")
+        row = [i.text for i in row_data]
+        length = len(data)
+        data.loc[length] = row
+
+    return data
+
+
+def get_supported_releases(html_calendar):
+    pd_data = get_pd_dataframe_from_html(html_calendar)
+
+    # AKS adds an asterisk to the LTS version, we remove it
+    pd_data["K8s version"] = pd_data["K8s version"].apply(lambda s: s.strip("*"))
+
+    pd_data = pd_data.set_index("K8s version")
+    pd_data["AKS GA date"] = pd.to_datetime(
+        pd_data["AKS GA"], format="%b %Y", errors="coerce"
+    ).fillna(datetime.now() + timedelta(days=50 * 365))
+
+    today = datetime.now()
+    supported_releases = []
+    for index, row in pd_data.iterrows():
+        if row["AKS GA date"] < today:
+            dependent_version = row["Platform support"].replace("GA", "").replace("Until", "").strip()
+            if dependent_version in pd_data.index:
+                dependent_version_GA_date = pd_data.loc[dependent_version][
+                    "AKS GA date"
+                ]
+                if dependent_version_GA_date > today:
+                    supported_releases.append(index)
+            else:
+                supported_releases.append(index)
+    return supported_releases
+
+
+def get_info():
+    cache_file = "cache/aks_calendar.html"
+    calendar_web_page = "https://learn.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#aks-kubernetes-release-calendar"
+    html_calendar = common.get_page(calendar_web_page, cache_file)
+    officially_supported = get_supported_releases(html_calendar)
+    common.get_info("AKS", officially_supported)

ci/check_dependencies/common.py

@@ -0,0 +1,71 @@
+import kubernetes_collection
+
+import requests
+import os
+import json
+import lxml.html as lh
+import traceback
+
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+from datetime import datetime, timedelta
+
+cache_path = "cache/"
+
+
+def prepare_cache_dir():
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+
+
+def is_cache_updated(cache_file):
+    if os.path.isfile(cache_file):
+        return True
+    return False
+
+
+def get_page(web_page, cache_file):
+    prepare_cache_dir()
+    calendar = ""
+    if is_cache_updated(cache_file):
+        # print("{} available in cache, getting from file".format(web_page))
+        with open(cache_file, "r") as f:
+            calendar = f.read()
+    else:
+        # print("{} not available in cache".format(web_page))
+        result = requests.get(web_page)
+        if result.status_code == 200:
+            with open(cache_file, "w") as f:
+                f.write(result.text)
+                calendar = result.text
+    return calendar
+
+
+def get_info(platform, officially_supported):
+    if platform == "OpenShift":
+        line_pattern = platform
+    else:
+        line_pattern = "K8s with {}".format(platform)
+
+    print("{} officially supported versions".format(platform))
+    print(officially_supported)
+
+    print(
+        "Currently supported {} versions for Sumologic Kubernetes Collection Helm Chart".format(
+            platform
+        )
+    )
+    now_suppported = kubernetes_collection.get_supported_versions(line_pattern)
+    print(now_suppported)
+    print("\n")
+
+    versions_to_add = sorted(set(officially_supported) - set(now_suppported))
+    if len(versions_to_add) != 0:
+        print("Please add support to following {} versions:".format(platform))
+        print(versions_to_add)
+
+    versions_to_remove = sorted(set(now_suppported) - set(officially_supported))
+    if len(versions_to_remove) != 0:
+        print("Please remove support to following {} versions:".format(platform))
+        print(versions_to_remove)

ci/check_dependencies/eks.py

@@ -0,0 +1,69 @@
+import requests
+import traceback
+import os
+import json
+from datetime import datetime
+import kubernetes_collection
+import common
+
+
+def get_eks_doc_from_github():
+    result = requests.get(
+        "https://raw.githubusercontent.com/awsdocs/amazon-eks-user-guide/master/doc_source/kubernetes-versions.md"
+    )
+    if result.status_code == 200:
+        return result.text
+    return None
+
+
+def get_eks_calendar_table(doc):
+    lines = doc.split("\n")
+    eks_calendar_table = []
+
+    found_calendar = False
+    for line in lines:
+        if "Amazon EKS end of support" in line:
+            found_calendar = True
+        elif found_calendar:
+            if line != "":
+                eks_calendar_table.append(line)
+            else:
+                break
+    # remove first line related to table formatting in markdown
+    eks_calendar_table = eks_calendar_table[1:]
+    return eks_calendar_table
+
+
+def parse_eks_end_of_support_date(date):
+    try:
+        date_elements_len = len(date.split(" "))
+
+        if date_elements_len == 2:
+            end_of_support_date = datetime.strptime(date, "%B %Y")
+        else:
+            end_of_support_date = datetime.strptime(date, "%B %d, %Y")
+        return end_of_support_date
+    except:
+        print("Unexpected date format, date={}".format(date))
+        traceback.print_exc()
+
+
+def get_eks_officially_supported_releases():
+    doc = get_eks_doc_from_github()
+    eks_calendar_table = get_eks_calendar_table(doc)
+    today = datetime.now()
+
+    eks_supported_releases = []
+    for row in eks_calendar_table:
+        elements = [x for x in row.strip().split("|") if x]
+        release_version = elements[0].replace("\\", "").strip()
+        end_of_support = elements[-1].strip()
+        eof_of_support_date = parse_eks_end_of_support_date(end_of_support)
+        if today < eof_of_support_date:
+            eks_supported_releases.append(release_version)
+    return sorted(eks_supported_releases)
+
+
+def get_info():
+    officially_supported = get_eks_officially_supported_releases()
+    common.get_info("EKS", officially_supported)