Add config file generation for RubyGems

SPMM-9901

Signed-off-by: Milan Tichavský <[email protected]>

cachito/workers/pkg_managers/rubygems.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
+import os
 import random
 import re
 import secrets
@@ -371,6 +372,10 @@ def resolve_rubygems(package_root, request):
         if dependency["kind"] == "GEM":
             _push_downloaded_gem(dependency, rubygems_repo_name)
 
+    for dep in dependencies:
+        if dep["kind"] == "GIT":
+            unpack_git_dependency(dep)
+
     dependencies = cleanup_metadata(dependencies)
     name, version = _get_metadata(package_root, request)
     if package_root == bundle_dir:
@@ -384,6 +389,22 @@ def resolve_rubygems(package_root, request):
     }
 
 
+def unpack_git_dependency(dep):
+    """
+    Unpack the archive with the downloaded dependency.
+
+    Only the unpacked directory is kept, the archive is deleted.
+    To get more info on local Git repos, see:
+    https://bundler.io/man/bundle-config.1.html#LOCAL-GIT-REPOS
+    :param dep: RubyGems GIT dependency
+    """
+    #
+    extracted_path = str(dep["path"]).removesuffix(".tar.gz")
+    shutil.unpack_archive(dep["path"], extracted_path)
+    os.remove(dep["path"])
+    dep["path"] = extracted_path
+
+
 def _upload_rubygems_package(repo_name, artifact_path):
     """
     Upload a RubyGems package to a Nexus repository.

cachito/workers/tasks/rubygems.py

@@ -1,9 +1,16 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+from os.path import relpath
+from pathlib import Path
+from textwrap import dedent
+
+from cachito.errors import CachitoError
 from cachito.workers import nexus
 from cachito.workers.pkg_managers.rubygems import (
     get_rubygems_hosted_repo_name,
     get_rubygems_nexus_username,
 )
 from cachito.workers.tasks.celery import app
+from cachito.workers.tasks.utils import make_base64_config_file
 
 __all__ = ["cleanup_rubygems_request"]
 
@@ -16,3 +23,61 @@ def cleanup_rubygems_request(request_id):
         "username": get_rubygems_nexus_username(request_id),
     }
     nexus.execute_script("rubygems_cleanup", payload)
+
+
+def _get_config_file_for_given_package(
+    dependencies, bundle_dir, package_source_dir, rubygems_hosted_url
+):
+    """
+    Get Bundler config file.
+
+    Returns a Bundler config file with a mirror set for RubyGems dependencies pointing to
+    `rubygems_hosted_repo` URL. All GIT dependencies are configured to be replaced by local git
+     repos.
+
+    :param dependencies: an array of dependencies (dictionaries) with keys
+        "name": package name,
+        "path": an absolute path to a locally downloaded git repo,
+        "kind": dependency kind
+    :param bundle_dir: an absolute path to the root of the Cachito bundle
+    :param package_source_dir: a path to the root directory of given package
+    :param rubygems_hosted_url: URL pointing to a request specific RubyGems hosted repo with
+     hardcoded user credentials
+    :return: dict with "content", "path" and "type" keys
+    """
+    base_config = dedent(
+        f"""
+        # Sets mirror for all RubyGems sources
+        BUNDLE_MIRROR__ALL: "{rubygems_hosted_url}"
+        # Turn off the probing
+        BUNDLE_MIRROR__ALL__FALLBACK_TIMEOUT: "false"
+        # Install only ruby platform gems (=> gems with native extensions are compiled from source).
+        # All gems should be platform independent already, so why not keep it here.
+        BUNDLE_FORCE_RUBY_PLATFORM: "true"
+        BUNDLE_DEPLOYMENT: "true"
+        # Defaults to true when deployment is set to true
+        BUNDLE_FROZEN: "true"
+        # For local Git replacements, branches don't have to be specified (commit hash is enough)
+        BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true"
+    """
+    )
+
+    config = [base_config]
+    for dependency in dependencies:
+        if dependency["kind"] == "GIT":
+            # These substitutions are required by Bundler
+            name = dependency["name"].upper().replace("-", "___").replace(".", "__")
+            relative_path = relpath(dependency["path"], package_source_dir)
+            dep_replacement = f'BUNDLE_LOCAL__{name}: "{relative_path + "/app"}"'
+            config.append(dep_replacement)
+
+    final_config = "\n".join(config)
+
+    config_file_path = package_source_dir / Path(".bundle/config")
+    if config_file_path.exists():
+        raise CachitoError(
+            f"Cachito wants to create a config file at location {config_file_path}, "
+            f"but it already exists."
+        )
+    final_path = config_file_path.relative_to(Path(bundle_dir))
+    return make_base64_config_file(final_config, final_path)

tests/test_workers/test_pkg_managers/test_rubygems.py

@@ -706,13 +706,23 @@ def test_resolve_rubygems_invalid_gemfile_lock_path(mock_request_bundle_dir, tmp
 
 
 @pytest.mark.parametrize("subpath_pkg", [True, False])
+@mock.patch("cachito.workers.pkg_managers.rubygems.unpack_git_dependency")
 @mock.patch("cachito.workers.pkg_managers.rubygems._get_metadata")
 @mock.patch("cachito.workers.pkg_managers.rubygems._upload_rubygems_package")
 @mock.patch("cachito.workers.pkg_managers.rubygems.download_dependencies")
 @mock.patch("cachito.workers.pkg_managers.rubygems.RequestBundleDir")
 def test_resolve_rubygems(
-    mock_request_bundle_dir, mock_download, mock_upload, mock_get_metadata, subpath_pkg, tmp_path
+    mock_request_bundle_dir,
+    mock_download,
+    mock_upload,
+    mock_get_metadata,
+    mock_unpack,
+    subpath_pkg,
+    tmp_path,
 ):
+    mock_bundle_dir = MockBundleDir(tmp_path)
+    mock_request_bundle_dir.return_value = mock_bundle_dir
+
     if subpath_pkg:
         package_root = tmp_path
         expected_path = None
@@ -721,8 +731,6 @@ def test_resolve_rubygems(
         package_root.mkdir()
         expected_path = Path("first_pkg")
 
-    mock_bundle_dir = MockBundleDir(tmp_path)
-    mock_request_bundle_dir.return_value = mock_bundle_dir
     mock_get_metadata.return_value = ("pkg_name", "1.0.0")
     gemfile_lock = package_root / rubygems.GEMFILE_LOCK
     text = dedent(
@@ -748,29 +756,39 @@ def test_resolve_rubygems(
     )
     gemfile_lock.write_text(text)
 
+    gem_dependency = {
+        "kind": "GEM",
+        "path": "some/path",
+        "name": "ci_reporter",
+        "version": "2.0.0",
+        "type": "rubygems",
+    }
+    git_dependency = {
+        "kind": "GIT",
+        "name": "ci_reporter_shell",
+        "version": f"git+{CI_REPORTER_URL}@{GIT_REF}",
+        "path": "path/to/downloaded.tar.gz",
+        "type": "rubygems",
+    }
     mock_download.return_value = [
-        {
-            "kind": "GEM",
-            "path": "some/path",
-            "name": "ci_reporter",
-            "version": "2.0.0",
-            "type": "rubygems",
-        },
-        {
-            "kind": "GIT",
-            "name": "ci_reporter_shell",
-            "version": f"git+{CI_REPORTER_URL}@{GIT_REF}",
-            "path": "path/to/downloaded",
-            "type": "rubygems",
-        },
+        gem_dependency,
+        git_dependency,
     ]
 
+    def side_effect(dep):
+        dep["path"] = "path/to/downloaded"
+        return dep
+
+    mock_unpack.side_effect = side_effect
+
     request = {"id": 1}
 
     pkg_info = rubygems.resolve_rubygems(package_root, request)
 
     mock_upload.assert_called_once_with("cachito-rubygems-hosted-1", "some/path")
     assert mock_upload.call_count == 1
+    mock_unpack.assert_called_once_with(git_dependency)
+
     expected = {
         "package": {
             "name": "pkg_name",
@@ -853,3 +871,15 @@ def test_get_metadata(mock_request_bundle_dir, tmp_path, package_subpath, expect
     name, version = rubygems._get_metadata(tmp_path / package_subpath, request)
     assert name == expected_name
     assert version == GIT_REF
+
+
+@mock.patch("cachito.workers.pkg_managers.rubygems.os.remove")
+@mock.patch("cachito.workers.pkg_managers.rubygems.shutil.unpack_archive")
+def test_unpack_git_dependency(mock_unpack, mock_remove):
+    original_path = "some/path.tar.gz"
+    new_path = "some/path"
+    dep = {"path": original_path}
+    rubygems.unpack_git_dependency(dep)
+    mock_unpack.assert_called_once_with(original_path, new_path)
+    mock_remove.assert_called_once_with(original_path)
+    assert dep["path"] == new_path

tests/test_workers/test_tasks/test_rubygems.py

@@ -1,5 +1,11 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+import base64
+from pathlib import Path
 from unittest import mock
 
+import pytest
+
+from cachito.errors import CachitoError
 from cachito.workers.tasks import rubygems
 
 
@@ -12,3 +18,55 @@ def test_cleanup_rubygems_request(mock_exec_script):
         "username": "cachito-rubygems-42",
     }
     mock_exec_script.assert_called_once_with("rubygems_cleanup", expected_payload)
+
+
+class MockBundleDir(type(Path())):
+    """Mocked RequestBundleDir."""
+
+    def __new__(cls, *args, **kwargs):
+        """Make a new MockBundleDir."""
+        self = super().__new__(cls, *args, **kwargs)
+        self.source_root_dir = self.joinpath("app")
+        self.rubygems_deps_dir = self / "deps" / "rubygems"
+        return self
+
+
+@pytest.mark.parametrize("exists", [True, False])
+def test_get_config_file(tmp_path, exists):
+    bundle_dir = MockBundleDir(tmp_path)
+
+    pkg_and_deps_info = {
+        "dependencies": [
+            {
+                "name": "rspec-core.3",
+                "path": bundle_dir.rubygems_deps_dir / "rspec-core.3" / "some-path",
+                "kind": "GIT",
+            }
+        ],
+    }
+    rubygems_hosted_repo = "https://admin:[email protected]"
+    package_root = bundle_dir.source_root_dir / "pkg1"
+
+    if exists:
+        config_file = package_root / Path(".bundle/config")
+        config_file.parent.mkdir(parents=True)
+        config_file.touch()
+        msg = (
+            f"Cachito wants to create a config file at location {config_file}, "
+            f"but it already exists."
+        )
+        with pytest.raises(CachitoError, match=msg):
+            rubygems._get_config_file_for_given_package(
+                pkg_and_deps_info["dependencies"], bundle_dir, package_root, rubygems_hosted_repo
+            )
+    else:
+        dep = rubygems._get_config_file_for_given_package(
+            pkg_and_deps_info["dependencies"], bundle_dir, package_root, rubygems_hosted_repo
+        )
+
+        assert dep["path"] == "app/pkg1/.bundle/config"
+        assert dep["type"] == "base64"
+        contents = base64.b64decode(dep["content"]).decode()
+        assert f'BUNDLE_MIRROR__ALL: "{rubygems_hosted_repo}"' in contents
+        git_dep = 'BUNDLE_LOCAL__RSPEC___CORE__3: "../../deps/rubygems/rspec-core.3/some-path/app"'
+        assert git_dep in contents