From 07f5cb1ea8307b08ac8a3b1746c28d03fda5c72c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milan=20Tichavsk=C3=BD?= Date: Mon, 15 Aug 2022 10:19:56 +0200 Subject: [PATCH] Add config file generation for RubyGems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPMM-9901 Signed-off-by: Milan Tichavský --- cachito/workers/pkg_managers/rubygems.py | 21 ++++++ cachito/workers/tasks/rubygems.py | 65 +++++++++++++++++++ .../test_pkg_managers/test_rubygems.py | 64 +++++++++++++----- .../test_workers/test_tasks/test_rubygems.py | 58 +++++++++++++++++ 4 files changed, 191 insertions(+), 17 deletions(-) diff --git a/cachito/workers/pkg_managers/rubygems.py b/cachito/workers/pkg_managers/rubygems.py index 48ed33fa7..58bd18115 100644 --- a/cachito/workers/pkg_managers/rubygems.py +++ b/cachito/workers/pkg_managers/rubygems.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging +import os import random import re import secrets @@ -371,6 +372,10 @@ def resolve_rubygems(package_root, request): if dependency["kind"] == "GEM": _push_downloaded_gem(dependency, rubygems_repo_name) + for dep in dependencies: + if dep["kind"] == "GIT": + unpack_git_dependency(dep) + dependencies = cleanup_metadata(dependencies) name, version = _get_metadata(package_root, request) if package_root == bundle_dir: @@ -384,6 +389,22 @@ def resolve_rubygems(package_root, request): } +def unpack_git_dependency(dep): + """ + Unpack the archive with the downloaded dependency. + + Only the unpacked directory is kept, the archive is deleted. + To get more info on local Git repos, see: + https://bundler.io/man/bundle-config.1.html#LOCAL-GIT-REPOS + :param dep: RubyGems GIT dependency + """ + # + extracted_path = str(dep["path"]).removesuffix(".tar.gz") + shutil.unpack_archive(dep["path"], extracted_path) + os.remove(dep["path"]) + dep["path"] = extracted_path + + def _upload_rubygems_package(repo_name, artifact_path): """ Upload a RubyGems package to a Nexus repository. diff --git a/cachito/workers/tasks/rubygems.py b/cachito/workers/tasks/rubygems.py index d12037a95..1c9738380 100644 --- a/cachito/workers/tasks/rubygems.py +++ b/cachito/workers/tasks/rubygems.py @@ -1,9 +1,16 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +from os.path import relpath +from pathlib import Path +from textwrap import dedent + +from cachito.errors import CachitoError from cachito.workers import nexus from cachito.workers.pkg_managers.rubygems import ( get_rubygems_hosted_repo_name, get_rubygems_nexus_username, ) from cachito.workers.tasks.celery import app +from cachito.workers.tasks.utils import make_base64_config_file __all__ = ["cleanup_rubygems_request"] @@ -16,3 +23,61 @@ def cleanup_rubygems_request(request_id): "username": get_rubygems_nexus_username(request_id), } nexus.execute_script("rubygems_cleanup", payload) + + +def _get_config_file_for_given_package( + dependencies, bundle_dir, package_source_dir, rubygems_hosted_url +): + """ + Get Bundler config file. + + Returns a Bundler config file with a mirror set for RubyGems dependencies pointing to + `rubygems_hosted_repo` URL. All GIT dependencies are configured to be replaced by local git + repos. + + :param dependencies: an array of dependencies (dictionaries) with keys + "name": package name, + "path": an absolute path to a locally downloaded git repo, + "kind": dependency kind + :param bundle_dir: an absolute path to the root of the Cachito bundle + :param package_source_dir: a path to the root directory of given package + :param rubygems_hosted_url: URL pointing to a request specific RubyGems hosted repo with + hardcoded user credentials + :return: dict with "content", "path" and "type" keys + """ + base_config = dedent( + f""" + # Sets mirror for all RubyGems sources + BUNDLE_MIRROR__ALL: "{rubygems_hosted_url}" + # Turn off the probing + BUNDLE_MIRROR__ALL__FALLBACK_TIMEOUT: "false" + # Install only ruby platform gems (=> gems with native extensions are compiled from source). + # All gems should be platform independent already, so why not keep it here. + BUNDLE_FORCE_RUBY_PLATFORM: "true" + BUNDLE_DEPLOYMENT: "true" + # Defaults to true when deployment is set to true + BUNDLE_FROZEN: "true" + # For local Git replacements, branches don't have to be specified (commit hash is enough) + BUNDLE_DISABLE_LOCAL_BRANCH_CHECK: "true" + """ + ) + + config = [base_config] + for dependency in dependencies: + if dependency["kind"] == "GIT": + # These substitutions are required by Bundler + name = dependency["name"].upper().replace("-", "___").replace(".", "__") + relative_path = relpath(dependency["path"], package_source_dir) + dep_replacement = f'BUNDLE_LOCAL__{name}: "{relative_path + "/app"}"' + config.append(dep_replacement) + + final_config = "\n".join(config) + + config_file_path = package_source_dir / Path(".bundle/config") + if config_file_path.exists(): + raise CachitoError( + f"Cachito wants to create a config file at location {config_file_path}, " + f"but it already exists." + ) + final_path = config_file_path.relative_to(Path(bundle_dir)) + return make_base64_config_file(final_config, final_path) diff --git a/tests/test_workers/test_pkg_managers/test_rubygems.py b/tests/test_workers/test_pkg_managers/test_rubygems.py index 7bb4ed51b..996821a51 100644 --- a/tests/test_workers/test_pkg_managers/test_rubygems.py +++ b/tests/test_workers/test_pkg_managers/test_rubygems.py @@ -706,13 +706,23 @@ def test_resolve_rubygems_invalid_gemfile_lock_path(mock_request_bundle_dir, tmp @pytest.mark.parametrize("subpath_pkg", [True, False]) +@mock.patch("cachito.workers.pkg_managers.rubygems.unpack_git_dependency") @mock.patch("cachito.workers.pkg_managers.rubygems._get_metadata") @mock.patch("cachito.workers.pkg_managers.rubygems._upload_rubygems_package") @mock.patch("cachito.workers.pkg_managers.rubygems.download_dependencies") @mock.patch("cachito.workers.pkg_managers.rubygems.RequestBundleDir") def test_resolve_rubygems( - mock_request_bundle_dir, mock_download, mock_upload, mock_get_metadata, subpath_pkg, tmp_path + mock_request_bundle_dir, + mock_download, + mock_upload, + mock_get_metadata, + mock_unpack, + subpath_pkg, + tmp_path, ): + mock_bundle_dir = MockBundleDir(tmp_path) + mock_request_bundle_dir.return_value = mock_bundle_dir + if subpath_pkg: package_root = tmp_path expected_path = None @@ -721,8 +731,6 @@ def test_resolve_rubygems( package_root.mkdir() expected_path = Path("first_pkg") - mock_bundle_dir = MockBundleDir(tmp_path) - mock_request_bundle_dir.return_value = mock_bundle_dir mock_get_metadata.return_value = ("pkg_name", "1.0.0") gemfile_lock = package_root / rubygems.GEMFILE_LOCK text = dedent( @@ -748,29 +756,39 @@ def test_resolve_rubygems( ) gemfile_lock.write_text(text) + gem_dependency = { + "kind": "GEM", + "path": "some/path", + "name": "ci_reporter", + "version": "2.0.0", + "type": "rubygems", + } + git_dependency = { + "kind": "GIT", + "name": "ci_reporter_shell", + "version": f"git+{CI_REPORTER_URL}@{GIT_REF}", + "path": "path/to/downloaded.tar.gz", + "type": "rubygems", + } mock_download.return_value = [ - { - "kind": "GEM", - "path": "some/path", - "name": "ci_reporter", - "version": "2.0.0", - "type": "rubygems", - }, - { - "kind": "GIT", - "name": "ci_reporter_shell", - "version": f"git+{CI_REPORTER_URL}@{GIT_REF}", - "path": "path/to/downloaded", - "type": "rubygems", - }, + gem_dependency, + git_dependency, ] + def side_effect(dep): + dep["path"] = "path/to/downloaded" + return dep + + mock_unpack.side_effect = side_effect + request = {"id": 1} pkg_info = rubygems.resolve_rubygems(package_root, request) mock_upload.assert_called_once_with("cachito-rubygems-hosted-1", "some/path") assert mock_upload.call_count == 1 + mock_unpack.assert_called_once_with(git_dependency) + expected = { "package": { "name": "pkg_name", @@ -853,3 +871,15 @@ def test_get_metadata(mock_request_bundle_dir, tmp_path, package_subpath, expect name, version = rubygems._get_metadata(tmp_path / package_subpath, request) assert name == expected_name assert version == GIT_REF + + +@mock.patch("cachito.workers.pkg_managers.rubygems.os.remove") +@mock.patch("cachito.workers.pkg_managers.rubygems.shutil.unpack_archive") +def test_unpack_git_dependency(mock_unpack, mock_remove): + original_path = "some/path.tar.gz" + new_path = "some/path" + dep = {"path": original_path} + rubygems.unpack_git_dependency(dep) + mock_unpack.assert_called_once_with(original_path, new_path) + mock_remove.assert_called_once_with(original_path) + assert dep["path"] == new_path diff --git a/tests/test_workers/test_tasks/test_rubygems.py b/tests/test_workers/test_tasks/test_rubygems.py index 7a87a7e6f..43476e179 100644 --- a/tests/test_workers/test_tasks/test_rubygems.py +++ b/tests/test_workers/test_tasks/test_rubygems.py @@ -1,5 +1,11 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +import base64 +from pathlib import Path from unittest import mock +import pytest + +from cachito.errors import CachitoError from cachito.workers.tasks import rubygems @@ -12,3 +18,55 @@ def test_cleanup_rubygems_request(mock_exec_script): "username": "cachito-rubygems-42", } mock_exec_script.assert_called_once_with("rubygems_cleanup", expected_payload) + + +class MockBundleDir(type(Path())): + """Mocked RequestBundleDir.""" + + def __new__(cls, *args, **kwargs): + """Make a new MockBundleDir.""" + self = super().__new__(cls, *args, **kwargs) + self.source_root_dir = self.joinpath("app") + self.rubygems_deps_dir = self / "deps" / "rubygems" + return self + + +@pytest.mark.parametrize("exists", [True, False]) +def test_get_config_file(tmp_path, exists): + bundle_dir = MockBundleDir(tmp_path) + + pkg_and_deps_info = { + "dependencies": [ + { + "name": "rspec-core.3", + "path": bundle_dir.rubygems_deps_dir / "rspec-core.3" / "some-path", + "kind": "GIT", + } + ], + } + rubygems_hosted_repo = "https://admin:admin@hosted.com" + package_root = bundle_dir.source_root_dir / "pkg1" + + if exists: + config_file = package_root / Path(".bundle/config") + config_file.parent.mkdir(parents=True) + config_file.touch() + msg = ( + f"Cachito wants to create a config file at location {config_file}, " + f"but it already exists." + ) + with pytest.raises(CachitoError, match=msg): + rubygems._get_config_file_for_given_package( + pkg_and_deps_info["dependencies"], bundle_dir, package_root, rubygems_hosted_repo + ) + else: + dep = rubygems._get_config_file_for_given_package( + pkg_and_deps_info["dependencies"], bundle_dir, package_root, rubygems_hosted_repo + ) + + assert dep["path"] == "app/pkg1/.bundle/config" + assert dep["type"] == "base64" + contents = base64.b64decode(dep["content"]).decode() + assert f'BUNDLE_MIRROR__ALL: "{rubygems_hosted_repo}"' in contents + git_dep = 'BUNDLE_LOCAL__RSPEC___CORE__3: "../../deps/rubygems/rspec-core.3/some-path/app"' + assert git_dep in contents