From fb369fc2539d8d6833991729f234015d084a37b9 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Fri, 24 Feb 2023 12:14:51 +0100 Subject: [PATCH 01/14] fix #836 --- bioconda_utils/build.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index e09cf60db8..45379798ea 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -116,6 +116,16 @@ def build(recipe: str, pkg_paths: List[str] = None, env=whitelisted_env, noarch=is_noarch) # Use presence of expected packages to check for success + if (docker_builder.pkg_dir is not None): + platform = os.environ.get('OSTYPE', sys.platform) + if platform.startswith("darwin"): + platform = 'osx' + elif platform == "linux-gnu": + platform = "linux" + + config = load_conda_build_config(platform=platform) + pkg_paths = [p.replace(config.output_folder, docker_builder.pkg_dir) for p in pkg_paths] + for pkg_path in pkg_paths: if not os.path.exists(pkg_path): logger.error( From f53f7d0ebb773a28b54a0f2f4d9d6bb2af923abb Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Fri, 24 Feb 2023 14:17:52 +0100 Subject: [PATCH 02/14] forgot sys import --- bioconda_utils/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 45379798ea..d006b25944 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -5,6 +5,7 @@ import subprocess as sp from collections import defaultdict, namedtuple import os +import sys import logging import itertools From a61b5b7cc1963be89604048bac3aefee6ba5bed9 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Sat, 25 Feb 2023 19:37:52 +0100 Subject: [PATCH 03/14] forgot module --- bioconda_utils/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index d006b25944..7c3c568109 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -124,7 +124,7 @@ def build(recipe: str, pkg_paths: List[str] = None, elif platform == "linux-gnu": platform = "linux" - config = load_conda_build_config(platform=platform) + config = utils.load_conda_build_config(platform=platform) pkg_paths = [p.replace(config.output_folder, docker_builder.pkg_dir) for p in pkg_paths] for pkg_path in pkg_paths: From 39e92b7c8efefc96bc390eca6ad66d9a1e64ca1f Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 27 Feb 2023 23:32:25 +0100 Subject: [PATCH 04/14] try to add some tests --- test/test_utils.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index 09c4abb9eb..62c4172b33 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -126,6 +126,31 @@ def single_build(request, recipes_fixture): yield recipes_fixture.pkgs['one'] for pkg in recipes_fixture.pkgs['one']: ensure_missing(pkg) + + +@pytest.fixture(scope='module', ids=IDS) +def single_build_pkg_dir(request, recipes_fixture): + """ + Builds the "one" recipe with pkg_dir. + """ + logger.error("Making recipe builder") + docker_builder = docker_utils.RecipeBuilder( + use_host_conda_bld=True, + pkg_dir=os.getcwd() + "/output" + docker_base_image=DOCKER_BASE_IMAGE) + mulled_test = True + logger.error("DONE") + logger.error("Fixture: Building 'one' within docker with pkg_dir") + build.build( + recipe=recipes_fixture.recipe_dirs['one'], + pkg_paths=recipes_fixture.pkgs['one'], + docker_builder=docker_builder, + mulled_test=mulled_test, + ) + logger.error("Fixture: Building 'one' within docker and pkg_dir -- DONE") + yield recipes_fixture.pkgs['one'] + for pkg in recipes_fixture.pkgs['one']: + ensure_missing(pkg) @pytest.fixture(scope='module', params=PARAMS, ids=IDS) From 0e66d3ba6f53dfc2bb66b0454fedd3b61887c3e5 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Mon, 27 Feb 2023 23:59:41 +0100 Subject: [PATCH 05/14] forgot comma --- test/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 62c4172b33..d9a99ba926 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -136,7 +136,7 @@ def single_build_pkg_dir(request, recipes_fixture): logger.error("Making recipe builder") docker_builder = docker_utils.RecipeBuilder( use_host_conda_bld=True, - pkg_dir=os.getcwd() + "/output" + pkg_dir=os.getcwd() + "/output", docker_base_image=DOCKER_BASE_IMAGE) mulled_test = True logger.error("DONE") From 8369757effac5b4866d87b4c8919d8d2798b82cc Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Tue, 28 Feb 2023 11:21:21 +0100 Subject: [PATCH 06/14] mark test for a test category --- test/test_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index d9a99ba926..55ec76d40c 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -241,7 +241,11 @@ def test_upload(single_upload): def test_single_build_only(single_build): for pkg in single_build: assert os.path.exists(pkg) - + +@pytest.mark.long_running_2 +def test_single_build_pkg_dir(single_build): + for pkg in single_build: + assert os.path.exists(pkg) @pytest.mark.skipif(SKIP_DOCKER_TESTS, reason='skipping on osx') def test_single_build_with_post_test(single_build): From f26f4ed4041e64f709727a8997a8e50babc1d8f6 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Tue, 28 Feb 2023 12:12:43 +0100 Subject: [PATCH 07/14] rearrange new test --- test/test_utils.py | 50 ++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 55ec76d40c..7852e7c818 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -126,31 +126,6 @@ def single_build(request, recipes_fixture): yield recipes_fixture.pkgs['one'] for pkg in recipes_fixture.pkgs['one']: ensure_missing(pkg) - - -@pytest.fixture(scope='module', ids=IDS) -def single_build_pkg_dir(request, recipes_fixture): - """ - Builds the "one" recipe with pkg_dir. - """ - logger.error("Making recipe builder") - docker_builder = docker_utils.RecipeBuilder( - use_host_conda_bld=True, - pkg_dir=os.getcwd() + "/output", - docker_base_image=DOCKER_BASE_IMAGE) - mulled_test = True - logger.error("DONE") - logger.error("Fixture: Building 'one' within docker with pkg_dir") - build.build( - recipe=recipes_fixture.recipe_dirs['one'], - pkg_paths=recipes_fixture.pkgs['one'], - docker_builder=docker_builder, - mulled_test=mulled_test, - ) - logger.error("Fixture: Building 'one' within docker and pkg_dir -- DONE") - yield recipes_fixture.pkgs['one'] - for pkg in recipes_fixture.pkgs['one']: - ensure_missing(pkg) @pytest.fixture(scope='module', params=PARAMS, ids=IDS) @@ -241,11 +216,30 @@ def test_upload(single_upload): def test_single_build_only(single_build): for pkg in single_build: assert os.path.exists(pkg) + @pytest.mark.long_running_2 -def test_single_build_pkg_dir(single_build): - for pkg in single_build: - assert os.path.exists(pkg) +def single_build_pkg_dir(): + """ + Builds the "one" recipe with pkg_dir. + """ + logger.error("Making recipe builder") + docker_builder = docker_utils.RecipeBuilder( + use_host_conda_bld=True, + pkg_dir=os.getcwd() + "/output", + docker_base_image=DOCKER_BASE_IMAGE) + mulled_test = False + logger.error("DONE") + logger.error("Fixture: Building 'one' within docker with pkg_dir") + res = build.build( + recipe=recipes_fixture.recipe_dirs['one'], + pkg_paths=recipes_fixture.pkgs['one'], + docker_builder=docker_builder, + mulled_test=mulled_test, + ) + logger.error("Fixture: Building 'one' within docker and pkg_dir -- DONE") + assert res.success + @pytest.mark.skipif(SKIP_DOCKER_TESTS, reason='skipping on osx') def test_single_build_with_post_test(single_build): From 0bcf881a64a5f198db18c5da5b78a0e7ce33e41a Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Tue, 28 Feb 2023 13:58:26 +0100 Subject: [PATCH 08/14] retrigger CI which is instable.. I do not have permissions to retrigger --- bioconda_utils/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 7c3c568109..22d1fe8572 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -4,10 +4,10 @@ import subprocess as sp from collections import defaultdict, namedtuple +import itertools +import logging import os import sys -import logging -import itertools from typing import List From 2167739e72c37ec7b9d4df3ce0fc06386faffebb Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Tue, 28 Feb 2023 14:43:49 +0100 Subject: [PATCH 09/14] forgot test prefix --- test/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 7852e7c818..2a10b5e677 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -219,7 +219,7 @@ def test_single_build_only(single_build): @pytest.mark.long_running_2 -def single_build_pkg_dir(): +def test_single_build_pkg_dir(): """ Builds the "one" recipe with pkg_dir. """ From 28ed12086c1e212ea620f9fe7a659d3b76792801 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Tue, 28 Feb 2023 17:22:12 +0100 Subject: [PATCH 10/14] depend on recipes fixture --- test/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 2a10b5e677..f43c2b2670 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -219,7 +219,7 @@ def test_single_build_only(single_build): @pytest.mark.long_running_2 -def test_single_build_pkg_dir(): +def test_single_build_pkg_dir(recipes_fixture): """ Builds the "one" recipe with pkg_dir. """ From b599538968c9e996db840b6265c6f0cd09ba0387 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Tue, 28 Feb 2023 19:00:38 +0100 Subject: [PATCH 11/14] skip on osx --- test/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index f43c2b2670..9a1f9fe937 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -217,7 +217,8 @@ def test_single_build_only(single_build): for pkg in single_build: assert os.path.exists(pkg) - + +@pytest.mark.skipif(SKIP_DOCKER_TESTS, reason='skipping on osx') @pytest.mark.long_running_2 def test_single_build_pkg_dir(recipes_fixture): """ From 06fd75c405772b39503bf38396406066bea9b587 Mon Sep 17 00:00:00 2001 From: Julianus Pfeuffer Date: Wed, 29 Mar 2023 09:55:09 +0200 Subject: [PATCH 12/14] try to use repodata --- bioconda_utils/build.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 22d1fe8572..2a163ada10 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -118,14 +118,10 @@ def build(recipe: str, pkg_paths: List[str] = None, noarch=is_noarch) # Use presence of expected packages to check for success if (docker_builder.pkg_dir is not None): - platform = os.environ.get('OSTYPE', sys.platform) - if platform.startswith("darwin"): - platform = 'osx' - elif platform == "linux-gnu": - platform = "linux" - - config = utils.load_conda_build_config(platform=platform) - pkg_paths = [p.replace(config.output_folder, docker_builder.pkg_dir) for p in pkg_paths] + platform = utils.RepoData().native_platform() + subfolder = utils.RepoData().platform2subdir(platform) + config = utils.load_conda_build_config(platform=subfolder) + pkg_paths = [p.replace(config.output_folder, docker_builder.pkg_dir) for p in pkg_paths] for pkg_path in pkg_paths: if not os.path.exists(pkg_path): From 11f812c174cb7158ec87ee56eb9ecf4eea5c4c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Wed, 9 Aug 2023 10:00:14 +0200 Subject: [PATCH 13/14] do not instatiate RepoData, as the required methods are static --- bioconda_utils/build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 2a163ada10..f2a62074bd 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -118,10 +118,10 @@ def build(recipe: str, pkg_paths: List[str] = None, noarch=is_noarch) # Use presence of expected packages to check for success if (docker_builder.pkg_dir is not None): - platform = utils.RepoData().native_platform() - subfolder = utils.RepoData().platform2subdir(platform) - config = utils.load_conda_build_config(platform=subfolder) - pkg_paths = [p.replace(config.output_folder, docker_builder.pkg_dir) for p in pkg_paths] + platform = utils.RepoData.native_platform() + subfolder = utils.RepoData.platform2subdir(platform) + conda_build_config = utils.load_conda_build_config(platform=subfolder) + pkg_paths = [p.replace(conda_build_config.output_folder, docker_builder.pkg_dir) for p in pkg_paths] for pkg_path in pkg_paths: if not os.path.exists(pkg_path): From 386b37638f05cdaba947b657e0497c9d28d9c7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Tue, 19 Nov 2024 16:13:12 +0100 Subject: [PATCH 14/14] Update bioconda_utils/build.py --- bioconda_utils/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index dc6eaedbfa..bbdd0c426a 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -142,7 +142,7 @@ def build(recipe: str, pkg_paths: List[str] = None, noarch=is_noarch, live_logs=live_logs) # Use presence of expected packages to check for success - if (docker_builder.pkg_dir is not None): + if docker_builder.pkg_dir is not None: platform = utils.RepoData.native_platform() subfolder = utils.RepoData.platform2subdir(platform) conda_build_config = utils.load_conda_build_config(platform=subfolder)