@JenkinsRecipe does not work with JUnit Jupiter extension

[mkobit]

Jenkins and plugins versions report

Environment

Apache Maven 3.9.6 (bc0240f3c744dd6b6ec2920b3cd08dcc295161ae)
Maven home: /usr/local/Cellar/maven/3.9.6/libexec
Java version: 17.0.9, vendor: Homebrew, runtime: /usr/local/Cellar/openjdk@17/17.0.9/libexec/openjdk.jdk/Contents/Home
Default locale: en_US, platform encoding: UTF-8
OS name: "mac os x", version: "14.2.1", arch: "x86_64", family: "mac"

What Operating System are you using (both controller, and any agents involved in the problem)?

N/A

Reproduction steps

package org.jvnet.hudson.test.junit.jupiter;

import hudson.model.FreeStyleProject;
import org.junit.jupiter.api.Test;
import org.jvnet.hudson.test.JenkinsRecipe;
import org.jvnet.hudson.test.JenkinsRule;
import org.jvnet.hudson.test.MockFolder;

import java.io.File;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;

@WithJenkins
@JenkinsRuleRecipeExecutionTest.FreestyleJobRecipe("onClass")
public class JenkinsRuleRecipeExecutionTest {
    @Test
    void recipeOnClassIsExecuted(JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasSize(1));
        assertThat(rule.jenkins.getJobNames(), hasItem("onClass"));
    }

    @Test
    @FreestyleJobRecipe("onMethod")
    void recipeOnMethodIsExecuted(JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasItem("onMethod"));
    }

    @Test
    void recipeOnParameterIsExecuted(@FreestyleJobRecipe("onParameter") JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasItem("onParameter"));
    }

    @Test
    @FreestyleJobRecipe("onMethod")
    void allRecipesAreExecuted(@FreestyleJobRecipe("onParameter") JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasSize(3));
        assertThat(rule.jenkins.getJobNames(), hasItems("onClass", "onMethod", "onParameter"));
    }

    @Test
    @FreestyleJobRecipe("freestyleOnMethod")
    @MockFolderRecipe("folderOnMethod")
    void differentRecipesOnMethodAreExecuted(JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasItems("freestyleOnMethod", "folderOnMethod"));
    }

    @Test
    void differentRecipesOnParameterAreExecuted(
            @FreestyleJobRecipe("freestyleOnParam") @MockFolderRecipe("folderOnParam") JenkinsRule rule
    ) {
        assertThat(rule.jenkins.getJobNames(), hasItems("freestyleOnParam", "folderOnParam"));
    }

    @Test
    @ComposedRecipe
    void composedRecipeAreBothExecutedOnMethod(JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasItems("composedFreestyle", "composedFolder"));
    }

    @Test
    void composedRecipeAreBothExecutedOnParameter(@ComposedRecipe JenkinsRule rule) {
        assertThat(rule.jenkins.getJobNames(), hasItems("composedFreestyle", "composedFolder"));
    }

    @JenkinsRecipe(FreestyleJobRecipe.RecipeImpl.class)
    @Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.TYPE})
    @Retention(RetentionPolicy.RUNTIME)
    public @interface FreestyleJobRecipe {
        String value();
        class RecipeImpl extends JenkinsRecipe.Runner<FreestyleJobRecipe> {
            @Override
            public void setup(JenkinsRule jenkinsRule, FreestyleJobRecipe recipe) throws Exception {
                jenkinsRule.createFreeStyleProject(recipe.value());
            }

            @Override
            public void decorateHome(JenkinsRule jenkinsRule, File home) throws Exception {
            }

            @Override
            public void tearDown(JenkinsRule jenkinsRule, FreestyleJobRecipe recipe) throws Exception {
                final FreeStyleProject item = jenkinsRule.jenkins.getItem(recipe.value(), jenkinsRule.jenkins, FreeStyleProject.class);
                assertThat(item, notNullValue());
                jenkinsRule.jenkins.remove(item);
            }
        }
    }

    @JenkinsRecipe(MockFolderRecipe.RecipeImpl.class)
    @Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.TYPE})
    @Retention(RetentionPolicy.RUNTIME)
    public @interface MockFolderRecipe {
        String value();
        class RecipeImpl extends JenkinsRecipe.Runner<MockFolderRecipe> {
            @Override
            public void setup(JenkinsRule jenkinsRule, MockFolderRecipe recipe) throws Exception {
                jenkinsRule.createFolder(recipe.value());
            }

            @Override
            public void decorateHome(JenkinsRule jenkinsRule, File home) throws Exception {
            }

            @Override
            public void tearDown(JenkinsRule jenkinsRule, MockFolderRecipe recipe) throws Exception {
                MockFolder item = jenkinsRule.jenkins.getItem(recipe.value(), jenkinsRule.jenkins, MockFolder.class);
                assertThat(item, notNullValue());
                jenkinsRule.jenkins.remove(item);
            }
        }
    }

    @FreestyleJobRecipe("composedFreestyle")
    @MockFolderRecipe("composedFolder")
    @Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.TYPE})
    @Retention(RetentionPolicy.RUNTIME)
    public @interface ComposedRecipe {
    }
}

mvn -Dtest=JenkinsRuleRecipeExecutionTest test

Expected Results

Tests (at least most of these) should be executed, and the recipe

Actual Results

[ERROR] Tests run: 8, Failures: 8, Errors: 0, Skipped: 0, Time elapsed: 8.040 s <<< FAILURE! -- in org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest
[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.composedRecipeAreBothExecutedOnMethod(JenkinsRule) -- Time elapsed: 4.021 s <<< FAILURE!
java.lang.AssertionError:

Expected: (a collection containing "composedFreestyle" and a collection containing "composedFolder")
     but: a collection containing "composedFreestyle" was empty
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.composedRecipeAreBothExecutedOnMethod(JenkinsRuleRecipeExecutionTest.java:65)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.differentRecipesOnParameterAreExecuted(JenkinsRule) -- Time elapsed: 0.506 s <<< FAILURE!
java.lang.AssertionError:

Expected: (a collection containing "freestyleOnParam" and a collection containing "folderOnParam")
     but: a collection containing "freestyleOnParam" was empty
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.differentRecipesOnParameterAreExecuted(JenkinsRuleRecipeExecutionTest.java:59)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.recipeOnClassIsExecuted(JenkinsRule) -- Time elapsed: 0.842 s <<< FAILURE!
java.lang.AssertionError:

Expected: a collection with size <1>
     but: collection size was <0>
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.recipeOnClassIsExecuted(JenkinsRuleRecipeExecutionTest.java:26)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.composedRecipeAreBothExecutedOnParameter(JenkinsRule) -- Time elapsed: 0.498 s <<< FAILURE!
java.lang.AssertionError:

Expected: (a collection containing "composedFreestyle" and a collection containing "composedFolder")
     but: a collection containing "composedFreestyle" was empty
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.composedRecipeAreBothExecutedOnParameter(JenkinsRuleRecipeExecutionTest.java:70)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.allRecipesAreExecuted(JenkinsRule) -- Time elapsed: 0.428 s <<< FAILURE!
java.lang.AssertionError:

Expected: a collection with size <3>
     but: collection size was <0>
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.allRecipesAreExecuted(JenkinsRuleRecipeExecutionTest.java:44)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.differentRecipesOnMethodAreExecuted(JenkinsRule) -- Time elapsed: 0.528 s <<< FAILURE!
java.lang.AssertionError:

Expected: (a collection containing "freestyleOnMethod" and a collection containing "folderOnMethod")
     but: a collection containing "freestyleOnMethod" was empty
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.differentRecipesOnMethodAreExecuted(JenkinsRuleRecipeExecutionTest.java:52)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.recipeOnMethodIsExecuted(JenkinsRule) -- Time elapsed: 0.529 s <<< FAILURE!
java.lang.AssertionError:

Expected: a collection containing "onMethod"
     but: was empty
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.recipeOnMethodIsExecuted(JenkinsRuleRecipeExecutionTest.java:33)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

[ERROR] org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.recipeOnParameterIsExecuted(JenkinsRule) -- Time elapsed: 0.656 s <<< FAILURE!
java.lang.AssertionError:

Expected: a collection containing "onParameter"
     but: was empty
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:20)
	at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:6)
	at org.jvnet.hudson.test.junit.jupiter.JenkinsRuleRecipeExecutionTest.recipeOnParameterIsExecuted(JenkinsRuleRecipeExecutionTest.java:38)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
	at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)

• Additionally, it doesn't seem like the methods for each recipe implementation are appropriately being executed

Anything else?

Unclear to me which of these (and others) should thoeretically be supported by @JenkinsRecipe, or if a new JUnit Jupiter @*Recipe-escue annotation should be introduced

Are you interested in contributing a fix?

Yes - would like to know which of these test cases (and missing coverage) should be supported by the extension