From 6541f2dc304e754ffec08293dea3483d5d0c3ac5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 1 Mar 2022 07:12:52 +0100 Subject: [PATCH 1/8] Remove copy tasks --- .../bakdata/gradle/SourceSetConfigurator.kt | 35 ++---------- .../gradle/AvroPluginIntegrationTest.kt | 54 +++++++++++++++++-- .../com/bakdata/gradle/AvroPluginTest.kt | 8 --- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt index 0a6f20a..b5e349c 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt @@ -29,10 +29,7 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.ConfigurationContainer -import org.gradle.api.file.Directory import org.gradle.api.file.FileCollection -import org.gradle.api.provider.Provider -import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Delete import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.compile.JavaCompile @@ -40,8 +37,6 @@ import java.io.File import java.util.zip.ZipEntry import java.util.zip.ZipFile -private const val EXTERNAL_AVRO_RESOURCES = "externalAvroResources" - private const val EXTERNAL_JAVA = "externalJava" class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { @@ -50,9 +45,6 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private val generateAvroJava: GenerateAvroJavaTask private val configureDeleteExternalJava: Task private val deleteExternalJava: Delete - private val externalAvroDir: Provider - private val configureCopyAvro: Task - private val copyAvro: Copy private val avroOutputs: FileCollection init { @@ -69,15 +61,6 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { dependsOn(configureDeleteExternalJava) group = generateAvroJava.group } - this.externalAvroDir = project.layout.buildDirectory.dir("external-${sourceSet.name}-avro") - this.configureCopyAvro = project.task(sourceSet.getTaskName("configureCopy", EXTERNAL_AVRO_RESOURCES)) { - group = generateAvroJava.group - } - this.copyAvro = - project.tasks.create(sourceSet.getTaskName("copy", EXTERNAL_AVRO_RESOURCES), Copy::class.java) { - dependsOn(configureCopyAvro) - group = generateAvroJava.group - } this.avroOutputs = generateAvroJava.outputs.files } @@ -120,21 +103,11 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private fun GenerateAvroJavaTask.addSources( avroConfiguration: Configuration ) { - configureCopyAvro.dependsOn(avroConfiguration) - // copy external avro files to separate build directory. - // Directly adding zipTree as source breaks caching: https://github.com/gradle/gradle/issues/18382 - configureCopyAvro.doLast { - copyAvro.from( - avroConfiguration.map { file: File -> - copyAvro.project.zipTree(file) - }) { - include("**/*.avsc") - } - copyAvro.into(externalAvroDir) - copyAvro.includeEmptyDirs = false + avroConfiguration.map { file: File -> + project.zipTree(file).files + }.forEach { + source(it) } - dependsOn(copyAvro) - source(externalAvroDir) } private fun Task.configureCompilation( diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt index 998d4c0..e9699ab 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt @@ -81,12 +81,8 @@ internal class AvroPluginIntegrationTest { softly.assertThat(result.tasks) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteExternalJava", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":configureCopyExternalAvroResources", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":copyExternalAvroResources", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteTestExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteTestExternalJava", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":configureCopyTestExternalAvroResources", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":copyTestExternalAvroResources", TaskOutcome.SUCCESS)) val javaClasses = testProjectDir.resolve("build/classes/java") softly.assertThat(javaClasses.resolve("main/com/bakdata/kafka/DeadLetter.class").toFile()) .doesNotExist() @@ -98,4 +94,54 @@ internal class AvroPluginIntegrationTest { .exists() } } + + @Test + fun shouldCacje(@TempDir testProjectDir: Path) { + Files.writeString( + testProjectDir.resolve("build.gradle.kts"), """ + plugins { + java + id("com.bakdata.avro") + } + repositories { + mavenCentral() + } + dependencies { + avroImplementation(group = "com.bakdata.kafka", name = "error-handling", version = "1.2.2") + } + """.trimIndent() + ) + Files.writeString( + testProjectDir.resolve("gradle.properties"), """ + org.gradle.caching=true + """.trimIndent() + ) + Files.createDirectories(testProjectDir.resolve("src/main/avro/")) + Files.copy( + AvroPluginIntegrationTest::class.java.getResourceAsStream("/Record.avsc"), + testProjectDir.resolve("src/main/avro/Record.avsc") + ) + Files.createDirectories(testProjectDir.resolve("src/test/avro/")) + Files.copy( + AvroPluginIntegrationTest::class.java.getResourceAsStream("/TestRecord.avsc"), + testProjectDir.resolve("src/test/avro/TestRecord.avsc") + ) + + GradleRunner.create() + .withProjectDir(testProjectDir.toFile()) + .withArguments("build") + .withProjectPluginClassPath() + .build() + + val result = GradleRunner.create() + .withProjectDir(testProjectDir.toFile()) + .withArguments("build") + .withProjectPluginClassPath() + .build() + + SoftAssertions.assertSoftly { softly -> + softly.assertThat(result.tasks) + .haveExactly(1, taskWithPathAndOutcome(":generateAvroJava", TaskOutcome.FROM_CACHE)) + } + } } \ No newline at end of file diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt index ab21e30..4a37d94 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt @@ -65,12 +65,8 @@ internal class AvroPluginTest { softly.assertThat(project.tasks) .haveExactly(1, taskWithName("configureDeleteExternalJava")) .haveExactly(1, taskWithName("deleteExternalJava")) - .haveExactly(1, taskWithName("configureCopyExternalAvroResources")) - .haveExactly(1, taskWithName("copyExternalAvroResources")) .haveExactly(1, taskWithName("configureDeleteTestExternalJava")) .haveExactly(1, taskWithName("deleteTestExternalJava")) - .haveExactly(1, taskWithName("configureCopyTestExternalAvroResources")) - .haveExactly(1, taskWithName("copyTestExternalAvroResources")) softly.assertThat(project.configurations) .haveExactly(1, configurationWithName("avroImplementation")) .haveExactly(0, configurationWithName("avroApi")) @@ -110,12 +106,8 @@ internal class AvroPluginTest { softly.assertThat(project.tasks) .haveExactly(1, taskWithName("configureDeleteExternalJava")) .haveExactly(1, taskWithName("deleteExternalJava")) - .haveExactly(1, taskWithName("configureCopyExternalAvroResources")) - .haveExactly(1, taskWithName("copyExternalAvroResources")) .haveExactly(1, taskWithName("configureDeleteTestExternalJava")) .haveExactly(1, taskWithName("deleteTestExternalJava")) - .haveExactly(1, taskWithName("configureCopyTestExternalAvroResources")) - .haveExactly(1, taskWithName("copyTestExternalAvroResources")) softly.assertThat(project.configurations) .haveExactly(1, configurationWithName("avroImplementation")) .haveExactly(1, configurationWithName("avroApi")) From 68e55744186c93a9a72745a21cf427bb1a70b8e3 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 1 Mar 2022 07:37:51 +0100 Subject: [PATCH 2/8] Remove copy tasks --- .../kotlin/com/bakdata/gradle/AvroPlugin.kt | 9 +++++---- .../bakdata/gradle/ConfigWithGenerateTask.kt | 15 +++++++++++++++ .../com/bakdata/gradle/SourceSetConfigurator.kt | 17 +++-------------- .../bakdata/gradle/AvroPluginIntegrationTest.kt | 2 +- .../kotlin/com/bakdata/gradle/AvroPluginTest.kt | 8 ++++---- 5 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt diff --git a/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt b/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt index 1326f96..33f8858 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt @@ -36,21 +36,22 @@ class AvroPlugin : Plugin { override fun apply(project: Project) { project.plugins.apply("com.github.davidmc24.gradle.plugin.avro") val sourceSets: SourceSetContainer = project.getSourceSets() - val configurationsWithAvroConfiguration: Map = sourceSets + val configurationsWithAvroConfiguration: Map = sourceSets .flatMap { sourceSet: SourceSet -> SourceSetConfigurator(project, sourceSet).configure() }.toMap() applyInheritance(configurationsWithAvroConfiguration) + configurationsWithAvroConfiguration.values.forEach(ConfigWithGenerateTask::addSources) } - private fun applyInheritance(configurationsWithAvroConfiguration: Map) { - configurationsWithAvroConfiguration.forEach { (originalConfiguration: Configuration, avroConfiguration: Configuration) -> + private fun applyInheritance(configurationsWithAvroConfiguration: Map) { + configurationsWithAvroConfiguration.forEach { (originalConfiguration: Configuration, configWithGenerateTask: ConfigWithGenerateTask) -> originalConfiguration.extendsFrom.forEach { extendsFrom: Configuration -> // check if there is an avro configuration for the configuration it extends from val extendsFromAvroConfiguration = configurationsWithAvroConfiguration[extendsFrom] extendsFromAvroConfiguration?.also { // if yes, avroConfiguration should extend from the respective avro configuration - avroConfiguration.extendsFrom(it) + configWithGenerateTask.avroConfiguration.extendsFrom(it.avroConfiguration) } } } diff --git a/avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt b/avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt new file mode 100644 index 0000000..cb84efb --- /dev/null +++ b/avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt @@ -0,0 +1,15 @@ +package com.bakdata.gradle + +import com.github.davidmc24.gradle.plugin.avro.GenerateAvroJavaTask +import org.gradle.api.artifacts.Configuration +import java.io.File + +class ConfigWithGenerateTask(val avroConfiguration: Configuration, val generateAvroJava: GenerateAvroJavaTask) { + fun addSources() { + avroConfiguration.map { file: File -> + generateAvroJava.project.zipTree(file).files + }.forEach { files: Set -> + generateAvroJava.source(files) + } + } +} diff --git a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt index b5e349c..ef2c2fa 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt @@ -64,7 +64,7 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { this.avroOutputs = generateAvroJava.outputs.files } - fun configure(): List> { + fun configure(): List> { val compileJava: JavaCompile = project.tasks.named(sourceSet.compileJavaTaskName, JavaCompile::class.java).get() compileJava.dependsOn(deleteExternalJava) @@ -81,14 +81,14 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private fun ConfigurationContainer.setupConfiguration( configurationName: String - ): Pair? { + ): Pair? { return findByName(configurationName)?.let { configuration: Configuration -> val name: String = sourceSet.getConfigurationName("avro", configurationName) val avroConfiguration: Configuration = create(name) configuration.setupConfiguration( avroConfiguration ) - configuration to avroConfiguration + configuration to ConfigWithGenerateTask(avroConfiguration, generateAvroJava) } } @@ -96,20 +96,9 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { avroConfiguration: Configuration ) { extendsFrom(avroConfiguration) - generateAvroJava.addSources(avroConfiguration) configureDeleteExternalJava.configureCompilation(avroConfiguration) } - private fun GenerateAvroJavaTask.addSources( - avroConfiguration: Configuration - ) { - avroConfiguration.map { file: File -> - project.zipTree(file).files - }.forEach { - source(it) - } - } - private fun Task.configureCompilation( avroConfiguration: Configuration ) { diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt index e9699ab..21d99c7 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt @@ -96,7 +96,7 @@ internal class AvroPluginIntegrationTest { } @Test - fun shouldCacje(@TempDir testProjectDir: Path) { + fun shouldCache(@TempDir testProjectDir: Path) { Files.writeString( testProjectDir.resolve("build.gradle.kts"), """ plugins { diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt index 4a37d94..aacdd1f 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt @@ -38,14 +38,14 @@ import org.junit.jupiter.api.Test import java.io.File internal class AvroPluginTest { - fun taskWithName(name: String): Condition = Condition({ it.name == name }, "Task with name $name") - fun configurationWithName(name: String): Condition = + private fun taskWithName(name: String): Condition = Condition({ it.name == name }, "Task with name $name") + private fun configurationWithName(name: String): Condition = Condition({ it.name == name }, "Configuration with name $name") - fun folderWithName(name: String): Condition = + private fun folderWithName(name: String): Condition = Condition({ it.path.endsWith(name.replace("/", File.separator)) }, "File with name $name") - fun Project.evaluate() { + private fun Project.evaluate() { (this as DefaultProject).evaluate() } From 9e0ccd7fdb9f077acd16f692331052a244b223ac Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 1 Mar 2022 07:45:35 +0100 Subject: [PATCH 3/8] Remove copy tasks --- .../kotlin/com/bakdata/gradle/AvroPlugin.kt | 9 ++-- .../bakdata/gradle/ConfigWithGenerateTask.kt | 15 ------- .../bakdata/gradle/SourceSetConfigurator.kt | 44 +++++++++++++++++-- .../gradle/AvroPluginIntegrationTest.kt | 4 ++ .../com/bakdata/gradle/AvroPluginTest.kt | 8 ++++ 5 files changed, 57 insertions(+), 23 deletions(-) delete mode 100644 avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt diff --git a/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt b/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt index 33f8858..1326f96 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/AvroPlugin.kt @@ -36,22 +36,21 @@ class AvroPlugin : Plugin { override fun apply(project: Project) { project.plugins.apply("com.github.davidmc24.gradle.plugin.avro") val sourceSets: SourceSetContainer = project.getSourceSets() - val configurationsWithAvroConfiguration: Map = sourceSets + val configurationsWithAvroConfiguration: Map = sourceSets .flatMap { sourceSet: SourceSet -> SourceSetConfigurator(project, sourceSet).configure() }.toMap() applyInheritance(configurationsWithAvroConfiguration) - configurationsWithAvroConfiguration.values.forEach(ConfigWithGenerateTask::addSources) } - private fun applyInheritance(configurationsWithAvroConfiguration: Map) { - configurationsWithAvroConfiguration.forEach { (originalConfiguration: Configuration, configWithGenerateTask: ConfigWithGenerateTask) -> + private fun applyInheritance(configurationsWithAvroConfiguration: Map) { + configurationsWithAvroConfiguration.forEach { (originalConfiguration: Configuration, avroConfiguration: Configuration) -> originalConfiguration.extendsFrom.forEach { extendsFrom: Configuration -> // check if there is an avro configuration for the configuration it extends from val extendsFromAvroConfiguration = configurationsWithAvroConfiguration[extendsFrom] extendsFromAvroConfiguration?.also { // if yes, avroConfiguration should extend from the respective avro configuration - configWithGenerateTask.avroConfiguration.extendsFrom(it.avroConfiguration) + avroConfiguration.extendsFrom(it) } } } diff --git a/avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt b/avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt deleted file mode 100644 index cb84efb..0000000 --- a/avro/src/main/kotlin/com/bakdata/gradle/ConfigWithGenerateTask.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.bakdata.gradle - -import com.github.davidmc24.gradle.plugin.avro.GenerateAvroJavaTask -import org.gradle.api.artifacts.Configuration -import java.io.File - -class ConfigWithGenerateTask(val avroConfiguration: Configuration, val generateAvroJava: GenerateAvroJavaTask) { - fun addSources() { - avroConfiguration.map { file: File -> - generateAvroJava.project.zipTree(file).files - }.forEach { files: Set -> - generateAvroJava.source(files) - } - } -} diff --git a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt index ef2c2fa..0a6f20a 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt @@ -29,7 +29,10 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.ConfigurationContainer +import org.gradle.api.file.Directory import org.gradle.api.file.FileCollection +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Delete import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.compile.JavaCompile @@ -37,6 +40,8 @@ import java.io.File import java.util.zip.ZipEntry import java.util.zip.ZipFile +private const val EXTERNAL_AVRO_RESOURCES = "externalAvroResources" + private const val EXTERNAL_JAVA = "externalJava" class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { @@ -45,6 +50,9 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private val generateAvroJava: GenerateAvroJavaTask private val configureDeleteExternalJava: Task private val deleteExternalJava: Delete + private val externalAvroDir: Provider + private val configureCopyAvro: Task + private val copyAvro: Copy private val avroOutputs: FileCollection init { @@ -61,10 +69,19 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { dependsOn(configureDeleteExternalJava) group = generateAvroJava.group } + this.externalAvroDir = project.layout.buildDirectory.dir("external-${sourceSet.name}-avro") + this.configureCopyAvro = project.task(sourceSet.getTaskName("configureCopy", EXTERNAL_AVRO_RESOURCES)) { + group = generateAvroJava.group + } + this.copyAvro = + project.tasks.create(sourceSet.getTaskName("copy", EXTERNAL_AVRO_RESOURCES), Copy::class.java) { + dependsOn(configureCopyAvro) + group = generateAvroJava.group + } this.avroOutputs = generateAvroJava.outputs.files } - fun configure(): List> { + fun configure(): List> { val compileJava: JavaCompile = project.tasks.named(sourceSet.compileJavaTaskName, JavaCompile::class.java).get() compileJava.dependsOn(deleteExternalJava) @@ -81,14 +98,14 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private fun ConfigurationContainer.setupConfiguration( configurationName: String - ): Pair? { + ): Pair? { return findByName(configurationName)?.let { configuration: Configuration -> val name: String = sourceSet.getConfigurationName("avro", configurationName) val avroConfiguration: Configuration = create(name) configuration.setupConfiguration( avroConfiguration ) - configuration to ConfigWithGenerateTask(avroConfiguration, generateAvroJava) + configuration to avroConfiguration } } @@ -96,9 +113,30 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { avroConfiguration: Configuration ) { extendsFrom(avroConfiguration) + generateAvroJava.addSources(avroConfiguration) configureDeleteExternalJava.configureCompilation(avroConfiguration) } + private fun GenerateAvroJavaTask.addSources( + avroConfiguration: Configuration + ) { + configureCopyAvro.dependsOn(avroConfiguration) + // copy external avro files to separate build directory. + // Directly adding zipTree as source breaks caching: https://github.com/gradle/gradle/issues/18382 + configureCopyAvro.doLast { + copyAvro.from( + avroConfiguration.map { file: File -> + copyAvro.project.zipTree(file) + }) { + include("**/*.avsc") + } + copyAvro.into(externalAvroDir) + copyAvro.includeEmptyDirs = false + } + dependsOn(copyAvro) + source(externalAvroDir) + } + private fun Task.configureCompilation( avroConfiguration: Configuration ) { diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt index 21d99c7..c628385 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt @@ -81,8 +81,12 @@ internal class AvroPluginIntegrationTest { softly.assertThat(result.tasks) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteExternalJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":configureCopyExternalAvroResources", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":copyExternalAvroResources", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteTestExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteTestExternalJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":configureCopyTestExternalAvroResources", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":copyTestExternalAvroResources", TaskOutcome.SUCCESS)) val javaClasses = testProjectDir.resolve("build/classes/java") softly.assertThat(javaClasses.resolve("main/com/bakdata/kafka/DeadLetter.class").toFile()) .doesNotExist() diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt index aacdd1f..0f688f0 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt @@ -65,8 +65,12 @@ internal class AvroPluginTest { softly.assertThat(project.tasks) .haveExactly(1, taskWithName("configureDeleteExternalJava")) .haveExactly(1, taskWithName("deleteExternalJava")) + .haveExactly(1, taskWithName("configureCopyExternalAvroResources")) + .haveExactly(1, taskWithName("copyExternalAvroResources")) .haveExactly(1, taskWithName("configureDeleteTestExternalJava")) .haveExactly(1, taskWithName("deleteTestExternalJava")) + .haveExactly(1, taskWithName("configureCopyTestExternalAvroResources")) + .haveExactly(1, taskWithName("copyTestExternalAvroResources")) softly.assertThat(project.configurations) .haveExactly(1, configurationWithName("avroImplementation")) .haveExactly(0, configurationWithName("avroApi")) @@ -106,8 +110,12 @@ internal class AvroPluginTest { softly.assertThat(project.tasks) .haveExactly(1, taskWithName("configureDeleteExternalJava")) .haveExactly(1, taskWithName("deleteExternalJava")) + .haveExactly(1, taskWithName("configureCopyExternalAvroResources")) + .haveExactly(1, taskWithName("copyExternalAvroResources")) .haveExactly(1, taskWithName("configureDeleteTestExternalJava")) .haveExactly(1, taskWithName("deleteTestExternalJava")) + .haveExactly(1, taskWithName("configureCopyTestExternalAvroResources")) + .haveExactly(1, taskWithName("copyTestExternalAvroResources")) softly.assertThat(project.configurations) .haveExactly(1, configurationWithName("avroImplementation")) .haveExactly(1, configurationWithName("avroApi")) From 8f9653887f0347637d75fd3a6d1e579bc3275212 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 1 Mar 2022 07:51:42 +0100 Subject: [PATCH 4/8] Remove copy tasks --- .../bakdata/gradle/SourceSetConfigurator.kt | 37 +++++-------------- .../gradle/AvroPluginIntegrationTest.kt | 6 +-- .../com/bakdata/gradle/AvroPluginTest.kt | 12 ++---- 3 files changed, 16 insertions(+), 39 deletions(-) diff --git a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt index 0a6f20a..f148a2b 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt @@ -29,10 +29,7 @@ import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.ConfigurationContainer -import org.gradle.api.file.Directory import org.gradle.api.file.FileCollection -import org.gradle.api.provider.Provider -import org.gradle.api.tasks.Copy import org.gradle.api.tasks.Delete import org.gradle.api.tasks.SourceSet import org.gradle.api.tasks.compile.JavaCompile @@ -40,7 +37,7 @@ import java.io.File import java.util.zip.ZipEntry import java.util.zip.ZipFile -private const val EXTERNAL_AVRO_RESOURCES = "externalAvroResources" +private const val AVRO_JAVA = "avroJava" private const val EXTERNAL_JAVA = "externalJava" @@ -50,16 +47,14 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private val generateAvroJava: GenerateAvroJavaTask private val configureDeleteExternalJava: Task private val deleteExternalJava: Delete - private val externalAvroDir: Provider private val configureCopyAvro: Task - private val copyAvro: Copy private val avroOutputs: FileCollection init { this.project = project this.sourceSet = sourceSet this.generateAvroJava = - project.tasks.named(sourceSet.getTaskName("generate", "avroJava"), GenerateAvroJavaTask::class.java).get() + project.tasks.named(sourceSet.getTaskName("generate", AVRO_JAVA), GenerateAvroJavaTask::class.java).get() this.configureDeleteExternalJava = project.task(sourceSet.getTaskName("configureDelete", EXTERNAL_JAVA)) { dependsOn(generateAvroJava) group = generateAvroJava.group @@ -69,15 +64,10 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { dependsOn(configureDeleteExternalJava) group = generateAvroJava.group } - this.externalAvroDir = project.layout.buildDirectory.dir("external-${sourceSet.name}-avro") - this.configureCopyAvro = project.task(sourceSet.getTaskName("configureCopy", EXTERNAL_AVRO_RESOURCES)) { + this.configureCopyAvro = project.task(sourceSet.getTaskName("configureGenerate", AVRO_JAVA)) { group = generateAvroJava.group } - this.copyAvro = - project.tasks.create(sourceSet.getTaskName("copy", EXTERNAL_AVRO_RESOURCES), Copy::class.java) { - dependsOn(configureCopyAvro) - group = generateAvroJava.group - } + generateAvroJava.dependsOn(configureCopyAvro) this.avroOutputs = generateAvroJava.outputs.files } @@ -113,28 +103,21 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { avroConfiguration: Configuration ) { extendsFrom(avroConfiguration) - generateAvroJava.addSources(avroConfiguration) + addSources(avroConfiguration) configureDeleteExternalJava.configureCompilation(avroConfiguration) } - private fun GenerateAvroJavaTask.addSources( + private fun addSources( avroConfiguration: Configuration ) { configureCopyAvro.dependsOn(avroConfiguration) - // copy external avro files to separate build directory. - // Directly adding zipTree as source breaks caching: https://github.com/gradle/gradle/issues/18382 configureCopyAvro.doLast { - copyAvro.from( - avroConfiguration.map { file: File -> - copyAvro.project.zipTree(file) - }) { - include("**/*.avsc") + avroConfiguration.map { file: File -> + project.zipTree(file).files + }.forEach { files: Set -> + generateAvroJava.source(files) } - copyAvro.into(externalAvroDir) - copyAvro.includeEmptyDirs = false } - dependsOn(copyAvro) - source(externalAvroDir) } private fun Task.configureCompilation( diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt index c628385..6f448d0 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt @@ -81,12 +81,10 @@ internal class AvroPluginIntegrationTest { softly.assertThat(result.tasks) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteExternalJava", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":configureCopyExternalAvroResources", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":copyExternalAvroResources", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":configureGenerateAvroJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteTestExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteTestExternalJava", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":configureCopyTestExternalAvroResources", TaskOutcome.SUCCESS)) - .haveExactly(1, taskWithPathAndOutcome(":copyTestExternalAvroResources", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":configureGenerateTestAvroJava", TaskOutcome.SUCCESS)) val javaClasses = testProjectDir.resolve("build/classes/java") softly.assertThat(javaClasses.resolve("main/com/bakdata/kafka/DeadLetter.class").toFile()) .doesNotExist() diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt index 0f688f0..587a6d3 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginTest.kt @@ -65,12 +65,10 @@ internal class AvroPluginTest { softly.assertThat(project.tasks) .haveExactly(1, taskWithName("configureDeleteExternalJava")) .haveExactly(1, taskWithName("deleteExternalJava")) - .haveExactly(1, taskWithName("configureCopyExternalAvroResources")) - .haveExactly(1, taskWithName("copyExternalAvroResources")) + .haveExactly(1, taskWithName("configureGenerateAvroJava")) .haveExactly(1, taskWithName("configureDeleteTestExternalJava")) .haveExactly(1, taskWithName("deleteTestExternalJava")) - .haveExactly(1, taskWithName("configureCopyTestExternalAvroResources")) - .haveExactly(1, taskWithName("copyTestExternalAvroResources")) + .haveExactly(1, taskWithName("configureGenerateTestAvroJava")) softly.assertThat(project.configurations) .haveExactly(1, configurationWithName("avroImplementation")) .haveExactly(0, configurationWithName("avroApi")) @@ -110,12 +108,10 @@ internal class AvroPluginTest { softly.assertThat(project.tasks) .haveExactly(1, taskWithName("configureDeleteExternalJava")) .haveExactly(1, taskWithName("deleteExternalJava")) - .haveExactly(1, taskWithName("configureCopyExternalAvroResources")) - .haveExactly(1, taskWithName("copyExternalAvroResources")) + .haveExactly(1, taskWithName("configureGenerateAvroJava")) .haveExactly(1, taskWithName("configureDeleteTestExternalJava")) .haveExactly(1, taskWithName("deleteTestExternalJava")) - .haveExactly(1, taskWithName("configureCopyTestExternalAvroResources")) - .haveExactly(1, taskWithName("copyTestExternalAvroResources")) + .haveExactly(1, taskWithName("configureGenerateTestAvroJava")) softly.assertThat(project.configurations) .haveExactly(1, configurationWithName("avroImplementation")) .haveExactly(1, configurationWithName("avroApi")) From ac829b620006ab2d6923b6713f28202cf8d0f2be Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 1 Mar 2022 07:52:30 +0100 Subject: [PATCH 5/8] Remove copy tasks --- .../kotlin/com/bakdata/gradle/SourceSetConfigurator.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt index f148a2b..12813bd 100644 --- a/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt +++ b/avro/src/main/kotlin/com/bakdata/gradle/SourceSetConfigurator.kt @@ -47,7 +47,7 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private val generateAvroJava: GenerateAvroJavaTask private val configureDeleteExternalJava: Task private val deleteExternalJava: Delete - private val configureCopyAvro: Task + private val configureGenerateAvroJava: Task private val avroOutputs: FileCollection init { @@ -64,10 +64,10 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { dependsOn(configureDeleteExternalJava) group = generateAvroJava.group } - this.configureCopyAvro = project.task(sourceSet.getTaskName("configureGenerate", AVRO_JAVA)) { + this.configureGenerateAvroJava = project.task(sourceSet.getTaskName("configureGenerate", AVRO_JAVA)) { group = generateAvroJava.group } - generateAvroJava.dependsOn(configureCopyAvro) + generateAvroJava.dependsOn(configureGenerateAvroJava) this.avroOutputs = generateAvroJava.outputs.files } @@ -110,8 +110,8 @@ class SourceSetConfigurator(project: Project, sourceSet: SourceSet) { private fun addSources( avroConfiguration: Configuration ) { - configureCopyAvro.dependsOn(avroConfiguration) - configureCopyAvro.doLast { + configureGenerateAvroJava.dependsOn(avroConfiguration) + configureGenerateAvroJava.doLast { avroConfiguration.map { file: File -> project.zipTree(file).files }.forEach { files: Set -> From 9c791377ebfa3e840b5abb257df45ad35f5d6ddd Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 1 Mar 2022 09:14:41 +0100 Subject: [PATCH 6/8] Fix tests --- .../com/bakdata/gradle/AvroPluginIntegrationTest.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt index 6f448d0..7ea5923 100644 --- a/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt +++ b/avro/src/test/kotlin/com/bakdata/gradle/AvroPluginIntegrationTest.kt @@ -82,9 +82,11 @@ internal class AvroPluginIntegrationTest { .haveExactly(1, taskWithPathAndOutcome(":configureDeleteExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":configureGenerateAvroJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":generateAvroJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":configureDeleteTestExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":deleteTestExternalJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":configureGenerateTestAvroJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":generateTestAvroJava", TaskOutcome.SUCCESS)) val javaClasses = testProjectDir.resolve("build/classes/java") softly.assertThat(javaClasses.resolve("main/com/bakdata/kafka/DeadLetter.class").toFile()) .doesNotExist() @@ -143,7 +145,14 @@ internal class AvroPluginIntegrationTest { SoftAssertions.assertSoftly { softly -> softly.assertThat(result.tasks) + .haveExactly(1, taskWithPathAndOutcome(":configureDeleteExternalJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":deleteExternalJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":configureGenerateAvroJava", TaskOutcome.SUCCESS)) .haveExactly(1, taskWithPathAndOutcome(":generateAvroJava", TaskOutcome.FROM_CACHE)) + .haveExactly(1, taskWithPathAndOutcome(":configureDeleteTestExternalJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":deleteTestExternalJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":configureGenerateTestAvroJava", TaskOutcome.SUCCESS)) + .haveExactly(1, taskWithPathAndOutcome(":generateTestAvroJava", TaskOutcome.FROM_CACHE)) } } } \ No newline at end of file From a7db868d0425e77e63e331e113fde9bda0c7deef Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 15 Mar 2022 09:36:20 +0100 Subject: [PATCH 7/8] Debug java library plugin --- azure-pipelines.yml | 1 + gradle.properties | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5aa69d4..aae15af 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,7 @@ resources: repositories: - repository: templates type: github + ref: tmp/upload-gradle-plugin name: bakdata/bakdata-project-templates endpoint: bot diff --git a/gradle.properties b/gradle.properties index 04378e7..93b56e8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -version=1.0.1-SNAPSHOT +version=0.0.2 org.gradle.caching=true org.gradle.parallel=true From 361811bd371f1ce64767fcce4e980899f554dfc6 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Tue, 15 Mar 2022 10:06:42 +0100 Subject: [PATCH 8/8] Debug java library plugin --- azure-pipelines.yml | 1 - gradle.properties | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aae15af..5aa69d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,6 @@ resources: repositories: - repository: templates type: github - ref: tmp/upload-gradle-plugin name: bakdata/bakdata-project-templates endpoint: bot diff --git a/gradle.properties b/gradle.properties index 93b56e8..04378e7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -version=0.0.2 +version=1.0.1-SNAPSHOT org.gradle.caching=true org.gradle.parallel=true