From 91924a7dec57e7eeeae56bbb2e78968859576219 Mon Sep 17 00:00:00 2001 From: Dmitry Tretyakov Date: Thu, 28 Dec 2017 14:09:01 +0100 Subject: [PATCH] Try to automatically recover from toolchain install failures Issue: #37 --- build.gradle | 2 + ...Factory.kt => CargoBuildSessionFactory.kt} | 11 +-- .../rust/CargoCommandBuildSession.kt | 85 +++++++++++++++++++ .../rust/CargoRunnerBuildService.kt | 26 +++--- .../rust/CommandExecutionAdapter.kt | 65 ++++++++++++++ .../buildServer/rust/RustupBuildService.kt | 79 +++++++++++++++++ .../buildServer/rust/logging/BlockListener.kt | 17 ++++ .../META-INF/build-agent-plugin-rust.xml | 2 +- 8 files changed, 266 insertions(+), 21 deletions(-) rename plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/{CargoRunnerBuildServiceFactory.kt => CargoBuildSessionFactory.kt} (65%) create mode 100644 plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoCommandBuildSession.kt create mode 100644 plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CommandExecutionAdapter.kt create mode 100644 plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/RustupBuildService.kt create mode 100644 plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/logging/BlockListener.kt diff --git a/build.gradle b/build.gradle index ec71eea..ac73501 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,8 @@ allprojects { subprojects { apply plugin: "kotlin" + kotlin { experimental { coroutines 'enable' } } + test.useTestNG() jar.version = null diff --git a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildServiceFactory.kt b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoBuildSessionFactory.kt similarity index 65% rename from plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildServiceFactory.kt rename to plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoBuildSessionFactory.kt index d64fdc9..aa56c2e 100644 --- a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildServiceFactory.kt +++ b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoBuildSessionFactory.kt @@ -9,18 +9,15 @@ package jetbrains.buildServer.rust import jetbrains.buildServer.agent.AgentBuildRunnerInfo import jetbrains.buildServer.agent.BuildAgentConfiguration -import jetbrains.buildServer.agent.runner.CommandLineBuildService -import jetbrains.buildServer.agent.runner.CommandLineBuildServiceFactory +import jetbrains.buildServer.agent.BuildRunnerContext +import jetbrains.buildServer.agent.runner.MultiCommandBuildSessionFactory /** * Cargo runner service factory. */ -class CargoRunnerBuildServiceFactory(private val commandExecutor: CommandExecutor) - : CommandLineBuildServiceFactory { +class CargoBuildSessionFactory : MultiCommandBuildSessionFactory { - override fun createService(): CommandLineBuildService { - return CargoRunnerBuildService(commandExecutor) - } + override fun createSession(runnerContext: BuildRunnerContext) = CargoCommandBuildSession(runnerContext) override fun getBuildRunnerInfo(): AgentBuildRunnerInfo { return object : AgentBuildRunnerInfo { diff --git a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoCommandBuildSession.kt b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoCommandBuildSession.kt new file mode 100644 index 0000000..c7336a7 --- /dev/null +++ b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoCommandBuildSession.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * See LICENSE in the project root for license information. + */ + +package jetbrains.buildServer.rust + +import jetbrains.buildServer.agent.BuildFinishedStatus +import jetbrains.buildServer.agent.BuildRunnerContext +import jetbrains.buildServer.agent.runner.CommandExecution +import jetbrains.buildServer.agent.runner.CommandLineBuildService +import jetbrains.buildServer.agent.runner.MultiCommandBuildSession +import jetbrains.buildServer.util.FileUtil +import java.io.File +import kotlin.coroutines.experimental.buildIterator + +/** + * Cargo runner service. + */ +class CargoCommandBuildSession(private val runnerContext: BuildRunnerContext) : MultiCommandBuildSession { + + private var buildSteps: Iterator? = null + private var lastCommands = arrayListOf() + + override fun sessionStarted() { + buildSteps = getSteps() + } + + override fun getNextCommand(): CommandExecution? { + buildSteps?.let { + if (it.hasNext()) { + return it.next() + } + } + + return null + } + + override fun sessionFinished(): BuildFinishedStatus? { + return lastCommands.last().result + } + + private fun getSteps() = buildIterator { + runnerContext.runnerParameters[CargoConstants.PARAM_TOOLCHAIN]?.let { + if (it.isNotBlank()) { + val installToolchain = RustupBuildService("install") + yield(addCommand(installToolchain)) + + // Rustup could fail to install toolchain + // We could try to resolve it by execution uninstall of toolchain + // and cleaning up temporary directories + if (installToolchain.errors.isNotEmpty()) { + val logger = runnerContext.build.buildLogger + logger.message("Installation has failed, will remove toolchain '${installToolchain.version}' and try again") + + val uninstallToolchain = RustupBuildService("uninstall") + yield(addCommand(uninstallToolchain)) + + val rustupCache = File(System.getProperty("user.home"), ".rustup") + installToolchain.version.let { + // Cleanup temp directories + FileUtil.delete(File(rustupCache, CargoConstants.RUSTUP_DOWNLOADS_DIR)) + FileUtil.delete(File(rustupCache, CargoConstants.RUSTUP_TMP_DIR)) + + // Remove toolchain files + FileUtil.delete(File(rustupCache, "${CargoConstants.RUSTUP_TOOLCHAINS_DIR}/$it")) + FileUtil.delete(File(rustupCache, "${CargoConstants.RUSTUP_HASHES_DIR}/$it")) + } + + yield(addCommand(RustupBuildService("install"))) + } + } + } + + yield(addCommand(CargoRunnerBuildService())) + } + + private fun addCommand(buildService: CommandLineBuildService) = CommandExecutionAdapter(buildService.apply { + this.initialize(runnerContext.build, runnerContext) + }).apply { + lastCommands.add(this) + } +} diff --git a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildService.kt b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildService.kt index 7c26857..139a427 100644 --- a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildService.kt +++ b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CargoRunnerBuildService.kt @@ -14,6 +14,7 @@ import jetbrains.buildServer.agent.runner.BuildServiceAdapter import jetbrains.buildServer.agent.runner.ProcessListener import jetbrains.buildServer.agent.runner.ProgramCommandLine import jetbrains.buildServer.rust.cargo.* +import jetbrains.buildServer.rust.logging.BlockListener import jetbrains.buildServer.rust.logging.CargoLoggerFactory import jetbrains.buildServer.rust.logging.CargoLoggingListener import jetbrains.buildServer.util.StringUtil @@ -21,7 +22,8 @@ import jetbrains.buildServer.util.StringUtil /** * Cargo runner service. */ -class CargoRunnerBuildService(private val commandExecutor: CommandExecutor) : BuildServiceAdapter() { +class CargoRunnerBuildService : BuildServiceAdapter() { + private val osName = System.getProperty("os.name").toLowerCase() private val myCargoWithStdErrVersion = Version.forIntegers(0, 13) private val myArgumentsProviders = mapOf( @@ -59,10 +61,6 @@ class CargoRunnerBuildService(private val commandExecutor: CommandExecutor) : Bu val toolchainVersion = parameters[CargoConstants.PARAM_TOOLCHAIN]?.trim() ?: "" val (toolPath, arguments) = if (toolchainVersion.isNotEmpty()) { val rustupPath = getPath(CargoConstants.RUSTUP_CONFIG_NAME) - - logger.message("Using rust toolchain: $toolchainVersion") - commandExecutor.executeWithReadLock(rustupPath, arrayListOf("toolchain", "install", toolchainVersion)) - rustupPath to argumentsProvider.getArguments(runnerContext).toMutableList().apply { addAll(0, arrayListOf("run", toolchainVersion, "cargo")) } @@ -72,16 +70,14 @@ class CargoRunnerBuildService(private val commandExecutor: CommandExecutor) : Bu runnerContext.configParameters[CargoConstants.CARGO_CONFIG_NAME]?.let { if (Version.valueOf(it).greaterThanOrEqualTo(myCargoWithStdErrVersion)) { - if (osName.startsWith("windows")) { - return createProgramCommandline("cmd.exe", arrayListOf("/c", "2>&1", toolPath).apply { + return if (osName.startsWith("windows")) { + createProgramCommandline("cmd.exe", arrayListOf("/c", "2>&1", toolPath).apply { addAll(arguments) }) } else if (osName.startsWith("freebsd") || osName.startsWith("sunos")) { - return createProgramCommandline("sh", - arrayListOf("-c", "$toolPath ${arguments.joinToString(" ")} 2>&1")) + createProgramCommandline("sh", arrayListOf("-c", "$toolPath ${arguments.joinToString(" ")} 2>&1")) } else { - return createProgramCommandline("bash", - arrayListOf("-c", "$toolPath ${arguments.joinToString(" ")} 2>&1")) + createProgramCommandline("bash", arrayListOf("-c", "$toolPath ${arguments.joinToString(" ")} 2>&1")) } } } @@ -89,7 +85,7 @@ class CargoRunnerBuildService(private val commandExecutor: CommandExecutor) : Bu return createProgramCommandline(toolPath, arguments) } - fun getPath(toolName: String): String { + private fun getPath(toolName: String): String { try { return getToolPath(toolName) } catch (e: ToolCannotBeFoundException) { @@ -103,6 +99,10 @@ class CargoRunnerBuildService(private val commandExecutor: CommandExecutor) : Bu override fun getListeners(): List { val loggerFactory = CargoLoggerFactory(logger) - return listOf(CargoLoggingListener(loggerFactory)) + val blockName = "cargo ${runnerParameters[CargoConstants.PARAM_COMMAND]}" + return listOf( + CargoLoggingListener(loggerFactory), + BlockListener(blockName, logger) + ) } } diff --git a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CommandExecutionAdapter.kt b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CommandExecutionAdapter.kt new file mode 100644 index 0000000..f9db301 --- /dev/null +++ b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/CommandExecutionAdapter.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * See LICENSE in the project root for license information. + */ + +package jetbrains.buildServer.rust + +import jetbrains.buildServer.agent.BuildFinishedStatus +import jetbrains.buildServer.agent.runner.* +import java.io.File + +class CommandExecutionAdapter(private val buildService: CommandLineBuildService) : CommandExecution { + + private val processListeners by lazy { buildService.listeners } + + var result: BuildFinishedStatus? = null + private set + + override fun processFinished(exitCode: Int) { + buildService.afterProcessFinished() + + processListeners.forEach { + it.processFinished(exitCode) + } + + result = buildService.getRunResult(exitCode) + if (result == BuildFinishedStatus.FINISHED_SUCCESS) { + buildService.afterProcessSuccessfullyFinished() + } + } + + override fun processStarted(programCommandLine: String, workingDirectory: File) { + processListeners.forEach { + it.processStarted(programCommandLine, workingDirectory) + } + } + + override fun onStandardOutput(text: String) { + processListeners.forEach { + it.onStandardOutput(text) + } + } + + override fun onErrorOutput(text: String) { + processListeners.forEach { + it.onErrorOutput(text) + } + } + + override fun interruptRequested(): TerminationAction { + return buildService.interrupt() + } + + override fun makeProgramCommandLine(): ProgramCommandLine { + return buildService.makeProgramCommandLine() + } + + override fun isCommandLineLoggingEnabled() = buildService.isCommandLineLoggingEnabled + + override fun beforeProcessStarted() { + buildService.beforeProcessStarted() + } +} \ No newline at end of file diff --git a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/RustupBuildService.kt b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/RustupBuildService.kt new file mode 100644 index 0000000..bfc023e --- /dev/null +++ b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/RustupBuildService.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2000-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * See LICENSE in the project root for license information. + */ + +package jetbrains.buildServer.rust + +import jetbrains.buildServer.RunBuildException +import jetbrains.buildServer.agent.ToolCannotBeFoundException +import jetbrains.buildServer.agent.runner.BuildServiceAdapter +import jetbrains.buildServer.agent.runner.ProcessListener +import jetbrains.buildServer.agent.runner.ProcessListenerAdapter +import jetbrains.buildServer.agent.runner.ProgramCommandLine +import jetbrains.buildServer.rust.logging.BlockListener + +/** + * Rustup runner service. + */ +class RustupBuildService(private val action: String) : BuildServiceAdapter() { + + val errors = arrayListOf() + var foundVersion: String? = null + + val version: String + get() = foundVersion ?: runnerParameters[CargoConstants.PARAM_TOOLCHAIN]!! + + override fun makeProgramCommandLine(): ProgramCommandLine { + val toolchainVersion = runnerParameters[CargoConstants.PARAM_TOOLCHAIN]!!.trim() + val rustupPath = getPath(CargoConstants.RUSTUP_CONFIG_NAME) + + return createProgramCommandline(rustupPath, arrayListOf("toolchain", action, toolchainVersion)) + } + + private fun getPath(toolName: String): String { + try { + return getToolPath(toolName) + } catch (e: ToolCannotBeFoundException) { + val buildException = RunBuildException(e) + buildException.isLogStacktrace = false + throw buildException + } + } + + override fun isCommandLineLoggingEnabled() = false + + override fun getListeners(): MutableList { + return arrayListOf().apply { + val blockName = "$action toolchain: ${runnerParameters[CargoConstants.PARAM_TOOLCHAIN]}" + this.add(BlockListener(blockName, logger)) + this.add(object : ProcessListenerAdapter() { + override fun onStandardOutput(text: String) { + processOutput(text) + } + + override fun onErrorOutput(text: String) { + processOutput(text) + } + }) + } + } + + fun processOutput(text: String) { + if (text.startsWith("error:")) { + errors.add(text) + } + + toolchainVersion.matchEntire(text)?.let { + foundVersion = it.groupValues.last() + } + + logger.message(text) + } + + companion object { + val toolchainVersion = Regex("info: syncing channel updates for '([^']+)'") + } +} diff --git a/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/logging/BlockListener.kt b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/logging/BlockListener.kt new file mode 100644 index 0000000..6a023a5 --- /dev/null +++ b/plugin-rust-agent/src/main/kotlin/jetbrains/buildServer/rust/logging/BlockListener.kt @@ -0,0 +1,17 @@ +package jetbrains.buildServer.rust.logging + +import jetbrains.buildServer.agent.BuildProgressLogger +import jetbrains.buildServer.agent.runner.ProcessListenerAdapter +import java.io.File + +class BlockListener(private val blockName:String, + private val logger: BuildProgressLogger) : ProcessListenerAdapter() { + + override fun processStarted(programCommandLine: String, workingDirectory: File) { + logger.message("##teamcity[blockOpened name='$blockName']") + } + + override fun processFinished(exitCode: Int) { + logger.message("##teamcity[blockClosed name='$blockName']") + } +} \ No newline at end of file diff --git a/plugin-rust-agent/src/main/resources/META-INF/build-agent-plugin-rust.xml b/plugin-rust-agent/src/main/resources/META-INF/build-agent-plugin-rust.xml index 4938bea..95881da 100644 --- a/plugin-rust-agent/src/main/resources/META-INF/build-agent-plugin-rust.xml +++ b/plugin-rust-agent/src/main/resources/META-INF/build-agent-plugin-rust.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd" default-autowire="constructor"> - +