From 032944a9c227930baa9994bfd8a169e24622af9c Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 8 Mar 2022 12:18:37 +0900 Subject: [PATCH 1/3] Add sstate-cache server to local.conf and changed to here document Signed-off-by: Cristian Le --- prepare-build.sh | 73 +++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/prepare-build.sh b/prepare-build.sh index db1bb2f..edb33cc 100755 --- a/prepare-build.sh +++ b/prepare-build.sh @@ -15,6 +15,8 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. declare -a devices=("anthias" "bass" "catfish" "dory" "firefish" "harmony" "inharmony" "lenok" "mooneye" "narwhal" "qemux86" "ray" "smelt" "sparrow" "sprat" "sturgeon" "sawfish" "skipjack" "swift" "tetra" "wren") +declare -a architectures=("armv7vehf-neon") +declare -a sstate_server="https://sstate.asteroidos.org" function printNoDeviceInfo { echo "Usage:" @@ -110,38 +112,57 @@ else clone_dir src/meta-asteroid-community https://github.com/AsteroidOS/meta-asteroid-community master clone_dir src/meta-smartwatch https://github.com/AsteroidOS/meta-smartwatch.git master + # Find all layers under src/meta-smartwatch, remove the src/ prefix, sort alphabetically, and store it in an array. + IFS=$'\n' layers=($(find src/meta-smartwatch -mindepth 1 -name "*meta-*" -type d | sed -e 's|src/||' | sort)) + # Create local.conf and bblayers.conf on first run if [ ! -e build/conf/local.conf ]; then echo -e "\e[32mWriting build/conf/local.conf\e[39m" - echo 'DISTRO = "asteroid" -PACKAGE_CLASSES = "package_ipk"' >> build/conf/local.conf + cat > build/conf/local.conf <<-EOF + DISTRO = "asteroid" + PACKAGE_CLASSES = "package_ipk" + SSTATE_MIRRORS ?= " \\ + $( + for device in ${devices[*]}; do + echo " file://.* ${sstate_server}/${device}/sstate-cache/PATH;downloadfilename=PATH \\ " + done + ) + $( + for arch in ${architectures[*]}; do + echo " file://.* ${sstate_server}/${arch}/sstate-cache/PATH;downloadfilename=PATH \\ " + done + ) + file://.* ${sstate_server}/allarch/sstate-cache/PATH;downloadfilename=PATH \\ + file://.* ${sstate_server}/other-sstate/sstate-cache/PATH;downloadfilename=PATH \\ + " + EOF fi if [ ! -e build/conf/bblayers.conf ]; then - echo -e "\e[32mWriting build/conf/bblayers.conf\e[39m" - echo 'BBPATH = "${TOPDIR}" -SRCDIR = "${@os.path.abspath(os.path.join("${TOPDIR}", "../src/"))}" - -BBLAYERS = " \ - ${SRCDIR}/meta-qt5 \ - ${SRCDIR}/oe-core/meta \ - ${SRCDIR}/meta-asteroid \ - ${SRCDIR}/meta-asteroid-community \ - ${SRCDIR}/meta-openembedded/meta-oe \ - ${SRCDIR}/meta-openembedded/meta-multimedia \ - ${SRCDIR}/meta-openembedded/meta-gnome \ - ${SRCDIR}/meta-openembedded/meta-networking \ - ${SRCDIR}/meta-smartphone/meta-android \ - ${SRCDIR}/meta-openembedded/meta-python \ - ${SRCDIR}/meta-openembedded/meta-filesystems \' > build/conf/bblayers.conf - - # Find all layers under src/meta-smartwatch, remove the src/ prefix, sort alphabetically, and store it in an array. - IFS=$'\n' layers=($(find src/meta-smartwatch -mindepth 1 -name "*meta-*" -type d | sed -e 's|src/||' | sort)) - for layer in ${layers[*]}; do - echo " \${SRCDIR}/$layer \\" >> build/conf/bblayers.conf - done - - echo "\"" >> build/conf/bblayers.conf + echo -e "\e[32mWriting build/conf/bblayers.conf\e[39m" + cat > build/conf/bblayers.conf <<-EOF + BBPATH = "\${TOPDIR}" + SRCDIR = "\${@os.path.abspath(os.path.join("\${TOPDIR}", "../src/"))}" + + BBLAYERS = " \\ + \${SRCDIR}/meta-qt5 \\ + \${SRCDIR}/oe-core/meta \\ + \${SRCDIR}/meta-asteroid \\ + \${SRCDIR}/meta-asteroid-community \\ + \${SRCDIR}/meta-openembedded/meta-oe \\ + \${SRCDIR}/meta-openembedded/meta-multimedia \\ + \${SRCDIR}/meta-openembedded/meta-gnome \\ + \${SRCDIR}/meta-openembedded/meta-networking \\ + \${SRCDIR}/meta-smartphone/meta-android \\ + \${SRCDIR}/meta-openembedded/meta-python \\ + \${SRCDIR}/meta-openembedded/meta-filesystems \\ + $( + for layer in ${layers[*]}; do + echo " \${SRCDIR}/$layer \\" + done + ) + " + EOF fi # Init build env From 8c00f4755eac5bcb44d85c3f809d4a6adc0b580e Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 8 Mar 2022 21:30:42 +0900 Subject: [PATCH 2/3] Copied TeamCity configuration from personal test repository Signed-off-by: Cristian Le --- .teamcity/asteroid/AsteroidProject.kt | 69 +++++ .teamcity/asteroid/CoreVCS.kt | 147 +++++++++ .teamcity/asteroid/GitAPIChecker.kt | 134 +++++++++ .teamcity/asteroid/InitWorkspace.kt | 188 ++++++++++++ .teamcity/asteroid/Settings.kt | 70 +++++ .teamcity/asteroid/devices/Device.kt | 189 ++++++++++++ .teamcity/asteroid/devices/DevicesProject.kt | 279 ++++++++++++++++++ .../asteroid/packages/AsteroidAppsProject.kt | 18 ++ .../asteroid/packages/CommunityAppsProject.kt | 29 ++ .teamcity/asteroid/packages/Package.kt | 165 +++++++++++ .../asteroid/packages/PackagesProject.kt | 140 +++++++++ .../asteroid/thirdparty/ThirdPartyProject.kt | 9 + .teamcity/overrides.json | 25 ++ .teamcity/pom.xml | 151 ++++++++++ .teamcity/settings.kts | 33 +++ 15 files changed, 1646 insertions(+) create mode 100644 .teamcity/asteroid/AsteroidProject.kt create mode 100644 .teamcity/asteroid/CoreVCS.kt create mode 100644 .teamcity/asteroid/GitAPIChecker.kt create mode 100644 .teamcity/asteroid/InitWorkspace.kt create mode 100644 .teamcity/asteroid/Settings.kt create mode 100644 .teamcity/asteroid/devices/Device.kt create mode 100644 .teamcity/asteroid/devices/DevicesProject.kt create mode 100644 .teamcity/asteroid/packages/AsteroidAppsProject.kt create mode 100644 .teamcity/asteroid/packages/CommunityAppsProject.kt create mode 100644 .teamcity/asteroid/packages/Package.kt create mode 100644 .teamcity/asteroid/packages/PackagesProject.kt create mode 100644 .teamcity/asteroid/thirdparty/ThirdPartyProject.kt create mode 100644 .teamcity/overrides.json create mode 100644 .teamcity/pom.xml create mode 100644 .teamcity/settings.kts diff --git a/.teamcity/asteroid/AsteroidProject.kt b/.teamcity/asteroid/AsteroidProject.kt new file mode 100644 index 0000000..9c891ab --- /dev/null +++ b/.teamcity/asteroid/AsteroidProject.kt @@ -0,0 +1,69 @@ +package asteroid + +import asteroid.devices.DevicesProject +import asteroid.packages.PackagesProject +import asteroid.thirdparty.ThirdPartyProject +import jetbrains.buildServer.configs.kotlin.v2019_2.Project +import jetbrains.buildServer.configs.kotlin.v2019_2.projectFeatures.activeStorage + +object AsteroidProject : Project({ + description = "Base Asteroid Project" + + // Attach vcsRoots from CoreVCS + for (vcs in CoreVCS.allVcs) + vcsRoot(vcs) + + // TODO: Remove when have common base + vcsRoot(DevicesProject.vcs) + + // Attach InitWorkspace build + buildType(InitWorkspace) + + // Attach the subProjects + // TODO: Link to subprojects externally + subProjects( + DevicesProject, + PackagesProject, + ThirdPartyProject + ) + + // Define the build parameters inherited by the subprojects + params { + text( + "system.sstate.server.address", + Settings.sstateServer.url, + label = "SState server public address", + description = "Public address serving sstate-cache", + readOnly = true, + allowEmpty = false + ) + if (Settings.deploySstate) { + text( + "system.sstate.server.location", + Settings.sstateServer.location, + label = "Sstate server location", + description = "Path location of the sstate-cache", + readOnly = true, allowEmpty = true + ) + text( + "system.sstate.server.user", + Settings.sstateServer.user, + label = "SState server user", + description = "Username used to upload sstate-cache", + readOnly = true, allowEmpty = true + ) + text( + "system.sstate.server.upload_address", + Settings.sstateServer.backendUrl, + label = "SState server backend address", + description = "Backend address to upload the sstate-cache", + readOnly = true, allowEmpty = false + ) + } + } + features { + activeStorage { + activeStorageID = "DefaultStorage" + } + } +}) diff --git a/.teamcity/asteroid/CoreVCS.kt b/.teamcity/asteroid/CoreVCS.kt new file mode 100644 index 0000000..91933e2 --- /dev/null +++ b/.teamcity/asteroid/CoreVCS.kt @@ -0,0 +1,147 @@ +package asteroid + +import asteroid.devices.DevicesProject +import asteroid.packages.CommunityAppsProject +import com.github.kittinunf.fuel.Fuel +import jetbrains.buildServer.configs.kotlin.v2019_2.VcsSettings +import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot +import org.json.JSONObject + +object CoreVCS : CoreVCSDefault() { + init { + // Override default values with thise in overrides.json + val json = Settings.overrides?.optJSONObject("coreVCS") + val vcsList = allVcs.toMutableList() + vcsList.add(DevicesProject.vcs) + vcsList.add(CommunityAppsProject.vcs) + if (json != null) + for (vcs in vcsList) { + val json2 = json.optJSONObject(vcs.name) + if (json2 != null) + checkVcsOverride(vcs,json2) + } + } +} + +fun checkVcsOverride(vcs: GitVcsRoot, json: JSONObject) { + val branchRegex = "\\((.*?)\\)".toRegex() + val branch = json.optString("branch", branchRegex.find(vcs.branch!!)!!.value) + val url = json.optString("url", vcs.url) + vcs.branch = "refs/heads/($branch)" + vcs.url = url +} + +open class CoreVCSDefault { + val Asteroid: GitVcsRoot + val OpenEmbeddedCore: GitVcsRoot + val Bitbake: GitVcsRoot + val MetaOpenEmbedded: GitVcsRoot + val MetaQt5: GitVcsRoot + val MetaSmartphone: GitVcsRoot + val MetaAsteroid: GitVcsRoot + + fun attachVCS(init: VcsSettings, forDevice: Boolean = false) { + init.root(OpenEmbeddedCore, "+:.=>src/oe-core") + init.root(Bitbake, "+:.=>src/oe-core/bitbake") + init.root(MetaOpenEmbedded, "+:.=>src/meta-openembedded") + init.root(MetaQt5, "+:.=>src/meta-qt5") + init.root(MetaSmartphone, "+:.=>src/meta-smartphone") + init.root(MetaAsteroid, "+:.=>src/meta-asteroid") + if (forDevice) + init.root(DevicesProject.vcs, "+:.=>src/meta-smartwatch") + + init.cleanCheckout = true + } + + init { + Asteroid = GitVcsRoot_fallback { + id("AsteroidVCS") + name = "Asteroid" + gitBase = "https://github.com/" + url = "${Settings.fork}/asteroid.git" + fallback_url = "${Settings.upstream}/asteroid.git" + branch = "refs/heads/(master)" + } + OpenEmbeddedCore = GitVcsRoot { + id("OpenEmbeddedVCS") + name = "OpenEmbedded Core" + url = "https://github.com/openembedded/openembedded-core.git" + branch = "refs/heads/(honister)" + } + Bitbake = GitVcsRoot { + id("BitBakeVCS") + name = "Bitbake" + url = "https://github.com/openembedded/bitbake.git" + branch = "refs/heads/(1.52)" + } + MetaOpenEmbedded = GitVcsRoot { + id("MetaOpenEmbeddedVCS") + name = "Meta OpenEmbedded" + url = "https://github.com/openembedded/meta-openembedded.git" + branch = "refs/heads/(honister)" + } + MetaQt5 = GitVcsRoot { + id("MetaQt5VCS") + name = "Meta Qt5" + url = "https://github.com/meta-qt5/meta-qt5" + branch = "refs/heads/(master)" + } + MetaSmartphone = GitVcsRoot { + id("MetaSmartphoneVCS") + name = "Meta Smartphone" + url = "https://github.com/shr-distribution/meta-smartphone" + branch = "refs/heads/(honister)" + } + MetaAsteroid = GitVcsRoot_fallback { + id("MetaAsteroidVCS") + name = "Meta Asteroid" + gitBase = "https://github.com/" + url = "${Settings.fork}/meta-asteroid" + fallback_url = "${Settings.upstream}/meta-asteroid" + branch = "refs/heads/(master)" + } + } + val allVcs = listOf( + Asteroid, + OpenEmbeddedCore, + Bitbake, + MetaOpenEmbedded, + MetaQt5, + MetaSmartphone, + MetaAsteroid + ) +} + +class GitVcsRoot_fallback(init: GitVcsRoot_fallback.() -> Unit) : GitVcsRoot() { + var gitBase: String? = null + var fallback_url: String? = null + + init { + init.invoke(this) + val json = Settings.overrides?.optJSONObject("coreVCS")?.optJSONObject(name) + if (json != null){ + checkVcsOverride(this,json) + } else { + if (!gitBase.isNullOrEmpty()) { + url = gitBase + url + if (!fallback_url.isNullOrEmpty()) + fallback_url = gitBase + fallback_url + } + if (!fallback_url.isNullOrEmpty()) { + if (Settings.canHttp) { + var testURL: String = url ?: "" + var code = Fuel.get(testURL).response().second.statusCode + if (code == 404) { + testURL = fallback_url ?: "" + code = Fuel.get(testURL).response().second.statusCode + } + if (code != 200) { + // TODO: Resolve other excetions + } + url = testURL + } else + url = fallback_url + } + } + } +} \ No newline at end of file diff --git a/.teamcity/asteroid/GitAPIChecker.kt b/.teamcity/asteroid/GitAPIChecker.kt new file mode 100644 index 0000000..7fc6c24 --- /dev/null +++ b/.teamcity/asteroid/GitAPIChecker.kt @@ -0,0 +1,134 @@ +package asteroid + +import com.github.kittinunf.fuel.Fuel +import com.github.kittinunf.fuel.core.Headers +import com.github.kittinunf.fuel.core.extensions.jsonBody +import com.github.kittinunf.fuel.json.responseJson + +enum class GitRepoHubType { + Github +} + +interface GitAPIChecker { + val repo: String + val token: String + val hubType: GitRepoHubType + val commitUser: String + fun checkCommitStatus(): Boolean + fun checkPR(): Boolean + fun checkToken(): Boolean + + companion object { + fun Create(repo: String, token: String): GitAPIChecker? { + with(repo) { + when { + contains("https://github.com") -> return GithubAPIChecker(repo, token) + else -> return null + } + } + } + } +} + +class GithubAPIChecker(override val repo: String, override val token: String) : GitAPIChecker { + override val hubType = GitRepoHubType.Github + override val commitUser = Settings.commitUser + val repoAPIBase = repo.removeSuffix(".git") + .replace("https://github.com", "https://api.github.com/repos") + + override fun checkToken(): Boolean { + // Cannot check if sandboxing is enabled or the token is not plain-text + if (!Settings.canHttp || token.startsWith("credentialsJSON:")) + return false + val request = Fuel.get("https://api.github.com") + .appendHeader(Headers.AUTHORIZATION, "token $token") + return when (request.response().second.statusCode) { + 401 -> false + 200 -> true + else -> { + // TODO: Add warning + false + } + } + } + + override fun checkCommitStatus(): Boolean { + // Cannot check if sandboxing is enabled or the token is not plain-text + if (!Settings.canHttp || token.startsWith("credentialsJSON:")) + return true + // If token is invalid throw warning + if (!checkToken()) { + // TODO: add warning + return false + } + + var request = Fuel.get("$repoAPIBase/commits/HEAD/status") + .appendHeader(Headers.AUTHORIZATION, "token $token") + var response = request.responseJson() + when (response.second.statusCode) { + // Token cannot access private repo + 404 -> return false + // Token does not have repo:status:read access to the repo + 403 -> return false + // Token has at least repo:status:read access to the repo + 200 -> {} + else -> { + // Unknown states + // TODO: Add warning + return false + } + } + val sha = response.third.component1()!!.obj()["sha"].toString() + request = Fuel.post("$repoAPIBase/commits/$sha/statuses") + .appendHeader(Headers.AUTHORIZATION, "token $token") + .jsonBody( + """ + { + "context": "test-connection", + "state": "dummy" + } + """.trimIndent() + ) + response = request.responseJson() + return when (response.second.statusCode) { + // Token does not have repo:status:write access to the repo + 403 -> false + // Token has repo:status:write access but we made ill-formed content + 422 -> true + // Created status. This should not have occured + 201 -> { + // TODO: Add warning + true + } + // Unknown + else -> false + } + } + + override fun checkPR(): Boolean { + // Cannot check if sandboxing is enabled or the token is not plain-text + if (!Settings.canHttp || token.startsWith("credentialsJSON:")) + return true + // If token is invalid throw warning + if (!checkToken()) { + // TODO: add warning + return false + } + + val request = Fuel.get("$repoAPIBase/pulls") + .appendHeader(Headers.AUTHORIZATION, "token $token") + return when (request.response().second.statusCode) { + // Token cannot access private repo + 404 -> false + // Token does not have repo:status:read access to the repo + 403 -> false + // Token has at least repo:status:read access to the repo + 200 -> true + else -> { + // Unknown states + // TODO: Add warning + false + } + } + } +} \ No newline at end of file diff --git a/.teamcity/asteroid/InitWorkspace.kt b/.teamcity/asteroid/InitWorkspace.kt new file mode 100644 index 0000000..89c1701 --- /dev/null +++ b/.teamcity/asteroid/InitWorkspace.kt @@ -0,0 +1,188 @@ +package asteroid + +import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType +import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ScriptBuildStep + +object InitWorkspace : BuildType({ + name = "Initialize workspace" + description = "Clone sources and initialize workspace" + + enablePersonalBuilds = false + + vcs { + CoreVCS.attachVCS(this) + } +}) + +fun initScript(buildStep: ScriptBuildStep, withSstate: Boolean = true) { + // TODO: Change the hardcoded sturgeon to generic + buildStep.name = "Prepare config files" + val sstateMirror: String = if (withSstate) + """ + SSTATE_MIRRORS ?= " \\ + file://.* %system.sstate.server.address%/sturgeon/sstate-cache/PATH;downloadfilename=PATH \\n \\ + file://.* %system.sstate.server.address%/armv7vehf-neon/sstate-cache/PATH;downloadfilename=PATH \\n \\ + file://.* %system.sstate.server.address%/allarch/sstate-cache/PATH;downloadfilename=PATH \\n \\ + file://.* %system.sstate.server.address%/other-sstate/sstate-cache/PATH;downloadfilename=PATH \\n \\ + " + """.trimStart().trimEnd() + else + """ + SSTATE_MIRRORS ?= " \\ + file://.* %system.sstate.server.address%/other-sstate/sstate-cache/PATH;downloadfilename=PATH \\n \\ + " + """.trimStart().trimEnd() + buildStep.scriptContent = """ + mkdir -p build/conf + cat > build/conf/local.conf <<-EOF + DISTRO = "asteroid" + MACHINE = "sturgeon" + PACKAGE_CLASSES = "package_ipk" + $sstateMirror + EOF + cat > build/conf/bblayers.conf <<-EOF + BBPATH = "\${'$'}{TOPDIR}" + SRCDIR = "\${'$'}{@os.path.abspath(os.path.join("\${'$'}{TOPDIR}", "../src/"))}" + + BBLAYERS = " \\ + \${'$'}{SRCDIR}/meta-qt5 \\ + \${'$'}{SRCDIR}/oe-core/meta \\ + \${'$'}{SRCDIR}/meta-asteroid \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-oe \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-multimedia \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-gnome \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-networking \ + \${'$'}{SRCDIR}/meta-smartphone/meta-android \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-python \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-filesystems \\ + \${'$'}{SRCDIR}/meta-smartwatch/meta-sturgeon \\ + " + EOF + + # Try to initialize OE environment + source ./src/oe-core/oe-init-build-env + """.trimIndent() +} + +fun initScript( + buildStep: ScriptBuildStep, + device: String, + architecture: String = "armv7vehf-neon", + meta: String = device, + withSstate: Boolean = true +) { + buildStep.name = "Prepare config files" + val sstateMirror: String = if (withSstate) + """ + SSTATE_MIRRORS ?= " \\ + file://.* %system.sstate.server.address%/${device}/sstate-cache/PATH;downloadfilename=PATH \\n \\ + file://.* %system.sstate.server.address%/${architecture}/sstate-cache/PATH;downloadfilename=PATH \\n \\ + file://.* %system.sstate.server.address%/allarch/sstate-cache/PATH;downloadfilename=PATH \\n \\ + file://.* %system.sstate.server.address%/other-sstate/sstate-cache/PATH;downloadfilename=PATH \\n \\ + " + """.trimStart().trimEnd() + else + """ + SSTATE_MIRRORS ?= " \\ + file://.* %system.sstate.server.address%/other-sstate/sstate-cache/PATH;downloadfilename=PATH \\n \\ + " + """.trimStart().trimEnd() + buildStep.scriptContent = """ + mkdir -p build/conf + cat > build/conf/local.conf <<-EOF + DISTRO = "asteroid" + MACHINE = "$device" + PACKAGE_CLASSES = "package_ipk" + $sstateMirror + EOF + cat > build/conf/bblayers.conf <<-EOF + BBPATH = "\${'$'}{TOPDIR}" + SRCDIR = "\${'$'}{@os.path.abspath(os.path.join("\${'$'}{TOPDIR}", "../src/"))}" + + BBLAYERS = " \\ + \${'$'}{SRCDIR}/meta-qt5 \\ + \${'$'}{SRCDIR}/oe-core/meta \\ + \${'$'}{SRCDIR}/meta-asteroid \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-oe \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-multimedia \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-gnome \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-networking \\ + \${'$'}{SRCDIR}/meta-smartphone/meta-android \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-python \\ + \${'$'}{SRCDIR}/meta-openembedded/meta-filesystems \\ + \${'$'}{SRCDIR}/meta-smartwatch/meta-$meta \\ + " + EOF + + # Try to initialize OE environment + source ./src/oe-core/oe-init-build-env + """.trimIndent() +} + +fun bitbakeBuild(buildStep: ScriptBuildStep, recipe: String? = null) { + buildStep.scriptContent = """ + source ./src/oe-core/oe-init-build-env > /dev/null + echo "Starting bitbake" + bitbake --ui=teamcity ${recipe ?: "asteroid-image%system.image.dev-suffix%"} + """.trimIndent() +} + +fun updateSstate(buildStep: ScriptBuildStep, cleanServer: Boolean = false) { + // TODO: Change the hardcoded sturgeon to generic + buildStep.name = "Upload sstate-cache" + val cleanCommand = if (cleanServer) + """ + # Clean destination + mkdir dummy_empty + rsync --delete \ + ./dummy_empty ${'$'}{ServerAddr} + """.trimStart().trimEnd() + else + "" + buildStep.scriptContent = """ + Opts="-a --prune-empty-dirs --remove-source-files \ + --checksum --progress" + ServerAddr="%system.sstate.server.user%@%system.sstate.server.upload_address%:%system.sstate.server.location%" + + $cleanCommand + rsync ${'$'}{Opts} \ + build/sstate-cache/fedora-35 ${'$'}{ServerAddr}/other-sstate/sstate-cache + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*::*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/other-sstate + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*:sturgeon:*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/sturgeon + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*:armv7vehf-neon:*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/armv7vehf-neon + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*:allarch:*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/all-arch + """.trimIndent() +} + +fun updateSstate(buildStep: ScriptBuildStep, device: String, architecture: String, cleanServer: Boolean = false) { + buildStep.name = "Upload sstate-cache" + buildStep.scriptContent = """ + Opts="-a --prune-empty-dirs --remove-source-files \ + --checksum --progress" + ServerAddr="%system.sstate.server.user%@%system.sstate.server.upload_address%:%system.sstate.server.location%" + + rsync ${'$'}{Opts} \ + build/sstate-cache/fedora-35 ${'$'}{ServerAddr}/other-sstate/sstate-cache + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*::*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/other-sstate + rsync ${'$'}{Opts} \ + ${if (cleanServer) "--delete" else ""} \ + --include '*/' --include '*:*:*:*:*:${device}:*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/${device} + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*:${architecture}:*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/${architecture} + rsync ${'$'}{Opts} \ + --include '*/' --include '*:*:*:*:*:allarch:*' --exclude '*' \ + build/sstate-cache ${'$'}{ServerAddr}/all-arch + """.trimIndent() +} \ No newline at end of file diff --git a/.teamcity/asteroid/Settings.kt b/.teamcity/asteroid/Settings.kt new file mode 100644 index 0000000..4d5a3df --- /dev/null +++ b/.teamcity/asteroid/Settings.kt @@ -0,0 +1,70 @@ +package asteroid + +import com.github.kittinunf.fuel.Fuel +import jetbrains.buildServer.configs.kotlin.v2019_2.DslContext +import org.json.JSONException +import org.json.JSONObject +import java.io.File + +object Settings { + val asteroidPackages = DslContext.getParameter("Packages") + .split("[\\s,]".toRegex()).toList() + .filterNot { it.isEmpty() } + val communityPackages = DslContext.getParameter("CommunityPackages","") + .split("[\\s,]".toRegex()).toList() + .filterNot { it.isEmpty() } + val devices = DslContext.getParameter("Devices") + .split("[\\s,]".toRegex()).toList() + .filterNot { it.isEmpty() } + val cleanBuilds = DslContext.getParameter("CleanBuilds", "false").toBoolean() + val withSstate = DslContext.getParameter("WithSstate", "true").toBoolean() + + val canHttp = fun(): Boolean { + val request = Fuel.get("https://api.github.com") + val response = request.response() + val code = response.second.statusCode + // This should be because of sandboxing + if (code < 0) + return false + // TODO: Add warning + return true + }.invoke() + val fork = DslContext.getParameter("Fork") + val upstream = DslContext.getParameter("Upstream", "AsteroidOS") + val deploySstate = DslContext.getParameter("DeploySstate", "false").toBoolean() + + object sstateServer { + val url = DslContext.getParameter("SstateServerURL", "https://sstate.asteroid.org") + val backendUrl = if (deploySstate) + DslContext.getParameter("SstateServerBackendURL", "sstate.asteroid.org") + else "" + val user = if (deploySstate) + DslContext.getParameter("SstateServerUser", "asteroidos") + else "" + val location = if (deploySstate) + DslContext.getParameter("SstateServerLocation", "") + else "" + } + + // TODO: Change to Github app when available + val GithubTokenID = DslContext.getParameter("GithubToken", "credentialsJSON:0b803d82-f0a8-42ee-b8f9-0fca109a14ab") + val commitStatus = DslContext.getParameter("CommitStatus", "false").toBoolean() + val commitUser = if (commitStatus) + DslContext.getParameter("CommitUser", fork) + else "" + val pullRequests = DslContext.getParameter("PullRequests", "false").toBoolean() + + var overrides: JSONObject? = null + + init { + val file = File(DslContext.baseDir, "overrides.json") + if (file.exists()) { + val text = file.readText() + try { + overrides = JSONObject(text) + } catch (err: JSONException) { + // TODO: Add warning not a JSON format + } + } + } +} \ No newline at end of file diff --git a/.teamcity/asteroid/devices/Device.kt b/.teamcity/asteroid/devices/Device.kt new file mode 100644 index 0000000..86b807c --- /dev/null +++ b/.teamcity/asteroid/devices/Device.kt @@ -0,0 +1,189 @@ +package asteroid.devices + +import asteroid.* +import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType +import jetbrains.buildServer.configs.kotlin.v2019_2.Project +import jetbrains.buildServer.configs.kotlin.v2019_2.PublishMode +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.commitStatusPublisher +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.sshAgent +import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script +import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs + +class DeviceProject(val device: String) : Project({ + id("Devices_${device}") + name = device +}) { + val buildImage: BuildType + val buildImageFromScratch: BuildType + val architecture: String + val meta: String + + init { + val json = Settings.overrides?.optJSONObject("devices")?.optJSONObject(device) + meta = json?.optString("meta") ?: device + architecture = json?.optString("architecture") ?: "armv7vehf-neon" + buildImage = BuildImage(device, architecture, meta) + buildImageFromScratch = BuildImageFromScratch(device, architecture, meta) + buildType(buildImage) + if (Settings.cleanBuilds) + buildType(buildImageFromScratch) + } +} + +/** + * Template for device image builder + */ +open class BuildImage(device: String, architecture: String, meta: String = device) : BuildType({ + id("Devices_${device}_BuildImage") + name = "Build Image" + description = "Build Asteroid image for $device with latest sstate-cache" + + artifactRules = """ + +:build/tmp-glibc/deploy/images/${device}/asteroid-image-${device}*.ext4 + +:build/tmp-glibc/deploy/images/${device}/zImage-dtb-${device}*.fastboot + """.trimIndent() + publishArtifacts = PublishMode.SUCCESSFUL + + vcs { + CoreVCS.attachVCS(this, true) + } + params { + param("system.image.dev-suffix","") + } + + steps { + script { + initScript(this, device, architecture, meta) + } + script { + name = "Build Image" + bitbakeBuild(this) + } + if (Settings.deploySstate) { + script { + updateSstate(this, device, architecture) + } + } + } + + triggers { + vcs { + triggerRules = """ + +:root=${DevicesProject.vcs.id}:/meta-$meta/** + """.trimIndent() + + branchFilter = "+:" + } + vcs { + triggerRules = """ + +:root=${DevicesProject.vcs.id};comment=^(?!\[NoBuild\]:).+:/meta-$meta/** + +:root=${DevicesProject.vcs.id};comment=^\[(?:[^\]\n]*)($device)(?:[^\]\n]*)\][:]:/** + """.trimIndent() + + branchFilter = "+:pull/*" + buildParams.param("system.image.dev-suffix","-dev") + } + } + + features { + if (Settings.deploySstate) { + sshAgent { + teamcitySshKey = "Sstate Server Key" + } + } + var gitChecker: GitAPIChecker? + if (Settings.pullRequests) { + gitChecker = GitAPIChecker.Create(DevicesProject.vcs.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${DevicesProject.vcs.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + } + if (Settings.commitStatus) { + gitChecker = GitAPIChecker.Create(DevicesProject.vcs.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + vcsRootExtId = "${DevicesProject.vcs.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + } + } +}) + +open class BuildImageFromScratch(device: String, architecture: String, meta: String = device) : BuildType({ + id("Devices_${device}_BuildImageFromScratch") + name = "Build Image (from scratch)" + description = "Build Asteroid image for $device from scratch and update the sstate-cache" + + artifactRules = "sstate-cache-${device}.tar.gz" + maxRunningBuilds = 1 + publishArtifacts = PublishMode.SUCCESSFUL + + vcs { + CoreVCS.attachVCS(this, true) + } + params { + param("system.image.dev-suffix","-dev") + } + + steps { + script { + initScript(this, device, architecture, meta, false) + } + script { + bitbakeBuild(this) + } + if (Settings.deploySstate) { + script { + updateSstate(this, device, architecture, true) + } + } + script { + name = "Compress sstate-cache" + scriptContent = """ + tar -cf sstate-cache-${device}.tar.gz build/sstate-cache + """.trimIndent() + } + } + + triggers { + vcs { + triggerRules = """ + +:root=${DevicesProject.vcs.id};comment=^\[Rebuild:(?:[^\]\n]*)(${device})(?:[^\]\n]*)\][:]:** + +:root=${CoreVCS.MetaAsteroid.id};comment=^\[Rebuild:(?:[^\]\n]*)(${device})(?:[^\]\n]*)\][:]:** + """.trimIndent() + + branchFilter = "+:" + } + } + + features { + if (Settings.deploySstate) { + sshAgent { + teamcitySshKey = "Sstate Server Key" + } + } + } +}) \ No newline at end of file diff --git a/.teamcity/asteroid/devices/DevicesProject.kt b/.teamcity/asteroid/devices/DevicesProject.kt new file mode 100644 index 0000000..7064700 --- /dev/null +++ b/.teamcity/asteroid/devices/DevicesProject.kt @@ -0,0 +1,279 @@ +package asteroid.devices + +import asteroid.* +import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType +import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction +import jetbrains.buildServer.configs.kotlin.v2019_2.Project +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.commitStatusPublisher +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.sshAgent +import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script +import jetbrains.buildServer.configs.kotlin.v2019_2.sequential +import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.schedule +import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs +import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot + +object DevicesProject : Project({ + id("Devices") + name = "Devices" + description = "Device projects" + params { + text( + "system.image.dev-suffix", + "", + "Development suffix", + "Recipe suffix to build development or debug version", + readOnly = false, allowEmpty = true + ) + } +}) { + val vcs: GitVcsRoot = GitVcsRoot_fallback { + id("MetaSmartwatchVCS") + name = "Meta Smatwtach" + gitBase = "https://github.com/" + url = "${Settings.fork}/meta-smartwatch.git" + fallback_url = "${Settings.upstream}/meta-smartwatch.git" + branch = "refs/heads/(master)" + } + val devices: List = Settings.devices.map { DeviceProject(it) } + + init { + // TODO: Uncomment when have generic device +// vcsRoot(vcs) + // Create the subProjects + for (device in devices) + subProject(device) + + if (Settings.withSstate) { + buildType(BuildBase) + buildType(BuildAll) + } + if (Settings.cleanBuilds) { + buildType(BuildBaseFromScratch) + buildType(BuildAllFromScratch) + } + + if (Settings.withSstate) { + sequential { + buildType(BuildBase) + parallel { + for (device in this@DevicesProject.devices) + buildType(device.buildImage) { + onDependencyFailure = FailureAction.CANCEL + } + } + buildType(BuildAll) { + onDependencyFailure = FailureAction.CANCEL + } + } + } + if (Settings.cleanBuilds) { + sequential { + buildType(BuildBaseFromScratch) + parallel { + for (device in this@DevicesProject.devices) + buildType(device.buildImageFromScratch) { + onDependencyFailure = FailureAction.CANCEL + } + } + buildType(BuildAllFromScratch) { + onDependencyFailure = FailureAction.CANCEL + } + } + } + } +} + +object BuildBase : BuildType({ + // TODO: Change to generic device + id("Devices_BuildBase") + name = "Build device base" + description = "Build a prototype device with sstate-server" + + vcs { + CoreVCS.attachVCS(this, true) + } + + steps { + script { + initScript(this) + } + script { + name = "Build Image" + bitbakeBuild(this) + } + if (Settings.deploySstate) { + script { + updateSstate(this, "sturgeon", "armv7vehf-neon") + } + } + } + + features { + if (Settings.deploySstate) { + sshAgent { + teamcitySshKey = "Sstate Server Key" + } + } + } +}) + +object BuildBaseFromScratch : BuildType({ + // TODO: Change to generic device + id("Devices_BuildBaseFromScratch") + name = "Build device base (from scratch)" + description = "Build a prototype device with clean environment" + + vcs { + CoreVCS.attachVCS(this, true) + } + + steps { + script { + initScript(this, false) + } + script { + name = "Build Image" + bitbakeBuild(this) + } + if (Settings.deploySstate) { + script { + updateSstate(this, true) + } + } + } + + features { + if (Settings.deploySstate) { + sshAgent { + teamcitySshKey = "Sstate Server Key" + } + } + } +}) + +object BuildAll : BuildType({ + id("Devices_BuildAll") + name = "Build all devices" + description = "Build Asteroid image for all devices with latest sstate-cache" + + vcs { + root(CoreVCS.Asteroid) + root(CoreVCS.MetaAsteroid) + } + + triggers { + vcs { + // TODO: Add quiet period + watchChangesInDependencies = true + triggerRules = """ + +:/** + -:root=${DevicesProject.vcs.id}:/** + +:root=${CoreVCS.MetaAsteroid.id};comment=^(?!\[NoBuild\]:).+:/** + -:root=${CoreVCS.MetaAsteroid.id}:/recipes-asteroid-apps/* + """.trimIndent() + + branchFilter = """ + +: + +:pull/* + """.trimIndent() + } + vcs { + triggerRules = """ + +:root=${CoreVCS.Asteroid.id}:/.teamcity/* + -:root=${CoreVCS.Asteroid.id}:/.teamcity/*/** + +:root=${CoreVCS.Asteroid.id}:/.teamcity/devices/** + """.trimIndent() + + branchFilter = """ + +: + """.trimIndent() + } + } + features { + var gitChecker: GitAPIChecker? + if (Settings.pullRequests) { + gitChecker = GitAPIChecker.Create(CoreVCS.MetaAsteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${CoreVCS.MetaAsteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + gitChecker = GitAPIChecker.Create(CoreVCS.Asteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${CoreVCS.Asteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + } + if (Settings.commitStatus) { + gitChecker = GitAPIChecker.Create(CoreVCS.MetaAsteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + vcsRootExtId = "${CoreVCS.MetaAsteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + gitChecker = GitAPIChecker.Create(CoreVCS.Asteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + vcsRootExtId = "${CoreVCS.Asteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + } + } +}) + +object BuildAllFromScratch : BuildType({ + id("Devices_BuildAllFromScratch") + name = "Build all devices (From scratch)" + description = "Build Asteroid image for all devices" + + triggers { + schedule { + schedulingPolicy = weekly { + } + branchFilter = "+:" + triggerBuild = always() + withPendingChangesOnly = false + } + } +}) \ No newline at end of file diff --git a/.teamcity/asteroid/packages/AsteroidAppsProject.kt b/.teamcity/asteroid/packages/AsteroidAppsProject.kt new file mode 100644 index 0000000..d2e26ad --- /dev/null +++ b/.teamcity/asteroid/packages/AsteroidAppsProject.kt @@ -0,0 +1,18 @@ +package asteroid.packages + +import asteroid.Settings +import jetbrains.buildServer.configs.kotlin.v2019_2.Project + +object AsteroidAppsProject : Project({ + id("Packages_AsteroidApps") + name = "Asteroid Apps" + description = "Core AsteroidOS packages" +}) { + // Create the subProjects + val packages = Settings.asteroidPackages.map { PackageProject(it) } + + init { + for (pkg in packages) + subProject(pkg) + } +} diff --git a/.teamcity/asteroid/packages/CommunityAppsProject.kt b/.teamcity/asteroid/packages/CommunityAppsProject.kt new file mode 100644 index 0000000..c270a91 --- /dev/null +++ b/.teamcity/asteroid/packages/CommunityAppsProject.kt @@ -0,0 +1,29 @@ +package asteroid.packages + +import asteroid.GitVcsRoot_fallback +import asteroid.Settings +import jetbrains.buildServer.configs.kotlin.v2019_2.Project +import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot + +object CommunityAppsProject : Project({ + id("Packages_CommuityApps") + name = "Asteroid Comunity Apps" + description = "Additional community app packages" +}) { + // Create the subProjects + val packages = Settings.communityPackages.map { PackageProject(it, false) } + val vcs: GitVcsRoot = GitVcsRoot_fallback { + id("MetaCommunityVCS") + name = "Meta Community" + gitBase = "https://github.com/" + url = "${Settings.fork}/asteroid.git" + fallback_url = "${Settings.upstream}/asteroid.git" + branch = "refs/heads/(master)" + } + + init { + for (pkg in packages) + subProject(pkg) + vcsRoot(vcs) + } +} diff --git a/.teamcity/asteroid/packages/Package.kt b/.teamcity/asteroid/packages/Package.kt new file mode 100644 index 0000000..d96724e --- /dev/null +++ b/.teamcity/asteroid/packages/Package.kt @@ -0,0 +1,165 @@ +package asteroid.packages + +import asteroid.* +import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType +import jetbrains.buildServer.configs.kotlin.v2019_2.Project +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.commitStatusPublisher +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.sshAgent +import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script +import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs +import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot + +class PackageProject(val pkg: String, coreApp: Boolean = true) : Project({ + id("Packages_AsteroidApps_${pkg.filter { it.isLetterOrDigit() }}") + name = pkg +}) { + val recipeVCS: GitVcsRoot + val buildPackage: BuildPackage + val recipe: String + + init { + val json = Settings.overrides?.optJSONObject("packages")?.optJSONObject(pkg) + val gitName = json?.optString("gitName") ?: pkg + val branch = json?.optString("branch") ?: "master" + recipe = json?.optString("recipe") ?: pkg + recipeVCS = GitVcsRoot_fallback { + id("Packages_AsteroidApps_${this@PackageProject.pkg.filter { it.isLetterOrDigit() }}VCS") + name = "${this@PackageProject.pkg} Source" + gitBase = "https://github.com/" + url = "${Settings.fork}/$gitName.git" + fallback_url = "${Settings.upstream}/$gitName.git" + this.branch = "refs/heads/($branch)" + } + buildPackage = BuildPackage(pkg, recipeVCS, recipe, coreApp) + buildPackage.vcs.root(recipeVCS, "+:.=>src/$gitName") + vcsRoot(recipeVCS) + buildType(buildPackage) + } +} + +open class BuildPackage(pkg: String, recipeVCS: GitVcsRoot, recipe: String = pkg, coreApp: Boolean = true) : BuildType({ + id("Packages_AsteroidApps_${recipe.filter { it.isLetterOrDigit() }}_BuildPackage") + name = "Build Package" + description = "Build a specific recipe" + + vcs { + CoreVCS.attachVCS(this, true) + } + + steps { + script { + initScript(this) + } + script { + name = "Build Package" + bitbakeBuild(this, recipe) + } + if (Settings.deploySstate) { + script { + updateSstate(this) + } + } + } + + triggers { + vcs { + val coreAppTrigger = if (coreApp) """ + +:root=${CoreVCS.MetaAsteroid.id};comment=^(?!\[NoBuild\]:).+:/recipes-asteroid/$pkg/** + +:root=${CoreVCS.MetaAsteroid.id};comment=^\[$pkg\][:]:** + """.trimStart().trimEnd() else "" + triggerRules = """ + +:root=${CoreVCS.MetaAsteroid.id};comment=^(?!\[NoBuild\]:).+:** + -:root=${CoreVCS.MetaAsteroid.id}:/recipes-asteroid/** + $coreAppTrigger + +:root=${recipeVCS.id};comment=^(?!\[NoBuild\]:).+:** + """.trimIndent() + + branchFilter = """ + +: + +:pull/* + """.trimIndent() + } + } + + features { + if (Settings.deploySstate) { + sshAgent { + teamcitySshKey = "Sstate Server Key" + } + } + var gitChecker: GitAPIChecker? + if (Settings.pullRequests) { + gitChecker = GitAPIChecker.Create(recipeVCS.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${recipeVCS.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + gitChecker = GitAPIChecker.Create(CoreVCS.MetaAsteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${CoreVCS.MetaAsteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + } + if (Settings.commitStatus) { + gitChecker = GitAPIChecker.Create(recipeVCS.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + vcsRootExtId = "${recipeVCS.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + if (coreApp) { + gitChecker = GitAPIChecker.Create(CoreVCS.MetaAsteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + // Disabled because it + // TODO: Re-enable when TW-75724 is resolved (https://youtrack.jetbrains.com/issue/TW-75724) + enabled = false + vcsRootExtId = "${CoreVCS.MetaAsteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + } + } + } +}) \ No newline at end of file diff --git a/.teamcity/asteroid/packages/PackagesProject.kt b/.teamcity/asteroid/packages/PackagesProject.kt new file mode 100644 index 0000000..cce5f53 --- /dev/null +++ b/.teamcity/asteroid/packages/PackagesProject.kt @@ -0,0 +1,140 @@ +package asteroid.packages + +import asteroid.* +import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType +import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction +import jetbrains.buildServer.configs.kotlin.v2019_2.Project +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.commitStatusPublisher +import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests +import jetbrains.buildServer.configs.kotlin.v2019_2.sequential +import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs + +object PackagesProject : Project({ + id("Packages") + name = "Packages" + description = "Package projects" + + subProject(AsteroidAppsProject) + subProject(CommunityAppsProject) + + buildType(BuildAll) + + sequential { + parallel { + for (pkg in AsteroidAppsProject.packages) + buildType(pkg.buildPackage) + } + buildType(BuildAll) { + onDependencyFailure = FailureAction.CANCEL + } + } +}) + +object BuildAll : BuildType({ + id("Packages_BuildAll") + name = "Build all Packages" + description = "Build all packages" + + vcs { + root(CoreVCS.Asteroid) + root(CoreVCS.MetaAsteroid) + } + + triggers { + vcs { + // TODO: Add quiet period + watchChangesInDependencies = true + triggerRules = """ + +:/** + +:root=${CoreVCS.MetaAsteroid.id};comment=^(?!\[NoBuild\]:).+:/** + -:root=${CoreVCS.MetaAsteroid.id}:/recipes-asteroid-apps/* + """.trimIndent() + + branchFilter = """ + +: + +:pull/* + """.trimIndent() + } + vcs { + triggerRules = """ + +:root=${CoreVCS.Asteroid.id}:/.teamcity/* + -:root=${CoreVCS.Asteroid.id}:/.teamcity/*/** + +:root=${CoreVCS.Asteroid.id}:/.teamcity/packages/** + """.trimIndent() + + branchFilter = """ + +: + """.trimIndent() + } + } + features { + var gitChecker: GitAPIChecker? + if (Settings.pullRequests) { + gitChecker = GitAPIChecker.Create(CoreVCS.MetaAsteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${CoreVCS.MetaAsteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + gitChecker = GitAPIChecker.Create(CoreVCS.Asteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkPR() == true) + pullRequests { + vcsRootExtId = "${CoreVCS.Asteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + provider = github { + authType = token { + token = Settings.GithubTokenID + } + filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER_OR_COLLABORATOR + } + } + } + } + } + if (Settings.commitStatus) { + gitChecker = GitAPIChecker.Create(CoreVCS.MetaAsteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + vcsRootExtId = "${CoreVCS.MetaAsteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + gitChecker = GitAPIChecker.Create(CoreVCS.Asteroid.url!!, Settings.GithubTokenID) + if (gitChecker?.checkCommitStatus() == true) + commitStatusPublisher { + vcsRootExtId = "${CoreVCS.Asteroid.id}" + when (gitChecker!!.hubType) { + GitRepoHubType.Github -> { + publisher = github { + githubUrl = "https://api.github.com" + authType = personalToken { + token = Settings.GithubTokenID + } + } + param("github_oauth_user", gitChecker!!.commitUser) + } + } + } + } + } +}) \ No newline at end of file diff --git a/.teamcity/asteroid/thirdparty/ThirdPartyProject.kt b/.teamcity/asteroid/thirdparty/ThirdPartyProject.kt new file mode 100644 index 0000000..c52000d --- /dev/null +++ b/.teamcity/asteroid/thirdparty/ThirdPartyProject.kt @@ -0,0 +1,9 @@ +package asteroid.thirdparty + +import jetbrains.buildServer.configs.kotlin.v2019_2.Project + +object ThirdPartyProject : Project({ + id("ThirdParty") + name = "ThirdParty Dependencies" + description = "Third party dependencies" +}) diff --git a/.teamcity/overrides.json b/.teamcity/overrides.json new file mode 100644 index 0000000..d2b1ead --- /dev/null +++ b/.teamcity/overrides.json @@ -0,0 +1,25 @@ +{ + "devices": { + "harmony": { + "meta": "mtk6580" + }, + "inharmony": { + "meta": "mtk6580" + }, + "firefish": { + "meta": "ray" + } + }, + "packages": { + "asteroid-icons": { + "gitName": "asteroid-icons-ion", + "recipe": "asteroid-icons-ion" + } + }, + "coreVCS": { + "Asteroid": { + "url": "https://github.com/LecrisUT/AsteroidOS", + "branch": "add-CI" + } + } +} \ No newline at end of file diff --git a/.teamcity/pom.xml b/.teamcity/pom.xml new file mode 100644 index 0000000..49d5c62 --- /dev/null +++ b/.teamcity/pom.xml @@ -0,0 +1,151 @@ + + + 4.0.0 + Asteroid Config DSL Script + Asteroid + Asteroid_dsl + 1.0-SNAPSHOT + + + org.jetbrains.teamcity + configs-dsl-kotlin-parent + 1.0-SNAPSHOT + + + + 2.0.0-beta-1 + 2.3.1 + + + + + jetbrains-all + https://download.jetbrains.com/teamcity-repository + + true + + + + teamcity-server + https://teamcity.lecris.dev/app/dsl-plugins-repository + + true + + + + + + + JetBrains + https://download.jetbrains.com/teamcity-repository + + + + + ${basedir} + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + 1.8 + + + + compile + compile + + compile + + + + test-compile + test-compile + + test-compile + + + + + + org.jetbrains.teamcity + teamcity-configs-maven-plugin + ${teamcity.dsl.version} + + kotlin + target/generated-configs + + AsteroidOS + AsteroidOS + catfish,harmony + asteroid-alarmclock,asteroid-icons + true + true + true + true + + + + + + + + + org.jetbrains.teamcity + configs-dsl-kotlin + ${teamcity.dsl.version} + compile + + + org.jetbrains.teamcity + configs-dsl-kotlin-plugins + 1.0-SNAPSHOT + pom + compile + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin.version} + compile + + + org.jetbrains.kotlin + kotlin-script-runtime + ${kotlin.version} + compile + + + + com.github.kittinunf.fuel + fuel + ${fuel.version} + + + com.github.kittinunf.fuel + fuel-json + ${fuel.version} + + + org.json + json + 20200518 + + + \ No newline at end of file diff --git a/.teamcity/settings.kts b/.teamcity/settings.kts new file mode 100644 index 0000000..73c3149 --- /dev/null +++ b/.teamcity/settings.kts @@ -0,0 +1,33 @@ +import jetbrains.buildServer.configs.kotlin.v2019_2.* + +/* +The settings script is an entry point for defining a single +TeamCity project. TeamCity looks for the 'settings.kts' file in a +project directory and runs it if it's found, so the script name +shouldn't be changed and its package should be the same as the +project's id. + +The script should contain a single call to the project() function +with a Project instance or an init function as an argument. + +VcsRoots, BuildTypes, and Templates of this project must be +registered inside project using the vcsRoot(), buildType(), and +template() methods respectively. + +Subprojects can be defined either in their own settings.kts or by +calling the subProjects() method in this project. + +To debug settings scripts in command-line, run the + + mvnDebug org.jetbrains.teamcity:teamcity-configs-maven-plugin:generate + +command and attach your debugger to the port 8000. + +To debug in IntelliJ Idea, open the 'Maven Projects' tool window (View -> +Tool Windows -> Maven Projects), find the generate task +node (Plugins -> teamcity-configs -> teamcity-configs:generate), +the 'Debug' option is available in the context menu for the task. +*/ + +version = "2021.2" +project(asteroid.AsteroidProject) \ No newline at end of file From 43a1bc23bd1a7d2799d6f60773f315657841b4b4 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 8 Mar 2022 22:59:22 +0900 Subject: [PATCH 3/3] Added CI discussion Signed-off-by: Cristian Le --- docs/CI.md | 75 ++++++++++++++++++++++++++++++++++++++++++ docs/CI_TeamCity.md | 47 ++++++++++++++++++++++++++ docs/Yocto(BitBake).md | 6 ++++ 3 files changed, 128 insertions(+) create mode 100644 docs/CI.md create mode 100644 docs/CI_TeamCity.md create mode 100644 docs/Yocto(BitBake).md diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 0000000..4aa3e9f --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,75 @@ +# Continuous Integration (CI) for AsteroidOS + +In the various repositories managed by AsteroidOS, we provide a few portable CI configurations that you can use to setup +your own CI server. We offer two CI configurations: + +- Monolith (this repository): Contains all CI subprojects +- Individual (project's repository): Links to/Contains the minimum parent CI projects to build the project + +AsteroidOS relies on CI to: + +- Build and deploy the images for each smartwatch at https://asteroidos.org/install/ +- (TBD) Update the apps managed by AsteroidOS and deploy them to AsteroidOS Updater (TBD) +- Manage and provide a public bitbake cache server to speed up the compilation time, both for our developers and anyone + who wishes to contribute + +Due to the complex interconnection of the AsteroidOS project, for a bare minimum CI setup, we require that: + +| Requirements | TeamCity | +|---------------------------------|:--------:| +| Trigger on third-party commits | V | +| Bootstrap configurations [1] | V | +| Trigger on changes of subfolder | V | +| Trigger on commit message | V | + +[1] You can setup/manage your own CI server by simply pointing to your own forks of the project. + +| | TeamCity | +|-------------------------|---------------------------------------------------------------------------------------------------| +| Maintainers | [LecrisUT](https://github.com/LecrisUT) | +| Open Source | Closed Source | +| Main Advantage | - Can be setup on local development setup
- Can interconnect with external TeamCity servers | +| Limitations | - 3 Free Build agents
- 100 Build Configurations | +| Docker integration | - Can be run without Docker | +| Integration with Github | - Minimal
- Lacks PullRequest build approval
- Lacks a proper Github app | +| Other noteworthy points | - Integrates with some Cloud agents | + +-------------- + +## Available CI workflows + +### Build device + +Builds a specific device's image and deploys it to https://asteroidos.org/install/. Triggerred by: + +- Changes in files at `meta-smartwatch/meta-$DEVICE` if the commit comment does not start with `[NoBuild]:` +- Commits starting with a comma separated list formatted like `$DEVICE1,$DEVICE2:`, where the relevant device is in the + list + +### Rebuild device (from scratch) + +Builds a specific device's image without using any `sstate-cache`. Triggerred by: + +- Commits starting with `[Rebuild:$DEVICE]:` on either `meta-smarwatch` or `meta-asteroid` +- Trigger by `Rebuild all devices` + +### Build package + +Builds a specific app's package and deploys it to the package repository connected to AsteroidOS Updated (TBD). +Triggerred by: + +- Changes in files at `meta-asteroid-apps/$PACKAGE` if the commit comment does not start with `[NoBuild]:` +- Changes to the repository managing the specific package + +### Build all devices + +Triggers a `Build device` for all smartwatches. Triggerred by: + +- Daily check if any of the `Build package` was triggerred +- Weekly schedule if that week there were no local contributions + +### Rebuild all devices + +Triggers a `Rebuild device (from scratch)` for all smarwatches. Triggerred by: + +- Monthly schedule \ No newline at end of file diff --git a/docs/CI_TeamCity.md b/docs/CI_TeamCity.md new file mode 100644 index 0000000..575fc0a --- /dev/null +++ b/docs/CI_TeamCity.md @@ -0,0 +1,47 @@ +# TeamCity CI setup + +This setup was constructed so that it can be easily ported by anyone who want to contribute and develop on their own +forks. Just follow these steps to get your CI server up and running (order of opperations is important): + +1. Install a local TeamCity server +2. Create a new project named `Asteroid` pointing to this repository and the specific branch you wish to track and/or + push these settings to +3. Configure the Context Parameters +4. Resolve the red errors related to the missing authenthication by providing a Github access token with at least read + permission to public repositories. +5. (optional) Change the settings repository to point to the project one. + +Feel free to change, archive, delete any of the sub-projects to fit your needs and write access. + +### Context Parameters + +For a full list check `.teamcity/$Project/Settings.kt`. + +- `Fork`: [AsteroidOS]
+The fork that should be attached to the current CI project. +- `Devices`: [sturgeon,catfish]
+A comma-separated list of devices you wish to manage. +- `Packages`: [asteroid-launcher]
+A comma-separated list of Asteroid packages you wish to manage. +- (optional) `Upstream`: [AsteroidOS]
+The upstream repository of all the other parent/child projects. +- (optional) `DeploySstate`: [false]
+Enables uploading to the `sstate-cache` server. +- (optional) `PullRequests`: [false]
+Enables tracking pull requests on `fork`. +- (optional) `CommitStatus`: [false]
+Enables pushing commit status to the `fork`. +- (optional) `CommitUser`: [`Fork`]
+The commit user pushing commit status. +*Required if `CommitStatus == true`*. + +### Other configurations + +- (optional) `$Project/SSH Keys/Sstate Server Key`
+The SSH key used to upload to your own `sstate-cache` server. +*Required if `DeploySstate == true`*. +- (optional) `$Project/Versioned Settings/Tokens/credentialsJSON:0b803d82-f0a8-42ee-b8f9-0fca109a14ab`
+Github OAuth token for `CommitUser` or your account. +*Required if `PullRequests || CommitStatus == true`*. + +### Other Notes \ No newline at end of file diff --git a/docs/Yocto(BitBake).md b/docs/Yocto(BitBake).md new file mode 100644 index 0000000..f5994c3 --- /dev/null +++ b/docs/Yocto(BitBake).md @@ -0,0 +1,6 @@ +# Setting up Yocto (Bitbake) development environment + +## Sstate-cache server + +AsteroidOS provides a public [`sstate-cache` server](https://sstate.asteroid.org) that you can use to speed up your +first time compilation (2 hours -> 10 min). \ No newline at end of file