Skip to content

Commit

Permalink
MDBF-588 Buildbot packages accessable in fixed repo URL per major ver…
Browse files Browse the repository at this point in the history 
…sion
  • Loading branch information
@grooverdan
grooverdan committed Oct 20, 2023
1 parent 5b20793 commit 4d06e70
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 109 deletions.
81 changes: 31 additions & 50 deletions common_factories.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import os
from utils import *
from constants import *
from buildbot.plugins import *
from buildbot.process.properties import Property, Properties
from buildbot.steps.package.rpm.rpmlint import RpmLint
Expand All @@ -10,6 +7,9 @@
from buildbot.process.remotecommand import RemoteCommand
from twisted.internet import defer

from utils import *
from constants import *


def getQuickBuildFactory(mtrDbPool):
f_quick_build = util.BuildFactory()
Expand Down Expand Up @@ -221,20 +221,12 @@ def getQuickBuildFactory(mtrDbPool):
timeout=7200,
haltOnFailure=True,
command=util.Interpolate(
"mkdir -p "
+ "/packages/"
+ "%(prop:tarbuildnum)s"
+ "/"
+ "%(prop:buildername)s"
+ " && sha256sum %(prop:mariadb_binary)s >> sha256sums.txt && cp "
+ "%(prop:mariadb_binary)s sha256sums.txt"
+ " /packages/"
+ "%(prop:tarbuildnum)s"
+ "/"
+ "%(prop:buildername)s"
+ "/"
+ " && sync /packages/"
+ "%(prop:tarbuildnum)s"
"""
mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s \
&& sha256sum %(prop:mariadb_binary)s >> sha256sums.txt \
&& cp %(prop:mariadb_binary)s sha256sums.txt /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ \
&& sync /packages/%(prop:tarbuildnum)s
"""
),
doStepIf=savePackage,
)
Expand Down Expand Up @@ -285,18 +277,17 @@ def getRpmAutobakeFactory(mtrDbPool):
)
f_rpm_autobake.addStep(
steps.ShellCommand(
name="Fetch packages for MariaDB-compat",
command=util.Interpolate(
"wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm "
+ os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+ "/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm"
+ "&&"
+ "wget --no-check-certificate -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm "
+ os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+ "/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm",
arch=getArch,
rpm_type=util.Property("rpm_type"),
),
name="fetch packages for MariaDB-compat",
command=[
"sh",
"-c",
util.Interpolate(
'wget --no-check-certificate -cO ../MariaDB-shared-5.3.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-5.3-%(kw:arch)s.rpm" && wget -cO ../MariaDB-shared-10.1.%(kw:arch)s.rpm "%(kw:url)s/helper_files/mariadb-shared-10.1-kvm-rpm-%(kw:rpm_type)s-%(kw:arch)s.rpm"',
arch=getArch,
url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
rpm_type=util.Property("rpm_type"),
),
],
doStepIf=hasCompat,
)
)
Expand Down Expand Up @@ -381,15 +372,14 @@ def getRpmAutobakeFactory(mtrDbPool):
cat << EOF > MariaDB.repo
[MariaDB-%(prop:branch)s]
name=MariaDB %(prop:branch)s repo (build %(prop:tarbuildnum)s)
baseurl="""
+ os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+ """/%(prop:tarbuildnum)s/%(prop:buildername)s/rpms
baseurl=%(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/rpms
gpgcheck=0
EOF
if [ "%(prop:rpm_type)s" = rhel8 ] || [ "%(prop:rpm_type)s" = centosstream8 ]; then
echo "module_hotfixes = 1" >> MariaDB.repo
fi
"""
""",
url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
),
]
)
Expand All @@ -400,27 +390,18 @@ def getRpmAutobakeFactory(mtrDbPool):
timeout=7200,
haltOnFailure=True,
command=util.Interpolate(
"mkdir -p "
+ "/packages/"
+ "%(prop:tarbuildnum)s"
+ "/"
+ "%(prop:buildername)s"
+ " && cp -r MariaDB.repo rpms srpms /packages/"
+ "%(prop:tarbuildnum)s"
+ "/"
+ "%(prop:buildername)s"
+ "/"
+ " && ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo %(prop:branch)s-latest-%(prop:buildername)s.repo "
+ " && sync /packages/"
+ "%(prop:tarbuildnum)s"
""""
mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s &&
cp -r MariaDB.repo rpms srpms /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/ &&
ln -sf %(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo %(prop:branch)s-latest-%(prop:buildername)s.repo &&
sync /packages/%(prop:tarbuildnum)s
"""
),
doStepIf=lambda step: hasFiles(step) and savePackage(step),
descriptionDone=util.Interpolate(
"""
Repository available with: curl """
+ os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")
+ """/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo
"""
Repository available with: curl %(kw:url)s/%(prop:tarbuildnum)s/%(prop:buildername)s/MariaDB.repo -o /etc/yum.repos.d/MariaDB.repo""",
url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
),
)
)
Expand Down
107 changes: 48 additions & 59 deletions master-galera/master.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@ from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCom
from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
from buildbot.steps.source.github import GitHub
from buildbot.process.remotecommand import RemoteCommand
from datetime import timedelta
from twisted.internet import defer

import docker
import os
import sys
import os
import docker
from datetime import timedelta

sys.setrecursionlimit(10000)

Expand Down Expand Up @@ -40,8 +39,12 @@ c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/serve
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.

c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/")

# Custom plugin
# exec(open("grid.py").read())

# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
Expand Down Expand Up @@ -169,12 +172,12 @@ for platform in all_platforms:

for w_name in machines:
for i in worker_ids:
for os_i in os_info:
if platform in os_info[os_i]["arch"]:
for os in os_info:
if platform in os_info[os]["arch"]:
quay_name = "quay.io/mariadb-foundation/bb-worker:" + "".join(
os_i.split("-")
os.split("-")
)
os_name = os_i
os_name = os
if "ubuntu" in quay_name:
quay_name = quay_name[:-2] + "." + quay_name[-2:]
if platform == "x86":
Expand Down Expand Up @@ -270,24 +273,22 @@ f_deb_build.addStep(
timeout=7200,
haltOnFailure=True,
command=util.Interpolate(
"mkdir -p "
+ "/packages/"
+ "%(prop:branch)s"
+ "/"
+ "%(prop:revision)s"
+ "/"
+ "%(prop:buildername)s"
+ " && cp -r debs/ sha256sums.txt /packages/"
+ "%(prop:branch)s"
+ "/"
+ "%(prop:revision)s"
+ "/"
+ "%(prop:buildername)s"
+ "/"
+ " && sync /packages/"
+ "%(prop:branch)s"
+ "/"
+ "%(prop:revision)s"
"""
source /etc/os-release; \
mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \
&& cp -r debs/ sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \
&& cat << EOF > /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources \
X-Repolib-Name: Galera
Types: deb
URIs: %(kw:url)s/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s
Suites: $VERSION_CODENAME
Components: main main/debug
Trusted: yes
EOF
&& ln -s %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.sources /packages/%(prop:branch)s-latest-%(prop:buildername)s.sources \
&& sync /packages/%(prop:branch)s/%(prop:revision)s
""",
url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
),
doStepIf=lambda step: savePackage(step, savedPackageBranches),
)
Expand Down Expand Up @@ -326,25 +327,19 @@ f_rpm_build.addStep(
timeout=7200,
haltOnFailure=True,
command=util.Interpolate(
"mkdir -p "
+ "/packages/"
+ "%(prop:branch)s"
+ "/"
+ "%(prop:revision)s"
+ "/"
+ "%(prop:buildername)s"
+ " && cp -r rpms srpms sha256sums.txt"
+ " /packages/"
+ "%(prop:branch)s"
+ "/"
+ "%(prop:revision)s"
+ "/"
+ "%(prop:buildername)s"
+ "/"
+ " && sync /packages/"
+ "%(prop:branch)s"
+ "/"
+ "%(prop:revision)s"
"""
mkdir -p /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s \
cat << EOF > packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo
[Galera-%(prop:branch)s]
name=Galera %(prop:branch)s repo (build %(prop:tarbuildnum)s)
baseurl=%(kw:url)s/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/rpms
gpgcheck=0
EOF
&& cp -r rpms srpms sha256sums.txt /packages/%(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/ \
&& ln -sf %(prop:branch)s/%(prop:revision)s/%(prop:buildername)s/galera.repo %(prop:branch)s-latest-%(prop:buildername)s.repo \
&& sync /packages/%(prop:branch)s/%(prop:revision)s
""",
url=os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org"),
),
doStepIf=lambda step: savePackage(step, savedPackageBranches),
)
Expand All @@ -358,36 +353,30 @@ f_rpm_build.addStep(
####### BUILDERS LIST
c["builders"] = []

for os_i in os_info:
for arch in os_info[os_i]["arch"]:
builder_name = "gal-" + arch + "-" + os_i
for os in os_info:
for arch in os_info[os]["arch"]:
builder_name = "gal-" + arch + "-" + os

assert builder_name in builders_galera

worker_name = arch + "-bbw-docker-" + os_i
worker_name = arch + "-bbw-docker-" + os
if arch == "amd64":
worker_name = "x64-bbw-docker-" + os_i
worker_name = "x64-bbw-docker-" + os
if arch == "x86":
worker_name = "x64-bbw-docker-" + os_i + "-i386"
worker_name = "x64-bbw-docker-" + os + "-i386"

if os_info[os_i]["type"] == "rpm":
if os_info[os]["type"] == "rpm":
factory = f_rpm_build
else:
factory = f_deb_build

env = {}
if os == "ubuntu-2004":
print("using gcc/++-10")
env = {"CC": "gcc-10", "CXX": "g++10"}

c["builders"].append(
util.BuilderConfig(
name=builder_name,
workernames=workers[worker_name],
tags=[os_i, "galera", "gcc"],
tags=[os, "galera", "gcc"],
collapseRequests=True,
nextBuild=nextBuild,
env=env,
factory=factory,
)
)
Expand Down

0 comments on commit 4d06e70

Please sign in to comment.