Skip to content

Commit

Permalink
handle unsupported builds in github build action (SynoCommunity#4029)
Browse files Browse the repository at this point in the history
- move checks to spksrc.pre-check.mk
- update build_status.sh to ignore errors due to unsupported builds
- use synocommunity/spksrc for build status job to avoid access errors
- ntopng: replace REQUIRED_DSM by UNSUPPORTED_ARCHS to allow build of armv7-1.2
  • Loading branch information
hgy59 authored Jun 16, 2020
1 parent c1efce7 commit 97938e3
Show file tree
Hide file tree
Showing 11 changed files with 132 additions and 121 deletions.
13 changes: 7 additions & 6 deletions .github/actions/build-push-pr.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ do
fi
done

echo "===> TARGET: ${GH_ARCH}"
echo "===> ARCH packages: ${arch_packages}"
echo "===> NOARCH packages: ${noarch_packages}"

Expand All @@ -66,18 +67,18 @@ for package in ${build_packages}
do
echo "----------------------------------------"
# make sure that the package exists
if [ -f "/github/workspace/spk/$package/Makefile" ]; then
# use TCVERSION and ARCH parameters to get real exit code.
make TCVERSION=${GH_ARCH##*-} ARCH=${GH_ARCH%%-*} -C /github/workspace/spk/${package}
if [ -f "./spk/${package}/Makefile" ]; then
# use TCVERSION and ARCH to get real exit codes.
make TCVERSION=${GH_ARCH##*-} ARCH=${GH_ARCH%%-*} -C ./spk/${package}
result=$?

if [ $result -eq 0 ];
then
echo "$(date --date=now +"%Y.%m.%d %H:%M:%S") - $package ($GH_ARCH) DONE" >> ${BUILD_SUCCESS_FILE}
echo "$(date --date=now +"%Y.%m.%d %H:%M:%S") - $package: ($GH_ARCH) DONE" >> ${BUILD_SUCCESS_FILE}
else
echo "$(date --date=now +"%Y.%m.%d %H:%M:%S") - $package ($GH_ARCH) FAILED ($result)" >> ${BUILD_ERROR_FILE}
echo "$(date --date=now +"%Y.%m.%d %H:%M:%S") - $package: ($GH_ARCH) FAILED" >> ${BUILD_ERROR_FILE}
fi
else
echo "/github/workspace/spk/${package}/Makefile not found"
echo "spk/${package}/Makefile not found"
fi
done
57 changes: 48 additions & 9 deletions .github/actions/build_status.sh
Original file line number Diff line number Diff line change
@@ -1,20 +1,59 @@
#!/bin/bash
# We do not terminate the build on errors as we want to build all packages.
#
# List the successfully built, the unsupported and the failed packages
# by name an timestamp.
#
# We do not want to terminate the build on errors as we want to build all
# packages.
# Therfore failed builds are logged in the build error file defined by the
# env variable $BUILD_ERROR_FILE
# If this file exists and contains at least one line, we show here the content
# of the file and exit with error.
# Otherwise we show the other file containing the sucessfully built packages.
# If this file exists and contains at least one line, we exit with error.
# A special log file $BUILD_UNSUPPORTED_FILE contains known make errors that
# are ignore here (unsupported ARCH or version of Toolchain)
# Such packages are separately logged and removed from errors file.
#
# Variables:
# BUILD_SUCCESS_FILE defines the name of the file with built packages
# BUILD_UNSUPPORTED_FILE defines the name of the file with unsupported packages
# BUILD_ERROR_FILE defines the name of the file with build errors
#

echo ""
echo "BUILD STATUS"
echo ""

echo "SUCCESS:"
if [ -f "${BUILD_SUCCESS_FILE}" ]; then
cat "${BUILD_SUCCESS_FILE}"
else
echo "none."
fi

echo ""
echo "UNSUPPORTED (skipped):"
if [ -f "${BUILD_UNSUPPORTED_FILE}" ]; then
cat "${BUILD_UNSUPPORTED_FILE}"
if [ -f "${BUILD_ERROR_FILE}" ]; then
# remove unsupported packages from errors:
unsupported_packages=$(cat "${BUILD_UNSUPPORTED_FILE}" | grep -Po "\- \K.*:" | sort -u | tr '\n' '|' | sed -e 's/|$//')
cat "${BUILD_ERROR_FILE}" | grep -Pv "\- (${unsupported_packages}) " > "${BUILD_ERROR_FILE}.tmp"
rm -f "${BUILD_ERROR_FILE}"
mv "${BUILD_ERROR_FILE}.tmp" "${BUILD_ERROR_FILE}"
fi
else
echo "none."
fi

echo ""
echo "ERRORS:"
if [ -f "${BUILD_ERROR_FILE}" ]; then
if [ $(cat "${BUILD_ERROR_FILE}" | wc -l) -gt 0 ]; then
cat "${BUILD_ERROR_FILE}"
echo ""
echo "Please analyze the log file of the build job."
echo "See log file of the build job to analyze the error(s)."
echo ""
# let build status job fail
exit 1
fi
fi
if [ -f "${BUILD_SUCCESS_FILE}" ]; then
cat "${BUILD_SUCCESS_FILE}"
fi
echo "none."
echo ""
19 changes: 12 additions & 7 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,9 @@ jobs:
GH_FILES: ${{ steps.getfile.outputs.files }} ${{ steps.getfile_pr.outputs.files }}
# https://github.com/SynoCommunity/spksrc/wiki/Compile-and-build-rules
GH_ARCH: ${{ matrix.arch }}
BUILD_ERROR_FILE: build_errors.txt
BUILD_SUCCESS_FILE: build_success.txt
BUILD_ERROR_FILE: /github/workspace/build_errors.txt
BUILD_UNSUPPORTED_FILE: /github/workspace/build_unsupported.txt
BUILD_SUCCESS_FILE: /github/workspace/build_success.txt

- name: Build Package (Tag)
if: startsWith(github.ref, 'refs/tags/')
Expand All @@ -82,8 +83,9 @@ jobs:
# https://github.com/SynoCommunity/spksrc/wiki/Compile-and-build-rules
GH_ARCH: ${{ matrix.arch }}
# API_KEY: ${{ secrets.PUBLISH_API_KEY }}
BUILD_ERROR_FILE: build_errors.txt
BUILD_SUCCESS_FILE: build_success.txt
BUILD_ERROR_FILE: /github/workspace/build_errors.txt
BUILD_UNSUPPORTED_FILE: /github/workspace/build_unsupported.txt
BUILD_SUCCESS_FILE: /github/workspace/build_success.txt

- name: List packages
run: mkdir -p packages; ls packages
Expand All @@ -103,10 +105,13 @@ jobs:
- name: Build Status
# We need this status since we don't want to stop the build on errors.
# Here we fail on build errors found in the build error file.
run: ./.github/actions/build_status.sh
uses: docker://synocommunity/spksrc:latest
with:
entrypoint: ./.github/actions/build_status.sh
env:
BUILD_ERROR_FILE: build_errors.txt
BUILD_SUCCESS_FILE: build_success.txt
BUILD_ERROR_FILE: /github/workspace/build_errors.txt
BUILD_UNSUPPORTED_FILE: /github/workspace/build_unsupported.txt
BUILD_SUCCESS_FILE: /github/workspace/build_success.txt

release:
needs: build
Expand Down
6 changes: 3 additions & 3 deletions mk/spksrc.common.mk
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ RUN = cd $(WORK_DIR)/$(PKG_DIR) && env $(ENV)

# Pip command
PIP ?= pip
# Why ask for the same thing twice? Always cache downloads
# Why ask for the same thing twice? Always cache downloads
PIP_CACHE_OPT ?= --cache-dir $(PIP_DIR)
PIP_WHEEL = $(PIP) wheel --no-binary :all: $(PIP_CACHE_OPT) --no-deps --requirement $(WORK_DIR)/wheelhouse/requirements.txt --wheel-dir $(WORK_DIR)/wheelhouse --build-dir $(WORK_DIR)/wheelbuild

Expand All @@ -32,8 +32,8 @@ LEGACY_ARCHS = $(sort $(filter-out $(SUPPORTED_ARCHS), $(AVAILABLE_ARCHS)))
# SRM - Synology Router Manager
SRM_ARCHS = northstarplus ipq806x dakota

# Use x64 when kernels are not needed
ARCHS_NO_KRNLSUPP = $(filter-out x64%, $(SUPPORTED_ARCHS))
# Use generic archs when kernels are not needed
ARCHS_NO_KRNLSUPP = $(filter-out x64% armv7% aarch64%, $(SUPPORTED_ARCHS))

# remove archs for generic x64 build
ARCHS_DUPES := $(filter-out apollolake% avoton% braswell% broadwell% broadwellnk% bromolow% cedarview% denverton% dockerx64% geminilake% grantley% purley% kvmx64% x86% x86_64%, $(SUPPORTED_ARCHS))
Expand Down
36 changes: 5 additions & 31 deletions mk/spksrc.cross-cc.mk
Original file line number Diff line number Diff line change
Expand Up @@ -22,37 +22,7 @@ endif

#####

ifneq ($(REQ_KERNEL),)
ifeq ($(ARCH),x64)
@$(error x64 arch cannot be used when REQ_KERNEL is set )
endif
endif

# Check if package supports ARCH
ifneq ($(UNSUPPORTED_ARCHS),)
ifneq (,$(findstring $(ARCH),$(UNSUPPORTED_ARCHS)))
@$(error Arch '$(ARCH)' is not a supported architecture )
endif
endif

# Check minimum DSM requirements of package
ifneq ($(REQUIRED_DSM),)
ifeq (,$(findstring $(ARCH),$(SRM_ARCHS)))
ifneq ($(REQUIRED_DSM),$(firstword $(sort $(TCVERSION) $(REQUIRED_DSM))))
@$(error DSM Toolchain $(TCVERSION) is lower than required version in Makefile $(REQUIRED_DSM))
endif
endif
endif
# Check minimum SRM requirements of package
ifneq ($(REQUIRED_SRM),)
ifeq ($(ARCH),$(findstring $(ARCH),$(SRM_ARCHS)))
ifneq ($(REQUIRED_SRM),$(firstword $(sort $(TCVERSION) $(REQUIRED_SRM))))
@$(error SRM Toolchain $(TCVERSION) is lower than required version in Makefile $(REQUIRED_SRM))
endif
endif
endif

#####
include ../../mk/spksrc.pre-check.mk

include ../../mk/spksrc.cross-env.mk

Expand Down Expand Up @@ -105,6 +75,10 @@ include ../../mk/spksrc.dependency-tree.mk
.PHONY: all-archs
all-archs: $(addprefix arch-,$(AVAILABLE_ARCHS))

####

arch-%:
@$(MSG) Building package for arch $*
-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*)))

####
16 changes: 5 additions & 11 deletions mk/spksrc.cross-go.mk
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,7 @@ ifneq ($(REQ_KERNEL),)
@$(error go modules cannot build when REQ_KERNEL is set)
endif

# Check if package supports ARCH
ifneq ($(UNSUPPORTED_ARCHS),)
ifneq (,$(findstring $(ARCH),$(UNSUPPORTED_ARCHS)))
@$(error Arch '$(ARCH)' is not a supported architecture )
endif
endif

#####
include ../../mk/spksrc.pre-check.mk

include ../../mk/spksrc.cross-env.mk

Expand Down Expand Up @@ -117,9 +110,6 @@ clean:

all: install plist

### For make kernel-required (used by spksrc.spk.mk)
include ../../mk/spksrc.kernel-required.mk

### For make digests
include ../../mk/spksrc.generate-digests.mk

Expand All @@ -129,6 +119,10 @@ include ../../mk/spksrc.dependency-tree.mk
.PHONY: all-archs
all-archs: $(addprefix arch-,$(AVAILABLE_ARCHS))

####

arch-%:
@$(MSG) Building package for arch $*
-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*)))

####
17 changes: 3 additions & 14 deletions mk/spksrc.install-resources.mk
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ endif
DIST_FILE = $(DISTRIB_DIR)/$(LOCAL_FILE)
DIST_EXT = $(PKG_EXT)


#####

ifneq ($(REQ_KERNEL),)
Expand Down Expand Up @@ -99,20 +98,10 @@ include ../../mk/spksrc.dependency-tree.mk
.PHONY: all-archs
all-archs: $(addprefix arch-,$(AVAILABLE_ARCHS))

####

arch-%:
@$(MSG) Building package for arch $*
-@MAKEFLAGS= $(MAKE) ARCH=$(basename $(subst -,.,$(basename $(subst .,,$*)))) TCVERSION=$(if $(findstring $*,$(basename $(subst -,.,$(basename $(subst .,,$*))))),$(DEFAULT_TC),$(notdir $(subst -,/,$*)))

.PHONY: kernel-required
kernel-required:
@if [ -n "$(REQ_KERNEL)" ]; then \
exit 1 ; \
fi
@for depend in $(BUILD_DEPENDS) $(DEPENDS) ; do \
if $(MAKE) --no-print-directory -C ../../$$depend kernel-required >/dev/null 2>&1 ; then \
exit 0 ; \
else \
exit 1 ; \
fi ; \
done

####
13 changes: 1 addition & 12 deletions mk/spksrc.native-cc.mk
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,4 @@ include ../../mk/spksrc.generate-digests.mk
### For make dependency-tree
include ../../mk/spksrc.dependency-tree.mk

.PHONY: kernel-required
kernel-required:
@if [ -n "$(REQ_KERNEL)" ]; then \
exit 1 ; \
fi
@for depend in $(DEPENDS) ; do \
if $(MAKE) --no-print-directory -C ../../$$depend kernel-required >/dev/null 2>&1 ; then \
exit 0 ; \
else \
exit 1 ; \
fi ; \
done
####
45 changes: 45 additions & 0 deletions mk/spksrc.pre-check.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Common requirement checks

# Check for build for generic archs, these are not supporting `require kernel`.
ifneq ($(REQ_KERNEL),)
ifneq (,$(findstring $(ARCH),x64 aarch64 armv7))
ifneq ($(BUILD_UNSUPPORTED_FILE),)
$(shell echo $(date --date=now +"%Y.%m.%d %H:%M:%S") - $(NAME): Arch '$(ARCH)' cannot be used when REQ_KERNEL is set >> $(BUILD_UNSUPPORTED_FILE))
endif
@$(error Arch '$(ARCH)' cannot be used when REQ_KERNEL is set)
endif
endif

# Check whether package supports ARCH
ifneq ($(UNSUPPORTED_ARCHS),)
ifneq (,$(findstring $(ARCH),$(UNSUPPORTED_ARCHS)))
ifneq (,$(BUILD_UNSUPPORTED_FILE))
$(shell echo $(date --date=now +"%Y.%m.%d %H:%M:%S") - $(NAME): Arch '$(ARCH)' is not a supported architecture >> $(BUILD_UNSUPPORTED_FILE))
endif
@$(error Arch '$(ARCH)' is not a supported architecture)
endif
endif

# Check minimum DSM requirements of package
ifneq ($(REQUIRED_DSM),)
ifeq (,$(findstring $(ARCH),$(SRM_ARCHS)))
ifneq ($(REQUIRED_DSM),$(firstword $(sort $(TCVERSION) $(REQUIRED_DSM))))
ifneq (,$(BUILD_UNSUPPORTED_FILE))
$(shell echo $(date --date=now +"%Y.%m.%d %H:%M:%S") - $(NAME): DSM Toolchain $(TCVERSION) is lower than required version $(REQUIRED_DSM) >> $(BUILD_UNSUPPORTED_FILE))
endif
@$(error DSM Toolchain $(TCVERSION) is lower than required version in Makefile $(REQUIRED_DSM))
endif
endif
endif

# Check minimum SRM requirements of package
ifneq ($(REQUIRED_SRM),)
ifeq ($(ARCH),$(findstring $(ARCH),$(SRM_ARCHS)))
ifneq ($(REQUIRED_SRM),$(firstword $(sort $(TCVERSION) $(REQUIRED_SRM))))
ifneq (,$(BUILD_UNSUPPORTED_FILE))
$(shell echo $(date --date=now +"%Y.%m.%d %H:%M:%S") - $(NAME): SRM Toolchain $(TCVERSION) is lower than required version $(REQUIRED_SRM) >> $(BUILD_UNSUPPORTED_FILE))
endif
@$(error SRM Toolchain $(TCVERSION) is lower than required version in Makefile $(REQUIRED_SRM))
endif
endif
endif
27 changes: 1 addition & 26 deletions mk/spksrc.spk.mk
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,7 @@ SPK_FILE_NAME = $(PACKAGES_DIR)/$(SPK_NAME)_$(SPK_NAME_ARCH)-$(SPK_TCVERS)_$(SPK

#####

# Check if package supports ARCH
ifneq ($(UNSUPPORTED_ARCHS),)
ifneq (,$(findstring $(ARCH),$(UNSUPPORTED_ARCHS)))
@$(error Arch '$(ARCH)' is not a supported architecture )
endif
endif

# Check minimum DSM requirements of package
ifneq ($(REQUIRED_DSM),)
ifeq (,$(findstring $(ARCH),$(SRM_ARCHS)))
ifneq ($(REQUIRED_DSM),$(firstword $(sort $(TCVERSION) $(REQUIRED_DSM))))
@$(error DSM Toolchain $(TCVERSION) is lower than required version in Makefile $(REQUIRED_DSM))
endif
endif
endif
# Check minimum SRM requirements of package
ifneq ($(REQUIRED_SRM),)
ifeq ($(ARCH),$(findstring $(ARCH),$(SRM_ARCHS)))
ifneq ($(REQUIRED_SRM),$(firstword $(sort $(TCVERSION) $(REQUIRED_SRM))))
@$(error SRM Toolchain $(TCVERSION) is lower than required version in Makefile $(REQUIRED_SRM))
endif
endif
endif

#####
include ../../mk/spksrc.pre-check.mk

# Even though this makefile doesn't cross compile, we need this to setup the cross environment.
include ../../mk/spksrc.cross-env.mk
Expand Down Expand Up @@ -104,7 +80,6 @@ ifneq ($(strip $(SPK_ICON)),)
include ../../mk/spksrc.icon.mk
endif

.PHONY: $(WORK_DIR)/INFO
$(WORK_DIR)/INFO:
$(create_target_dir)
@$(MSG) "Creating INFO file for $(SPK_NAME)"
Expand Down
4 changes: 2 additions & 2 deletions spk/ntopng/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ DEPENDS = cross/$(SPK_NAME)

SPK_DEPENDS = "redis"

# toolchains of DSM <=5.2 have outdated compiler
REQUIRED_DSM = 6.1
# toolchains of DSM <=5.2 have outdated compiler (PPC archs except qoriq)
UNSUPPORTED_ARCHS = powerpc ppc824x ppc853x ppc854x

MAINTAINER = hgy59
DESCRIPTION = ntopng is the next generation version of the original ntop, a network traffic probe that monitors network usage. ntopng is based on libpcap and it has been written in a portable way in order to virtually run on every Unix platform, MacOSX and on Windows as well.
Expand Down

0 comments on commit 97938e3

Please sign in to comment.