Skip to content

Commit

Permalink
Support for docker/container images (#1056)
Browse files Browse the repository at this point in the history
This PR integrates a first-hand implementation for generating docker/container versions of RaspberryMatic. Please note, that this is still partly WIP and not released yet. So if you are testing the docker implementation you are somehwat on your own until we release this to the public. However, please warmly welcome this new supported virtual platform for RaspberryMatic! 

Co-authored-by: Jens Maus <[email protected]>
  • Loading branch information
angelnu and jens-maus authored Jan 5, 2021
1 parent 4e08a20 commit 7032899
Show file tree
Hide file tree
Showing 54 changed files with 1,779 additions and 64 deletions.
86 changes: 83 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ jobs:
runs-on: ubuntu-20.04
needs: repo-check
timeout-minutes: 480
if: always() && github.event_name != 'schedule' || needs.repo-check.outputs.has-commits > 0
if: github.event_name != 'schedule' || needs.repo-check.outputs.has-commits > 0

strategy:
fail-fast: false
matrix:
platform: [rpi0, rpi2, rpi3, rpi4, tinkerboard, intelnuc, ova]
platform: [rpi0, rpi2, rpi3, rpi4, tinkerboard, intelnuc, ova, oci_amd64, oci_arm64, oci_arm]

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -167,7 +167,6 @@ jobs:
max_releases: 1

- name: upload build as workflow artifact
if: github.event_name != 'schedule'
uses: actions/upload-artifact@v2
with:
path: release/RaspberryMatic-${{ env.VERSION }}-${{ matrix.platform }}.zip*
Expand All @@ -189,3 +188,84 @@ jobs:
path: release/RaspberryMatic-${{ env.VERSION }}.ova*
name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}.ova
continue-on-error: true

oci-multiarch-build-push:
runs-on: ubuntu-latest
needs: build
if: github.event_name != 'pull_request' #Cannot enable for PRs since secrets are not available ^^^

steps:
- uses: actions/checkout@v2

- name: setup env
run: |
echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
echo "OCCU_VERSION=$(grep 'OCCU_VERSION =' buildroot-external/package/occu/occu.mk | cut -d' ' -f3 | cut -d'-' -f1)" >> $GITHUB_ENV
echo "VERSION=$(grep 'OCCU_VERSION =' buildroot-external/package/occu/occu.mk | cut -d' ' -f3 | cut -d'-' -f1).$(date +%Y%m%d)" >> $GITHUB_ENV
echo "GITHUB_SHA7=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV
- name: download oci_amd64.zip
uses: actions/download-artifact@v2
with:
name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}-oci_amd64.zip
- name: download oci_arm64.zip
uses: actions/download-artifact@v2
with:
name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}-oci_arm64.zip
- name: download oci_arm.zip
uses: actions/download-artifact@v2
with:
name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}-oci_arm.zip

- name: extract artifacts
run: |
rm -rf oci_build
mkdir -p oci_build
cd oci_build
for f in ../*.zip; do
unzip -o $f
rm $f
done
- name: Build tags
shell: bash
run: |
BASE_TAG="ghcr.io/${{ github.repository_owner }}/raspberrymatic"
UNIQUE_TAG="${BASE_TAG}:${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}"
if [[ ${GITHUB_EVENT_NAME} = 'schedule' || ${GITHUB_EVENT_NAME} = 'push' ]]; then
BRANCH="${GITHUB_REF##*/}"
if [ $BRANCH = master ]; then
BRANCH_TAG="${BASE_TAG}:snapshot"
else
BRANCH_TAG="${BASE_TAG}:snapshot-${BRANCH}"
fi
fi
echo "##[set-output name=unique_tag;]${UNIQUE_TAG}"
echo "##[set-output name=branch_tag;]${BRANCH_TAG}"
id: extract_branch

- uses: docker/setup-buildx-action@v1

- name: Login to GitHub Container Registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.CR_PAT }}

- name: build and push container image
uses: docker/build-push-action@v2
id: docker_build
with:
context: oci_build
file: buildroot-external/board/oci/Dockerfile
platforms: linux/amd64,linux/arm64,linux/arm
push: true
build-args: |
tar_prefix=RaspberryMatic-${{ env.VERSION }}-oci_
tags: |
${{ steps.extract_branch.outputs.unique_tag }}
${{ steps.extract_branch.outputs.branch_tag }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
/download
/release/*.zip
/release/*.img
/release/*.tar
/release/*.sha256
28 changes: 23 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,19 +46,37 @@ $(PRODUCTS): %:

build: | buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)/.config
@echo "[build: $(PRODUCT)]"
ifeq (,$(wildcard ./DUMMY_BUILD))
cd build-$(PRODUCT) && $(MAKE) O=$(shell pwd)/build-$(PRODUCT) -C ../buildroot-$(BUILDROOT_VERSION) BR2_EXTERNAL=../$(BUILDROOT_EXTERNAL) BR2_DL_DIR=$(BR2_DL_DIR) PRODUCT=$(PRODUCT) PRODUCT_VERSION=$(PRODUCT_VERSION)
else
#Dummy build - mainly for testing CI
echo -n "DO NOT MERGE THIS CHANGE - FOR CI test only!!!!"
mkdir -p build-$(PRODUCT)/images
tar -cf build-$(PRODUCT)/images/rootfs.tar LICENSE
touch build-$(PRODUCT)/images/sdcard.img
touch build-$(PRODUCT)/images/RaspberryMatic.ova
for f in `cat release/updatepkg/$(PRODUCT)/files-images.txt`; do touch build-$(PRODUCT)/images/$${f}; done
endif

release-all: $(addsuffix -release, $(PRODUCTS))
$(addsuffix -release, $(PRODUCTS)): %:
@$(MAKE) PRODUCT=$(subst -release,,$@) PRODUCT_VERSION=$(PRODUCT_VERSION) release

release: build
@echo "[creating release: $(PRODUCT)]"
$(eval BOARD := $(shell echo $(PRODUCT) | cut -d'_' -f2))
$(eval BOARD := $(shell echo $(PRODUCT) | cut -d'_' -f2-))
ifeq ($(findstring _oci_,$(PRODUCT)),_oci_)
#container images
$(eval IMG_EXT := 'tar')
cp -a build-$(PRODUCT)/images/rootfs.tar ./release/RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).tar
else
#regular images
$(eval IMG_EXT := 'img')
cp -a build-$(PRODUCT)/images/sdcard.img ./release/RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img
cd ./release && sha256sum RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img >RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img.sha256
endif
cd ./release && sha256sum RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT) >RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT).sha256
rm -f ./release/RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip
cd ./release && zip --junk-paths ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img.sha256 ../LICENSE ./updatepkg/$(PRODUCT)/EULA.de ./updatepkg/$(PRODUCT)/EULA.en
cd ./release && zip --junk-paths ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT) ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT).sha256 ../LICENSE ./updatepkg/$(PRODUCT)/EULA.de ./updatepkg/$(PRODUCT)/EULA.en
cd ./release && sha256sum RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip >RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip.sha256

updatePkg:
Expand All @@ -83,11 +101,11 @@ distclean: clean-all
@rm -rf download

.PHONY: menuconfig
menuconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)
menuconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)/.config
cd build-$(PRODUCT) && $(MAKE) O=$(shell pwd)/build-$(PRODUCT) -C ../buildroot-$(BUILDROOT_VERSION) BR2_EXTERNAL=../$(BUILDROOT_EXTERNAL) BR2_DL_DIR=$(BR2_DL_DIR) PRODUCT=$(PRODUCT) PRODUCT_VERSION=$(PRODUCT_VERSION) menuconfig

.PHONY: xconfig
xconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)
xconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)/.config
cd build-$(PRODUCT) && $(MAKE) O=$(shell pwd)/build-$(PRODUCT) -C ../buildroot-$(BUILDROOT_VERSION) BR2_EXTERNAL=../$(BUILDROOT_EXTERNAL) BR2_DL_DIR=$(BR2_DL_DIR) PRODUCT=$(PRODUCT) PRODUCT_VERSION=$(PRODUCT_VERSION) xconfig

.PHONY: savedefconfig
Expand Down
10 changes: 10 additions & 0 deletions buildroot-external/board/oci/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
FROM scratch

ARG TARGETARCH
ARG tar_prefix=rootfs

ADD $tar_prefix$TARGETARCH.tar /

CMD ["/sbin/init"]
VOLUME /usr/local
EXPOSE 80 22 161 443 1900 1901 1902 5987 1999 2000 2001 2002 2010 8088 8181 9099 9292 9293 10000 23272 41999 42000 42001 42010 43438 43439 48181 48899 49292 49880
22 changes: 22 additions & 0 deletions buildroot-external/board/oci/post-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/sh

# create VERSION file
echo "VERSION=${PRODUCT_VERSION}" >${TARGET_DIR}/VERSION
echo "PRODUCT=${PRODUCT}" >>${TARGET_DIR}/VERSION
echo "PLATFORM=oci" >>${TARGET_DIR}/VERSION

# fix some permissions
[ -e ${TARGET_DIR}/etc/monitrc ] && chmod 600 ${TARGET_DIR}/etc/monitrc

# remove unnecessary stuff from TARGET_DIR
rm -f ${TARGET_DIR}/etc/init.d/S01InitZRAMSwap
rm -f ${TARGET_DIR}/etc/init.d/S03InitURandom
rm -f ${TARGET_DIR}/etc/init.d/S21rngd
rm -f ${TARGET_DIR}/etc/init.d/S31bluetooth
#rm -f ${TARGET_DIR}/etc/init.d/S40network
rm -f ${TARGET_DIR}/etc/init.d/S49chrony
rm -f ${TARGET_DIR}/bin/dhcp.script

# link VERSION in /boot on rootfs
mkdir -p ${TARGET_DIR}/boot
ln -sf ../VERSION ${TARGET_DIR}/boot/VERSION
26 changes: 26 additions & 0 deletions buildroot-external/board/oci/post-image.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/bash

BOARD_DIR="$(dirname $0)"
BOARD_NAME="$(basename ${BOARD_DIR})"

# define DOCKER_ARCH on the BR2_CONFIG setup
if grep -Eq "^BR2_x86_64=y$" ${BR2_CONFIG}; then
DOCKER_ARCH=amd64
elif grep -Eq "^BR2_i386=y$" ${BR2_CONFIG}; then
DOCKER_ARCH=i386
elif grep -Eq "^BR2_aarch64=y$" ${BR2_CONFIG}; then
DOCKER_ARCH=arm64
elif grep -Eq "^BR2_arm=y$" ${BR2_CONFIG}; then
DOCKER_ARCH=arm
else
echo "Unknown architecture"
exit 1
fi

# make sure a factory reset is performed upon fresh start
touch "${TARGET_DIR}/usr/local/.doFactoryReset"

#Create docker image
docker build --file="${BOARD_DIR}/Dockerfile" --build-arg=tar_prefix=rootfs --tag=raspberrymatic:${DOCKER_ARCH}-latest ${BINARIES_DIR}

exit $?
111 changes: 111 additions & 0 deletions buildroot-external/configs/raspmatic_oci_amd64_defconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
BR2_x86_64=y
BR2_DL_DIR="$(TOPDIR)/../download"
BR2_KERNEL_MIRROR="https://www.kernel.org/pub"
BR2_GNU_MIRROR="http://ftp.gnu.org/pub/gnu"
BR2_OPTIMIZE_2=y
BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_EQ3_PATH)/patches"
# BR2_COMPILER_PARANOID_UNSAFE_PATH is not set
BR2_TOOLCHAIN_BUILDROOT_GLIBC=y
BR2_KERNEL_HEADERS_4_4=y
BR2_PACKAGE_GLIBC_UTILS=y
BR2_TOOLCHAIN_BUILDROOT_CXX=y
BR2_ROOTFS_LIB_DIR="lib"
BR2_ROOTFS_LIB32_DIR="lib32"
BR2_TARGET_GENERIC_HOSTNAME=""
BR2_TARGET_GENERIC_ISSUE="Welcome to RaspberryMatic"
BR2_TARGET_GENERIC_PASSWD_SHA512=y
BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y
# BR2_TARGET_GENERIC_GETTY is not set
# BR2_TARGET_GENERIC_REMOUNT_ROOTFS_RW is not set
# BR2_ENABLE_LOCALE_PURGE is not set
BR2_TARGET_TZ_INFO=y
BR2_TARGET_LOCALTIME="Europe/Berlin"
BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_EQ3_PATH)/overlay/base $(BR2_EXTERNAL_EQ3_PATH)/overlay/base-raspmatic $(BR2_EXTERNAL_EQ3_PATH)/overlay/WebUI $(BR2_EXTERNAL_EQ3_PATH)/overlay/WebUI-raspmatic $(BR2_EXTERNAL_EQ3_PATH)/overlay/RFD $(BR2_EXTERNAL_EQ3_PATH)/overlay/base-raspmatic_oci"
BR2_ROOTFS_POST_BUILD_SCRIPT="$(BR2_EXTERNAL_EQ3_PATH)/board/oci/post-build.sh"
BR2_ROOTFS_POST_IMAGE_SCRIPT="$(BR2_EXTERNAL_EQ3_PATH)/board/oci/post-image.sh"
BR2_PACKAGE_BUSYBOX_CONFIG="$(BR2_EXTERNAL_EQ3_PATH)/Busybox.config"
BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y
BR2_PACKAGE_STRACE=y
BR2_PACKAGE_JQ=y
BR2_PACKAGE_DOSFSTOOLS=y
BR2_PACKAGE_DOSFSTOOLS_FATLABEL=y
BR2_PACKAGE_DOSFSTOOLS_FSCK_FAT=y
BR2_PACKAGE_DOSFSTOOLS_MKFS_FAT=y
BR2_PACKAGE_E2FSPROGS=y
BR2_PACKAGE_E2FSPROGS_E4DEFRAG=y
BR2_PACKAGE_E2FSPROGS_RESIZE2FS=y
BR2_PACKAGE_EXFAT=y
BR2_PACKAGE_EXFAT_UTILS=y
BR2_PACKAGE_F2FS_TOOLS=y
BR2_PACKAGE_NTFS_3G=y
BR2_PACKAGE_DEJAVU=y
BR2_PACKAGE_LIBERATION=y
BR2_PACKAGE_PSPLASH=y
BR2_PACKAGE_PSPLASH_PATCH="$(BR2_EXTERNAL_EQ3_PATH)/patches/psplash/raspmatic"
BR2_PACKAGE_I2C_TOOLS=y
BR2_PACKAGE_RNG_TOOLS=y
BR2_PACKAGE_NODEJS=y
BR2_PACKAGE_NODEJS_NPM=y
BR2_PACKAGE_NODEJS_MODULES_ADDITIONAL="async accepts mc charenc encodeurl inherits moment range-parser unorm adm-zip compress-commons end-of-stream ipaddr.js moment-timezone raw-body unpipe archiver concat-map escape-html is-buffer ms readable-stream url archiver-utils content-disposition etag isarray negotiator remove-trailing-separator util-deprecate array-flatten content-type express jmespath node-dir safe-buffer utils-merge async cookie finalhandler jsonfile node-localstorage sax uuid aws-sdk cookie-signature forwarded lazystream normalize-path send vary balanced-match core-util-is fresh lodash on-finished serve-static walkdir base64-js crc fs-extra loop-protect once setprototypeof winreg big-integer crc32-stream fs.realpath md5 parseurl slide wrappy bl cron glob media-typer path-is-absolute statuses write-file-atomic body-parser crypt graceful-fs merge-descriptors path-to-regexp string_decoder ws brace-expansion crypto-browserify http-errors methods process-nextick-args suncalc xml2js buffer debug iconv-lite mime proxy-addr tar-stream xmlbuilder buffer-crc32 depd ieee754 mime-db punycode type-is xtend buffer-shims destroy imurmurhash mime-types qs ultron zip-stream bytes ee-first inflight minimatch querystring universalify"
BR2_PACKAGE_CA_CERTIFICATES=y
BR2_PACKAGE_LIBOPENSSL_BIN=y
BR2_PACKAGE_FONTCONFIG=y
BR2_PACKAGE_LIBUSB=y
BR2_PACKAGE_LIBUSB_COMPAT=y
BR2_PACKAGE_LIBCURL=y
BR2_PACKAGE_LIBCURL_CURL=y
BR2_PACKAGE_LIBDNET=y
BR2_PACKAGE_LIBNETFILTER_CONNTRACK=y
BR2_PACKAGE_MSMTP=y
BR2_PACKAGE_AVAHI=y
BR2_PACKAGE_CHRONY=y
BR2_PACKAGE_ETHTOOL=y
BR2_PACKAGE_IPTABLES=y
BR2_PACKAGE_IW=y
BR2_PACKAGE_LIGHTTPD=y
BR2_PACKAGE_LIGHTTPD_OPENSSL=y
BR2_PACKAGE_LIGHTTPD_ZLIB=y
BR2_PACKAGE_LIGHTTPD_PCRE=y
BR2_PACKAGE_NETSNMP=y
BR2_PACKAGE_NETSNMP_WITH_MIB_MODULES="host ucd-snmp/dlmod ucd-snmp/diskio ucd-snmp/extensible ucd-snmp/pass mibII/mta_sendmail etherlike-mib/dot3StatsTable disman/event-mib agentx"
BR2_PACKAGE_NTP=y
# BR2_PACKAGE_NTP_NTPD is not set
BR2_PACKAGE_NTP_NTPDATE=y
BR2_PACKAGE_OPENSSH=y
BR2_PACKAGE_OPENVPN=y
BR2_PACKAGE_RSYNC=y
BR2_PACKAGE_SER2NET=y
BR2_PACKAGE_SOCAT=y
BR2_PACKAGE_TCPDUMP=y
BR2_PACKAGE_WGET=y
BR2_PACKAGE_WIREGUARD_TOOLS=y
BR2_PACKAGE_XINETD=y
BR2_PACKAGE_FILE=y
BR2_PACKAGE_LOCKFILE_PROGS=y
BR2_PACKAGE_LOGROTATE=y
BR2_PACKAGE_IRQBALANCE=y
BR2_PACKAGE_MONIT=y
BR2_PACKAGE_NUT=y
BR2_PACKAGE_TAR=y
BR2_PACKAGE_UTIL_LINUX_BINARIES=y
BR2_PACKAGE_UTIL_LINUX_LOSETUP=y
BR2_PACKAGE_UTIL_LINUX_RFKILL=y
BR2_PACKAGE_UTIL_LINUX_ZRAMCTL=y
BR2_PACKAGE_HOST_DOSFSTOOLS=y
BR2_PACKAGE_HOST_E2FSPROGS=y
BR2_PACKAGE_MULTILIB32=y
BR2_PACKAGE_MULTILIB32_DEFCONFIG="multilib32_i686_defconfig"
BR2_PACKAGE_OCCU=y
BR2_PACKAGE_OCCU_RF_PROTOCOL_HM_HMIP=y
BR2_PACKAGE_OCCU_WEBUI=y
BR2_PACKAGE_OCCU_WEBUI_REGAHSS_BETA=y
BR2_PACKAGE_OCCU_HMSERVER=y
BR2_PACKAGE_CLOUDMATIC=y
BR2_PACKAGE_HMLANGW=y
BR2_PACKAGE_DETECT_RADIO_MODULE=y
BR2_PACKAGE_DAEMONIZE=y
BR2_PACKAGE_SSDPD=y
BR2_PACKAGE_EQ3CONFIGD=y
BR2_PACKAGE_NEOSERVER=y
BR2_PACKAGE_RNDADDTOENTCNT=y
BR2_PACKAGE_JAVA_AZUL=y
Loading

0 comments on commit 7032899

Please sign in to comment.