[WIP] Initial support for containers

.github/workflows/ci.yml

@@ -20,12 +20,12 @@ jobs:
     runs-on: ubuntu-20.04
     needs: repo-check
     timeout-minutes: 480
-    if: always() && github.event_name != 'schedule' || needs.repo-check.outputs.has-commits > 0
+    if: github.event_name != 'schedule' || needs.repo-check.outputs.has-commits > 0
 
     strategy:
       fail-fast: false
       matrix:
-        platform: [rpi0, rpi2, rpi3, rpi4, tinkerboard, intelnuc, ova]
+        platform: [rpi0, rpi2, rpi3, rpi4, tinkerboard, intelnuc, ova, oci_amd64, oci_arm64, oci_arm]
 
     steps:
     - uses: actions/checkout@v2
@@ -167,7 +167,6 @@ jobs:
         max_releases: 1
 
     - name: upload build as workflow artifact
-      if: github.event_name != 'schedule'
       uses: actions/upload-artifact@v2
       with:
         path: release/RaspberryMatic-${{ env.VERSION }}-${{ matrix.platform }}.zip*
@@ -189,3 +188,84 @@ jobs:
         path: release/RaspberryMatic-${{ env.VERSION }}.ova*
         name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}.ova
       continue-on-error: true
+
+  oci-multiarch-build-push:
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.event_name != 'pull_request' #Cannot enable for PRs since secrets are not available ^^^
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: setup env
+      run: |
+        echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+        echo "OCCU_VERSION=$(grep 'OCCU_VERSION =' buildroot-external/package/occu/occu.mk | cut -d' ' -f3 | cut -d'-' -f1)" >> $GITHUB_ENV
+        echo "VERSION=$(grep 'OCCU_VERSION =' buildroot-external/package/occu/occu.mk | cut -d' ' -f3 | cut -d'-' -f1).$(date +%Y%m%d)" >> $GITHUB_ENV
+        echo "GITHUB_SHA7=$(echo ${GITHUB_SHA::7})" >> $GITHUB_ENV
+
+    - name: download oci_amd64.zip
+      uses: actions/download-artifact@v2
+      with:
+        name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}-oci_amd64.zip
+    - name: download oci_arm64.zip
+      uses: actions/download-artifact@v2
+      with:
+        name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}-oci_arm64.zip
+    - name: download oci_arm.zip
+      uses: actions/download-artifact@v2
+      with:
+        name: RaspberryMatic-${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}-oci_arm.zip
+
+    - name: extract artifacts
+      run: |
+        rm -rf oci_build
+        mkdir -p oci_build
+        cd oci_build
+        for f in ../*.zip; do
+          unzip -o $f
+          rm $f
+        done
+
+    - name: Build tags
+      shell: bash
+      run: |
+        BASE_TAG="ghcr.io/${{ github.repository_owner }}/raspberrymatic"
+        UNIQUE_TAG="${BASE_TAG}:${{ env.VERSION }}-${{ env.GITHUB_SHA7 }}"
+        if [[ ${GITHUB_EVENT_NAME} = 'schedule' || ${GITHUB_EVENT_NAME} = 'push' ]]; then
+          BRANCH="${GITHUB_REF##*/}"
+          if [ $BRANCH = master ]; then
+            BRANCH_TAG="${BASE_TAG}:snapshot"
+          else
+            BRANCH_TAG="${BASE_TAG}:snapshot-${BRANCH}"
+          fi
+        fi
+        echo "##[set-output name=unique_tag;]${UNIQUE_TAG}"
+        echo "##[set-output name=branch_tag;]${BRANCH_TAG}"
+      id: extract_branch
+
+    - uses: docker/setup-buildx-action@v1
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.CR_PAT }}
+
+    - name: build and push container image
+      uses: docker/build-push-action@v2
+      id: docker_build
+      with:
+        context: oci_build
+        file: buildroot-external/board/oci/Dockerfile
+        platforms: linux/amd64,linux/arm64,linux/arm
+        push: true
+        build-args: |
+          tar_prefix=RaspberryMatic-${{ env.VERSION }}-oci_
+        tags: |
+          ${{ steps.extract_branch.outputs.unique_tag }}
+          ${{ steps.extract_branch.outputs.branch_tag }}
+
+    - name: Image digest
+      run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -5,4 +5,5 @@
 /download
 /release/*.zip
 /release/*.img
+/release/*.tar
 /release/*.sha256

Makefile

@@ -46,19 +46,37 @@ $(PRODUCTS): %:
 
 build: | buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)/.config
 	@echo "[build: $(PRODUCT)]"
+ifeq (,$(wildcard ./DUMMY_BUILD))
 	cd build-$(PRODUCT) && $(MAKE) O=$(shell pwd)/build-$(PRODUCT) -C ../buildroot-$(BUILDROOT_VERSION) BR2_EXTERNAL=../$(BUILDROOT_EXTERNAL) BR2_DL_DIR=$(BR2_DL_DIR) PRODUCT=$(PRODUCT) PRODUCT_VERSION=$(PRODUCT_VERSION)
+else
+	#Dummy build - mainly for testing CI
+	echo -n "DO NOT MERGE THIS CHANGE - FOR CI test only!!!!"
+	mkdir -p build-$(PRODUCT)/images
+	tar -cf build-$(PRODUCT)/images/rootfs.tar LICENSE
+	touch build-$(PRODUCT)/images/sdcard.img
+	touch build-$(PRODUCT)/images/RaspberryMatic.ova
+	for f in `cat release/updatepkg/$(PRODUCT)/files-images.txt`; do touch build-$(PRODUCT)/images/$${f}; done
+endif
 
 release-all: $(addsuffix -release, $(PRODUCTS))
 $(addsuffix -release, $(PRODUCTS)): %:
 	@$(MAKE) PRODUCT=$(subst -release,,$@) PRODUCT_VERSION=$(PRODUCT_VERSION) release
 
 release: build
 	@echo "[creating release: $(PRODUCT)]"
-	$(eval BOARD := $(shell echo $(PRODUCT) | cut -d'_' -f2))
+	$(eval BOARD := $(shell echo $(PRODUCT) | cut -d'_' -f2-))
+ifeq ($(findstring _oci_,$(PRODUCT)),_oci_)
+	#container images
+	$(eval IMG_EXT := 'tar')
+	cp -a build-$(PRODUCT)/images/rootfs.tar ./release/RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).tar
+else
+	#regular images
+	$(eval IMG_EXT := 'img')
 	cp -a build-$(PRODUCT)/images/sdcard.img ./release/RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img
-	cd ./release && sha256sum RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img >RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img.sha256
+endif
+	cd ./release && sha256sum RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT) >RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT).sha256
 	rm -f ./release/RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip
-	cd ./release && zip --junk-paths ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).img.sha256 ../LICENSE ./updatepkg/$(PRODUCT)/EULA.de ./updatepkg/$(PRODUCT)/EULA.en
+	cd ./release && zip --junk-paths ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT) ./RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).$(IMG_EXT).sha256 ../LICENSE ./updatepkg/$(PRODUCT)/EULA.de ./updatepkg/$(PRODUCT)/EULA.en
 	cd ./release && sha256sum RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip >RaspberryMatic-$(PRODUCT_VERSION)-$(BOARD).zip.sha256
 
 updatePkg:
@@ -83,11 +101,11 @@ distclean: clean-all
 	@rm -rf download
 
 .PHONY: menuconfig
-menuconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)
+menuconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)/.config
 	cd build-$(PRODUCT) && $(MAKE) O=$(shell pwd)/build-$(PRODUCT) -C ../buildroot-$(BUILDROOT_VERSION) BR2_EXTERNAL=../$(BUILDROOT_EXTERNAL) BR2_DL_DIR=$(BR2_DL_DIR) PRODUCT=$(PRODUCT) PRODUCT_VERSION=$(PRODUCT_VERSION) menuconfig
 
 .PHONY: xconfig
-xconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)
+xconfig: buildroot-$(BUILDROOT_VERSION) build-$(PRODUCT)/.config
 	cd build-$(PRODUCT) && $(MAKE) O=$(shell pwd)/build-$(PRODUCT) -C ../buildroot-$(BUILDROOT_VERSION) BR2_EXTERNAL=../$(BUILDROOT_EXTERNAL) BR2_DL_DIR=$(BR2_DL_DIR) PRODUCT=$(PRODUCT) PRODUCT_VERSION=$(PRODUCT_VERSION) xconfig
 
 .PHONY: savedefconfig

buildroot-external/board/oci/Dockerfile

@@ -0,0 +1,10 @@
+FROM scratch
+
+ARG TARGETARCH
+ARG tar_prefix=rootfs
+
+ADD $tar_prefix$TARGETARCH.tar /
+
+CMD ["/sbin/init"]
+VOLUME /usr/local
+EXPOSE 80 22 161 443 1900 1901 1902 5987 1999 2000 2001 2002 2010 8088 8181 9099 9292 9293 10000 23272 41999 42000 42001 42010 43438 43439 48181 48899 49292 49880

buildroot-external/board/oci/post-build.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# create VERSION file
+echo "VERSION=${PRODUCT_VERSION}" >${TARGET_DIR}/VERSION
+echo "PRODUCT=${PRODUCT}" >>${TARGET_DIR}/VERSION
+echo "PLATFORM=oci" >>${TARGET_DIR}/VERSION
+
+# fix some permissions
+[ -e ${TARGET_DIR}/etc/monitrc ] && chmod 600 ${TARGET_DIR}/etc/monitrc
+
+# remove unnecessary stuff from TARGET_DIR
+rm -f ${TARGET_DIR}/etc/init.d/S01InitZRAMSwap
+rm -f ${TARGET_DIR}/etc/init.d/S03InitURandom
+rm -f ${TARGET_DIR}/etc/init.d/S21rngd
+rm -f ${TARGET_DIR}/etc/init.d/S31bluetooth
+#rm -f ${TARGET_DIR}/etc/init.d/S40network
+rm -f ${TARGET_DIR}/etc/init.d/S49chrony
+rm -f ${TARGET_DIR}/bin/dhcp.script
+
+# link VERSION in /boot on rootfs
+mkdir -p ${TARGET_DIR}/boot
+ln -sf ../VERSION ${TARGET_DIR}/boot/VERSION

buildroot-external/board/oci/post-image.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+BOARD_DIR="$(dirname $0)"
+BOARD_NAME="$(basename ${BOARD_DIR})"
+
+# define DOCKER_ARCH on the BR2_CONFIG setup
+if grep -Eq "^BR2_x86_64=y$" ${BR2_CONFIG}; then
+  DOCKER_ARCH=amd64
+elif grep -Eq "^BR2_i386=y$" ${BR2_CONFIG}; then
+  DOCKER_ARCH=i386
+elif grep -Eq "^BR2_aarch64=y$" ${BR2_CONFIG}; then
+  DOCKER_ARCH=arm64
+elif grep -Eq "^BR2_arm=y$" ${BR2_CONFIG}; then
+  DOCKER_ARCH=arm
+else
+  echo "Unknown architecture"
+  exit 1
+fi
+
+# make sure a factory reset is performed upon fresh start
+touch "${TARGET_DIR}/usr/local/.doFactoryReset"
+
+#Create docker image
+docker build --file="${BOARD_DIR}/Dockerfile" --build-arg=tar_prefix=rootfs --tag=raspberrymatic:${DOCKER_ARCH}-latest ${BINARIES_DIR}
+
+exit $?

buildroot-external/configs/raspmatic_oci_amd64_defconfig

@@ -0,0 +1,111 @@
+BR2_x86_64=y
+BR2_DL_DIR="$(TOPDIR)/../download"
+BR2_KERNEL_MIRROR="https://www.kernel.org/pub"
+BR2_GNU_MIRROR="http://ftp.gnu.org/pub/gnu"
+BR2_OPTIMIZE_2=y
+BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_EQ3_PATH)/patches"
+# BR2_COMPILER_PARANOID_UNSAFE_PATH is not set
+BR2_TOOLCHAIN_BUILDROOT_GLIBC=y
+BR2_KERNEL_HEADERS_4_4=y
+BR2_PACKAGE_GLIBC_UTILS=y
+BR2_TOOLCHAIN_BUILDROOT_CXX=y
+BR2_ROOTFS_LIB_DIR="lib"
+BR2_ROOTFS_LIB32_DIR="lib32"
+BR2_TARGET_GENERIC_HOSTNAME=""
+BR2_TARGET_GENERIC_ISSUE="Welcome to RaspberryMatic"
+BR2_TARGET_GENERIC_PASSWD_SHA512=y
+BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y
+# BR2_TARGET_GENERIC_GETTY is not set
+# BR2_TARGET_GENERIC_REMOUNT_ROOTFS_RW is not set
+# BR2_ENABLE_LOCALE_PURGE is not set
+BR2_TARGET_TZ_INFO=y
+BR2_TARGET_LOCALTIME="Europe/Berlin"
+BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_EQ3_PATH)/overlay/base $(BR2_EXTERNAL_EQ3_PATH)/overlay/base-raspmatic $(BR2_EXTERNAL_EQ3_PATH)/overlay/WebUI $(BR2_EXTERNAL_EQ3_PATH)/overlay/WebUI-raspmatic $(BR2_EXTERNAL_EQ3_PATH)/overlay/RFD $(BR2_EXTERNAL_EQ3_PATH)/overlay/base-raspmatic_oci"
+BR2_ROOTFS_POST_BUILD_SCRIPT="$(BR2_EXTERNAL_EQ3_PATH)/board/oci/post-build.sh"
+BR2_ROOTFS_POST_IMAGE_SCRIPT="$(BR2_EXTERNAL_EQ3_PATH)/board/oci/post-image.sh"
+BR2_PACKAGE_BUSYBOX_CONFIG="$(BR2_EXTERNAL_EQ3_PATH)/Busybox.config"
+BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y
+BR2_PACKAGE_STRACE=y
+BR2_PACKAGE_JQ=y
+BR2_PACKAGE_DOSFSTOOLS=y
+BR2_PACKAGE_DOSFSTOOLS_FATLABEL=y
+BR2_PACKAGE_DOSFSTOOLS_FSCK_FAT=y
+BR2_PACKAGE_DOSFSTOOLS_MKFS_FAT=y
+BR2_PACKAGE_E2FSPROGS=y
+BR2_PACKAGE_E2FSPROGS_E4DEFRAG=y
+BR2_PACKAGE_E2FSPROGS_RESIZE2FS=y
+BR2_PACKAGE_EXFAT=y
+BR2_PACKAGE_EXFAT_UTILS=y
+BR2_PACKAGE_F2FS_TOOLS=y
+BR2_PACKAGE_NTFS_3G=y
+BR2_PACKAGE_DEJAVU=y
+BR2_PACKAGE_LIBERATION=y
+BR2_PACKAGE_PSPLASH=y
+BR2_PACKAGE_PSPLASH_PATCH="$(BR2_EXTERNAL_EQ3_PATH)/patches/psplash/raspmatic"
+BR2_PACKAGE_I2C_TOOLS=y
+BR2_PACKAGE_RNG_TOOLS=y
+BR2_PACKAGE_NODEJS=y
+BR2_PACKAGE_NODEJS_NPM=y
+BR2_PACKAGE_NODEJS_MODULES_ADDITIONAL="async accepts mc charenc encodeurl inherits moment range-parser unorm adm-zip compress-commons end-of-stream ipaddr.js moment-timezone raw-body unpipe archiver concat-map escape-html is-buffer ms readable-stream url archiver-utils content-disposition etag isarray negotiator remove-trailing-separator util-deprecate array-flatten content-type express jmespath node-dir safe-buffer utils-merge async cookie finalhandler jsonfile node-localstorage sax uuid aws-sdk cookie-signature forwarded lazystream normalize-path send vary balanced-match core-util-is fresh lodash on-finished serve-static walkdir base64-js crc fs-extra loop-protect once setprototypeof winreg big-integer crc32-stream fs.realpath md5 parseurl slide wrappy bl cron glob media-typer path-is-absolute statuses write-file-atomic body-parser crypt graceful-fs merge-descriptors path-to-regexp string_decoder ws brace-expansion crypto-browserify http-errors methods process-nextick-args suncalc xml2js buffer debug iconv-lite mime proxy-addr tar-stream xmlbuilder buffer-crc32 depd ieee754 mime-db punycode type-is xtend buffer-shims destroy imurmurhash mime-types qs ultron zip-stream bytes ee-first inflight minimatch querystring universalify"
+BR2_PACKAGE_CA_CERTIFICATES=y
+BR2_PACKAGE_LIBOPENSSL_BIN=y
+BR2_PACKAGE_FONTCONFIG=y
+BR2_PACKAGE_LIBUSB=y
+BR2_PACKAGE_LIBUSB_COMPAT=y
+BR2_PACKAGE_LIBCURL=y
+BR2_PACKAGE_LIBCURL_CURL=y
+BR2_PACKAGE_LIBDNET=y
+BR2_PACKAGE_LIBNETFILTER_CONNTRACK=y
+BR2_PACKAGE_MSMTP=y
+BR2_PACKAGE_AVAHI=y
+BR2_PACKAGE_CHRONY=y
+BR2_PACKAGE_ETHTOOL=y
+BR2_PACKAGE_IPTABLES=y
+BR2_PACKAGE_IW=y
+BR2_PACKAGE_LIGHTTPD=y
+BR2_PACKAGE_LIGHTTPD_OPENSSL=y
+BR2_PACKAGE_LIGHTTPD_ZLIB=y
+BR2_PACKAGE_LIGHTTPD_PCRE=y
+BR2_PACKAGE_NETSNMP=y
+BR2_PACKAGE_NETSNMP_WITH_MIB_MODULES="host ucd-snmp/dlmod ucd-snmp/diskio ucd-snmp/extensible ucd-snmp/pass mibII/mta_sendmail etherlike-mib/dot3StatsTable disman/event-mib agentx"
+BR2_PACKAGE_NTP=y
+# BR2_PACKAGE_NTP_NTPD is not set
+BR2_PACKAGE_NTP_NTPDATE=y
+BR2_PACKAGE_OPENSSH=y
+BR2_PACKAGE_OPENVPN=y
+BR2_PACKAGE_RSYNC=y
+BR2_PACKAGE_SER2NET=y
+BR2_PACKAGE_SOCAT=y
+BR2_PACKAGE_TCPDUMP=y
+BR2_PACKAGE_WGET=y
+BR2_PACKAGE_WIREGUARD_TOOLS=y
+BR2_PACKAGE_XINETD=y
+BR2_PACKAGE_FILE=y
+BR2_PACKAGE_LOCKFILE_PROGS=y
+BR2_PACKAGE_LOGROTATE=y
+BR2_PACKAGE_IRQBALANCE=y
+BR2_PACKAGE_MONIT=y
+BR2_PACKAGE_NUT=y
+BR2_PACKAGE_TAR=y
+BR2_PACKAGE_UTIL_LINUX_BINARIES=y
+BR2_PACKAGE_UTIL_LINUX_LOSETUP=y
+BR2_PACKAGE_UTIL_LINUX_RFKILL=y
+BR2_PACKAGE_UTIL_LINUX_ZRAMCTL=y
+BR2_PACKAGE_HOST_DOSFSTOOLS=y
+BR2_PACKAGE_HOST_E2FSPROGS=y
+BR2_PACKAGE_MULTILIB32=y
+BR2_PACKAGE_MULTILIB32_DEFCONFIG="multilib32_i686_defconfig"
+BR2_PACKAGE_OCCU=y
+BR2_PACKAGE_OCCU_RF_PROTOCOL_HM_HMIP=y
+BR2_PACKAGE_OCCU_WEBUI=y
+BR2_PACKAGE_OCCU_WEBUI_REGAHSS_BETA=y
+BR2_PACKAGE_OCCU_HMSERVER=y
+BR2_PACKAGE_CLOUDMATIC=y
+BR2_PACKAGE_HMLANGW=y
+BR2_PACKAGE_DETECT_RADIO_MODULE=y
+BR2_PACKAGE_DAEMONIZE=y
+BR2_PACKAGE_SSDPD=y
+BR2_PACKAGE_EQ3CONFIGD=y
+BR2_PACKAGE_NEOSERVER=y
+BR2_PACKAGE_RNDADDTOENTCNT=y
+BR2_PACKAGE_JAVA_AZUL=y