diff --git a/.github/data/datafile.ini b/.github/data/datafile.ini new file mode 100644 index 0000000..788ed62 --- /dev/null +++ b/.github/data/datafile.ini @@ -0,0 +1,4 @@ + + +VM_NAME=Base-Builder + diff --git a/.github/tpl/README.tpl.md b/.github/tpl/README.tpl.md new file mode 100644 index 0000000..ed71856 --- /dev/null +++ b/.github/tpl/README.tpl.md @@ -0,0 +1,11 @@ +# Run GitHub CI in {{VM_NAME}} ![Test](https://github.com/{{GITHUB_REPOSITORY}}/workflows/Test/badge.svg) + +Use this action to run your CI in {{VM_NAME}}. + +The github workflow only supports Ubuntu, Windows and MacOS. But what if you need to use {{VM_NAME}}? + +This action is to support {{VM_NAME}}. + + + + diff --git a/.github/tpl/build.tpl.yml b/.github/tpl/build.tpl.yml new file mode 100644 index 0000000..a4da5fc --- /dev/null +++ b/.github/tpl/build.tpl.yml @@ -0,0 +1,62 @@ +name: "Build" +on: + push: + branches: + - '*' + paths: + - '*.sh' + - '.github/workflows/build.yml' + - '*.txt' + - 'conf/*' + - 'hooks/*' + pull_request: + branches: + - '*' + paths: + - '*.sh' + - '.github/workflows/build.yml' + - '*.txt' + - 'conf/*' + - 'hooks/*' + workflow_dispatch: + + release: + types: [ created ] + + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + +jobs: + build: + strategy: + matrix: + os: [{{ALL_RELEASES}}] + runs-on: ubuntu-22.04 + env: + SEC_VBOX : ${{ secrets.SEC_VBOX }} + steps: + - uses: actions/checkout@v4 + - name: Creating Web console + uses: vmactions/cf-tunnel@v0 + id: tunnel + with: + protocol: http + port: 8000 + - run: | + bash build.sh conf/{{VM_OS_NAME}}-${{ matrix.os }}.conf + - name: Upload Release Asset + if: "contains(github.ref, 'refs/tags/')" + id: upload-release-asset + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + files: | + {{VM_OS_NAME}}-${{ matrix.os }}.qcow2.xz + {{VM_OS_NAME}}-${{ matrix.os }}-id_rsa.pub + {{VM_OS_NAME}}-${{ matrix.os }}-host.id_rsa + + diff --git a/.github/tpl/generate.tpl.yml b/.github/tpl/generate.tpl.yml new file mode 100644 index 0000000..0fca38b --- /dev/null +++ b/.github/tpl/generate.tpl.yml @@ -0,0 +1,93 @@ +name: "Step 2, Generate the project from Base builder" +on: + schedule: + - cron: '10 1 * * *' + workflow_dispatch: + push: + branches: + - 'main' + paths: + - 'conf/*' + - '.github/data/*' + - '.github/workflows/generate.yml' + + +jobs: + generate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.VM_TOKEN }} + fetch-depth: '2' + - name: Get latest release + id: get-latest-release + uses: InsonusK/get-latest-release@v1.0.1 + with: + myToken: ${{ github.token }} + view_top: 1 + + - name: Preparing env variables + run: | + git switch main || (git fetch --all && git checkout -b main origin/main) + . conf/all.release.conf + echo "ALL_RELEASES=$ALL_RELEASES" >> $GITHUB_ENV + oldHash="$(cat .github/workflows/build.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)" + echo "OLD_HASH=${oldHash}" >> $GITHUB_ENV + git clone https://github.com/vmactions/base-builder.git + mkdir -p .github/tpl + cat base-builder/.github/tpl/README.tpl.md >.github/tpl/README.tpl.md + + - name: Generate files + uses: vmactions/render@v0.0.1 + with: + datafile: .github/data/datafile.ini + files: | + base-builder/.github/tpl/build.tpl.yml : .github/workflows/build.yml + base-builder/.github/tpl/manual.tpl.yml : .github/workflows/manual.yml + base-builder/.github/FUNDING.yml : .github/FUNDING.yml + base-builder/.github/workflows/readme.yml : .github/workflows/readme.yml + base-builder/build.sh : build.sh + + - name: Check modifications + run: | + #if only hash id in build.yml changes, skip + currentHash="$(cat .github/workflows/build.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)" + echo "Current hash: $currentHash" + lastHash="$(git rev-parse 'HEAD~1')" + echo "Last hash: $lastHash" + oldHash="$lastHash" + echo "OLD_HASH: $OLD_HASH" + echo "Old hash using last hash: $oldHash" + git diff + cp .github/workflows/build.yml build.yml.back + sed -i "s/$currentHash/$oldHash/g" .github/workflows/build.yml + if git diff --quiet; then + echo "no changes" + else + #has changes: + rm -f build.yml.back + rm -rf base-builder + #commit all changes and get new hash + git config user.email "ghactions@vmactions.org" + git config user.name "Generate.yml" + git add . + git commit -m "Generated from base-builder" + newHash="$(git rev-parse main)" + echo "New hash: $newHash" + #modify build.yml + sed -i "s/$oldHash/$newHash/g" .github/workflows/build.yml + fi + - uses: EndBug/add-and-commit@v9 + with: + message: "Update from base vm" + pull: '--rebase --autostash ' + add: | + .github/workflows/readme.yml + .github/workflows/manual.yml + .github/workflows/build.yml + .github/tpl/README.tpl.md + .github/FUNDING.yml + build.sh + + diff --git a/.github/tpl/manual.tpl.yml b/.github/tpl/manual.tpl.yml new file mode 100644 index 0000000..8c7af76 --- /dev/null +++ b/.github/tpl/manual.tpl.yml @@ -0,0 +1,45 @@ +name: "Manual" +on: + workflow_dispatch: + + +jobs: + build: + runs-on: ubuntu-22.04 + env: + DEBUG: 1 + VM_OS_NAME: {{VM_NAME}} + NGROK_TOKEN : ${{ secrets.NGROK_TOKEN }} + SEC_VBOX : ${{ secrets.SEC_VBOX }} + steps: + - uses: actions/checkout@v4 + - name: Creating web console + uses: vmactions/cf-tunnel@v0 + id: tunnel + with: + protocol: http + port: 8000 + - name: Run vnc tunnel + id: test + uses: vmactions/ngrok-tunnel@v0.0.1 + with: + protocol: tcp + port: 5900 + - name: Sleep + run: | + for i in $(seq 1 10) ; do echo $i; sleep 1; done + latest=$(cat $(ls conf/{{VM_NAME}}-*.conf | tail -1) | grep VM_RELEASE | cut -d = -f 2) + echo "please loging to ssh below, and run:" + echo "=============================" + echo "bash build.sh conf/{{VM_NAME}}-$latest.conf" + echo "=============================" + touch /tmp/keepalive + + - uses: neilpang/debugger-action@master + + + + + + + diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml new file mode 100644 index 0000000..46c4538 --- /dev/null +++ b/.github/workflows/readme.yml @@ -0,0 +1,51 @@ +name: "Update Readme" +on: + workflow_dispatch: + push: + branches: + - 'main' + paths: + - '.github/tpl/*' + - '.github/data/*' + - '.github/workflows/readme.yml' + + release: + types: [ published ] + + +jobs: + readme: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Get latest release + id: get-latest-release + uses: InsonusK/get-latest-release@v1.0.1 + with: + myToken: ${{ github.token }} + view_top: 1 + + - name: Using main branch + run: | + git switch main || (git fetch --all && git checkout -b main origin/main) + . conf/default.release.conf + echo "DEFAULT_RELEASE=$DEFAULT_RELEASE" >> $GITHUB_ENV + echo "LATEST_MAJOR=$(echo ${{ steps.get-latest-release.outputs.tag_name }} | cut -d . -f 1)" >> $GITHUB_ENV + echo "LATEST_TAG=${{ steps.get-latest-release.outputs.tag_name }}" >> $GITHUB_ENV + echo "ALL_RELEASES=$(ls conf/ | grep -v default | sed 's/.conf//g' | tr '\n' ',' | sed "s/,\$//" | sed 's/,/, /g')" >> $GITHUB_ENV + + - name: Update the readme.md + uses: vmactions/render@v0.0.1 + with: + datafile: .github/data/datafile.ini + files: | + .github/tpl/README.tpl.md : README.md + + - uses: EndBug/add-and-commit@v9 + with: + message: "Update version to ${{ steps.get-latest-release.outputs.tag_name }}" + pull: '--rebase --autostash ' + add: | + README.md + diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 0000000..359f239 --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: vmactions +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..2424f6f --- /dev/null +++ b/build.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash + +set -e + + +_conf="$1" + +if [ -z "$_conf" ] ; then + echo "Please give the conf file" + exit 1 +fi + + +. "$_conf" + + +############################################################## +osname="$VM_OS_NAME" +ostype="$VM_OS_TYPE" +sshport=$VM_SSH_PORT + + +opts="$VM_OPTS" + +vboxlink="${SEC_VBOX:-$VM_VBOX_LINK}" + + +vmsh="$VM_VBOX" + + +export VM_OS_NAME +export VM_RELEASE +export VM_OCR +export VM_DISK + + +############################################################## + + +waitForText() { + _text="$1" + $vmsh waitForText $osname "$_text" +} + +#keys splitted by ; +#eg: enter +#eg: down; enter +#eg: down; up; tab; enter + + +inputKeys() { + $vmsh input $osname "$1" +} + + + +if [ ! -e "$vmsh" ] ; then + echo "Downloading $vboxlink" + wget -O "$vmsh" "$vboxlink" +fi + +chmod +x "$vmsh" + + + +$vmsh startWeb $osname + + +$vmsh setup + +if ! $vmsh clearVM $osname; then + echo "vm does not exists" +fi + + +if [ "$VM_ISO_LINK" ]; then + $vmsh createVM $VM_ISO_LINK $osname $ostype $sshport + + sleep 2 + + if [ -e "hooks/installOpts.sh" ]; then + echo "hooks/installOpts.sh" + cat "hooks/installOpts.sh" + . "hooks/installOpts.sh" + else + $vmsh processOpts $osname "$opts" + + echo "sleep 60 seconds. just wait" + sleep 60 + + if $vmsh isRunning $osname; then + if ! $vmsh shutdownVM $osname; then + echo "shutdown error" + fi + if ! $vmsh destroyVM $osname; then + echo "destroyVM error" + fi + fi + fi + + while $vmsh isRunning $osname; do + sleep 5 + done + +elif [ "$VM_VHD_LINK" ]; then + if [ ! -e "$osname.qcow2.xz" ]; then + $vmsh download "$VM_VHD_LINK" $osname.qcow2.xz + fi + + if [ ! -e "$osname.qcow2" ]; then + xz -d -T 0 --verbose "$osname.qcow2.xz" + fi + + $vmsh createVMFromVHD $osname $ostype $sshport + + sleep 5 + +else + echo "no VM_ISO_LINK or VM_VHD_LINK, can not build." + exit 1 +fi + +$vmsh startVM $osname + +sleep 2 + + +############################################### + +if [ -e "hooks/waitForLoginTag.sh" ]; then + echo "hooks/waitForLoginTag.sh" + cat "hooks/waitForLoginTag.sh" + . "hooks/waitForLoginTag.sh" +else + waitForText "$VM_LOGIN_TAG" +fi + +sleep 3 + +inputKeys "string root; enter; sleep 1;" +if [ "$VM_ROOT_PASSWORD" ]; then + inputKeys "string $VM_ROOT_PASSWORD ; enter" +fi +inputKeys "enter" +sleep 2 + + +if [ ! -e ~/.ssh/id_rsa ] ; then + ssh-keygen -f ~/.ssh/id_rsa -q -N "" +fi + +cat enablessh.txt >enablessh.local + + +#add ssh key twice, to avoid bugs. +echo "echo '$(base64 -w 0 ~/.ssh/id_rsa.pub)' | openssl base64 -d >>~/.ssh/authorized_keys" >>enablessh.local +echo "" >>enablessh.local + +echo "echo '$(cat ~/.ssh/id_rsa.pub)' >>~/.ssh/authorized_keys" >>enablessh.local +echo "" >>enablessh.local + + +echo >>enablessh.local +echo "chmod 600 ~/.ssh/authorized_keys">>enablessh.local +echo "exit">>enablessh.local +echo >>enablessh.local + + +$vmsh inputFile $osname enablessh.local + + +############################################################### + +$vmsh addSSHHost $osname + + +ssh $osname sh <.ssh/config + +echo "Host host" >>.ssh/config +echo " HostName 192.168.122.1" >>.ssh/config +echo " User runner" >>.ssh/config +echo " ServerAliveInterval 1" >>.ssh/config + +EOF + + +############################################################### + + +if [ -e "hooks/postBuild.sh" ]; then + echo "hooks/postBuild.sh" + cat "hooks/postBuild.sh" + ssh $osname sh<"hooks/postBuild.sh" +fi + + +ssh $osname 'cat ~/.ssh/id_rsa.pub' >$osname-$VM_RELEASE-id_rsa.pub + + +if [ "$VM_PRE_INSTALL_PKGS" ]; then + echo "$VM_INSTALL_CMD $VM_PRE_INSTALL_PKGS" + ssh $osname sh <<<"$VM_INSTALL_CMD $VM_PRE_INSTALL_PKGS" +fi + + +#upload reboot.sh +if [ -e "hooks/reboot.sh" ]; then + echo "hooks/reboot.sh" + cat "hooks/reboot.sh" + scp hooks/reboot.sh $osname:/reboot.sh +else + ssh "$osname" "cat - >/reboot.sh" <$osname.rebooted + +END + +EOF +fi + + +#set cronjob +ssh "$osname" sh </dev/null; then +crontab -l | { cat; echo "* * * * * /reboot.sh"; } | crontab -- +else +crontab -l | { cat; echo "@reboot /reboot.sh"; } | crontab - +fi +crontab -l + +EOF + + +ssh $osname "$VM_SHUTDOWN_CMD" + +sleep 30 + +############################################################### + +if $vmsh isRunning $osname; then + if ! $vmsh shutdownVM $osname; then + echo "shutdown error" + fi +fi + +while $vmsh isRunning $osname; do + sleep 5 +done + + +############################################################## + + + + +ova="$osname-$VM_RELEASE.qcow2" + + +echo "Exporting $ova" +$vmsh exportOVA $osname "$ova" + +cp ~/.ssh/id_rsa $osname-$VM_RELEASE-host.id_rsa + + +ls -lah + + +############################################################## + +echo "Checking the packages: $VM_RSYNC_PKG $VM_SSHFS_PKG" + +if [ -z "$VM_RSYNC_PKG$VM_SSHFS_PKG" ]; then + echo "skip" +else + $vmsh addSSHAuthorizedKeys $osname-$VM_RELEASE-id_rsa.pub + $vmsh startVM $osname + $vmsh waitForVMReady $osname + if [ "$VM_RSYNC_PKG" ]; then + ssh $osname sh <<<"$VM_INSTALL_CMD $VM_RSYNC_PKG" + fi + if [ "$VM_SSHFS_PKG" ]; then + ssh $osname sh <<<"$VM_INSTALL_CMD $VM_SSHFS_PKG" + fi +fi + +