Use devcontainer in build (#142)

* WIP * WIP magic * adjust scoper config * Resolve dependency hell * Make something * add build step for the nextcloud builder * add the checkout step * login to registry * use devcontainer as build-runner2 * skip the postCreate stuff when in CI * change default shell for docker build * extract some functions from the script to features * run poststartcommands in ci * fix filename * fix script for non vscode environments * remove extra install scoper step * add fail fast on install * use bash as default shell * make apt less noisy * try another fail * remove trap in distclean * add memory.ini to top * give the shell job control * just use bash * Scope dependencies (#138) Scope dependencies --------- Co-authored-by: Matthias Simonis <[email protected]> * Fix if the file size is zero, the app evaluates the file as too large to scan (cherry picked from commit c37472d) * WIP * WIP magic * adjust scoper config * Resolve dependency hell * Make something * add fail fast on install * exit early in the install script * fix stuff * add composer path to github path * split postCreate and add to path * remove the scoper installation (done in the postCreateCommand) * add container user vscode * remove vscode user * add docker ps * replace hostname in ci * add debug stuff * use static ip * show json ps * try connect the builder container to the compose network * fix name * move connect to after installation * fix hostname * add hostname to trusted_domains array * fix smtp hostname * replace quote types * remove debugging stuff --------- Co-authored-by: Lennart Dohmann <[email protected]> Co-authored-by: PT-ATA No One <[email protected]> (cherry picked from commit 06d33a6)
GDATASoftwareAG · Oct 22, 2024 · 37bc604 · 37bc604
1 parent 9289eba
commit 37bc604
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 66 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -31,6 +31,10 @@
 	"features": {
 		"ghcr.io/devcontainers/features/node:1": {},
 		"ghcr.io/edouard-lopez/devcontainer-features/bats:0": {},
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/devcontainers/features/github-cli:1": {},
+		"ghcr.io/wxw-matt/devcontainer-features/apt:latest": {
+			"packages": "bash-completion vim iputils-ping telnet"
+		}
 	}
 }
diff --git a/.devcontainer/postCreateCommands.sh b/.devcontainer/postCreateCommands.sh
@@ -1,13 +1,21 @@
 #!/bin/bash
 
-sudo apt-get update 
-sudo apt-get install -y bash-completion vim iputils-ping telnet
+echo "setup php-scoper"
+composer global require humbug/php-scoper
+$(composer config home)/vendor/bin/php-scoper completion bash >> $HOME.bash_completion
+echo "export PATH=$(composer config home)/vendor/bin/:\$PATH" >> $HOME/.bashrc
+export PATH=$(composer config home)/vendor/bin/:$PATH
+
+sudo cp memory.ini /usr/local/etc/php/conf.d/memory.ini
+
+if [[ "$IS_CI" == "true" ]]; then
+    exit 0
+fi
+
 sudo bash -c "docker completion bash > /usr/share/bash-completion/completions/docker"
 sudo bash -c "composer completion bash > /usr/share/bash-completion/completions/composer"
 sudo bash -c "npm completion > /usr/share/bash-completion/completions/npm"
 sudo cp xdebug.local.ini /usr/local/etc/php/conf.d/xdebug.ini
-sudo cp memory.ini /usr/local/etc/php/conf.d/memory.ini
-sudo curl -sS https://webi.sh/gh | sh
 
 echo ". /usr/share/bash-completion/bash_completion" >> /home/vscode/.bashrc
 
@@ -21,7 +29,4 @@ git clone --depth 1 --recurse-submodules --single-branch --branch v$NEXTCLOUD_VE
 cd nextcloud-server
 git submodule update --init
 cd -
-composer global require humbug/php-scoper
-/home/vscode/.composer/vendor/bin/php-scoper completion bash >> /home/vscode/.bash_completion
-echo 'export PATH=/home/vscode/.composer/vendor/bin/:$PATH' >>~/.bashrc
 ./install.sh
diff --git a/.github/workflows/release-app.yml b/.github/workflows/release-app.yml
@@ -1,5 +1,8 @@
 name: Build and Test
 
+defaults:
+  run:
+    shell: bash
 on:
   push:
     branches: ["main"]
@@ -17,23 +20,47 @@ jobs:
       - id: get-matrix
         run: echo "nextcloud_versions=$(./get-matrix.sh)" | tee -a "$GITHUB_OUTPUT"
 
+  build-devcontainer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2 
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Pre-build dev container image
+        uses: devcontainers/[email protected]
+        with:
+          imageName: ghcr.io/gdatasoftwareag/nextcloud/builder
+          cacheFrom: ghcr.io/gdatasoftwareag/nextcloud/builder
+          push: always
+
   test:
     runs-on: ubuntu-latest
-    needs: define-matrix
+    container:
+      image: ghcr.io/gdatasoftwareag/nextcloud/builder:latest
+      env:
+        IS_CI: true
+      options: --name nextcloud-antivirus-build-container
+    needs: 
+      - define-matrix
+      - build-devcontainer
     strategy:
       matrix:
         nextcloud_version: ${{fromJson(needs.define-matrix.outputs.nextcloud_versions)}}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
-        name: setup node
-        with:
-          node-version: 20
 
-      - name: Setup BATS
-        uses: mig4/setup-bats@v1
-        with:
-          bats-version: 1.11.0
+      - name: postCreateCommands
+        run: |
+          source .devcontainer/postCreateCommands.sh
+      
+      - name: add composer bin to path
+        run: |
+          echo $(composer config home --global)/vendor/bin >> $GITHUB_PATH
+          echo "/home/vscode/bin/" >> $GITHUB_PATH
 
       - name: replace version in ./appinfo/info.xml
         if: startsWith(github.ref, 'refs/tags/')
@@ -42,11 +69,6 @@ jobs:
           RELEASE_VERSION=${GITHUB_REF#refs/tags/}
           sed -i "s/<version>0.0.0<\/version>/<version>$RELEASE_VERSION<\/version>/g" ./appinfo/info.xml
 
-      - uses: docker-practice/actions-setup-docker@master
-      - uses: shivammathur/setup-php@v2
-        with:
-          php-version: '8.3'
-
       - name: unittests
         env:
           CLIENT_ID: ${{ secrets.VAAS_CLIENT_ID }}
@@ -55,17 +77,16 @@ jobs:
           composer install
           ./vendor/bin/phpunit --bootstrap tests/unittests/bootstrap.php tests/unittests/ --testdox
 
-      - name: install php-scoper
-        run: |
-          composer global require humbug/php-scoper
-          echo $(composer config home --global) >> $GITHUB_PATH
-
       - name: install nextcloud
         env:
           CLIENT_ID: ${{ secrets.VAAS_CLIENT_ID }}
           CLIENT_SECRET: ${{ secrets.VAAS_CLIENT_SECRET }}
         run: ./install.sh ${{ matrix.nextcloud_version }} 1
 
+      - name: add builder into nextcloud network
+        run: |
+          docker network connect nextcloud-gdata-antivirus_nextcloud-network nextcloud-antivirus-build-container
+
       - name: run tests
         id: bats-tests
         env:

diff --git a/.gitignore b/.gitignore
@@ -69,4 +69,5 @@ core.1
 nextcloud-server/
 core-dump.zip
 apache/
-**/vendor/*
+**/vendor/*
+test.sh
diff --git a/Dockerfile.Nextcloud b/Dockerfile.Nextcloud
@@ -3,7 +3,9 @@ ARG INSTALL_XDEBUG=1
 
 FROM nextcloud:${NEXTCLOUD_VERSION}
 
-RUN apt update && apt install -y \
+SHELL [ "/bin/bash", "-c" ]
+
+RUN apt-get -q update && apt-get -q install -y \
     less vim telnet iputils-ping gdb libexpat1-dev libapr1-dev libaprutil1-dev devscripts debmake \
     bison jdupes libbrotli-dev liblua5.4-dev libnghttp2-dev libssl-dev libxml2-dev libcurl4-openssl-dev libjansson-dev
 # RUN curl -L -o /tmp/apache2_2.4.61.orig.tar.gz https://launchpad.net/debian/+archive/primary/+sourcefiles/apache2/2.4.61-1/apache2_2.4.61.orig.tar.gz \

diff --git a/compose-install.yaml b/compose-install.yaml
@@ -21,7 +21,7 @@ services:
       - smtp
     restart: unless-stopped
     networks:
-      - nextcloud-network
+      nextcloud-network:
   smtp:
     image: rnwood/smtp4dev:v3
     container_name: smtp

diff --git a/foobar.json b/foobar.json
diff --git a/install.sh b/install.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 export NEXTCLOUD_VERSION=${1:-29.0.6}
 export INSTALL_XDEBUG=${2:-1}
@@ -12,7 +13,7 @@ setup_nextcloud () {
   docker compose -f compose-install.yaml rm --force --stop --volumes
   docker compose -f compose-install.yaml up --build --quiet-pull --wait -d --force-recreate --renew-anon-volumes --remove-orphans
 
-  docker exec -i nextcloud-container ulimit -c unlimited
+  docker exec -i nextcloud-container bash -c 'ulimit -c unlimited'
   docker exec -i nextcloud-container bash -c 'echo "/tmp/apache2-coredump/core-%e-%s-%u-%g-%p-%t" > /proc/sys/kernel/core_pattern'
 
   until docker exec --user www-data -i nextcloud-container php occ status | grep "installed: false"
@@ -35,6 +36,7 @@ setup_nextcloud () {
   docker exec --user www-data -i nextcloud-container php occ app:disable firstrunwizard
   docker exec --user www-data -i nextcloud-container php occ app:disable weather_status
   docker exec --user www-data -i nextcloud-container php occ config:system:set trusted_domains 2 --value=192.168.5.80
+  docker exec --user www-data -i nextcloud-container php occ config:system:set trusted_domains 3 --value=nextcloud-container
 
   echo "setup nextcloud finished"
 }
@@ -43,7 +45,7 @@ build_app () {
   echo "build app"
   make distclean
   make appstore
-  tar -xf ./build/artifacts/gdatavaas.tar.gz -C ./build/artifacts
+  tar -xf ./build/artifacts/gdatavaas.tar.gz -C ./build/artifacts 
   echo "build app finished"
 }
 
@@ -53,8 +55,10 @@ if [  -z "$CLIENT_ID" ] || [ -z "$CLIENT_SECRET" ]; then
 fi
 
 setup_nextcloud &
+setup_nextcloud_pid=$!
 build_app &
-wait
+wait %2 || exit 1
+wait %1 || exit 1
 
 docker cp ./build/artifacts/gdatavaas nextcloud-container:/var/www/html/apps/
 docker exec -i nextcloud-container chown -R www-data:www-data /var/www/html/apps/gdatavaas

diff --git a/tests/bats/.env-test b/tests/bats/.env-test
@@ -1,3 +1,17 @@
+#!/bin/bash
+
+if [[ "$IS_CI" == "true" ]]; then
+    export HOSTNAME=nextcloud-container
+else
+    export HOSTNAME=127.0.0.1
+fi
+
+if [[ "$IS_CI" == "true" ]]; then
+    export MAIL_HOSTNAME=smtp
+else
+    export MAIL_HOSTNAME=127.0.0.1:8001
+fi
+
 export FOLDER_PREFIX=./tmp/functionality-parallel
 export TESTUSER=testuser
 export TESTUSER_PASSWORD=myfancysecurepassword234

diff --git a/tests/bats/functionality-parallel.bats b/tests/bats/functionality-parallel.bats
@@ -16,7 +16,7 @@ setup_file() {
 
 @test "test admin eicar Upload" {
     EICAR_LENGTH=$(echo $EICAR_STRING | wc -c)
-    RESULT=$(echo $EICAR_STRING | curl -v -X PUT -d"$EICAR_STRING" -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "curl failed")
+    RESULT=$(echo $EICAR_STRING | curl -v -X PUT -d"$EICAR_STRING" -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "curl failed")
 
     if [[ "$RESULT" =~ "curl failed" ]]; then
         echo "debugging stuff"
@@ -38,12 +38,12 @@ setup_file() {
     fi
 
     echo "Actual: $RESULT"
-    curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "file not found"
+    curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.eicar.com.txt || echo "file not found"
     [[ "$RESULT" =~ "Upload cannot be completed." ]]
 }
 
 @test "test admin clean upload" {
-    RESULT=$(echo $CLEAN_STRING | curl -w "%{http_code}" -u admin:admin -T - http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "curl failed")
+    RESULT=$(echo $CLEAN_STRING | curl -w "%{http_code}" -u admin:admin -T - http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "curl failed")
 
     if [[ "$RESULT" =~ "curl failed" ]]; then
         echo "debugging stuff"
@@ -61,36 +61,36 @@ setup_file() {
     fi
 
     echo "Actual: $RESULT"
-    curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "file not found"
+    curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.clean.txt || echo "file not found"
     [[ $RESULT -ge 200 && $RESULT -lt 300 ]]
 }
 
 @test "test admin pup Upload" {
-    RESULT=$(curl --silent -w "%{http_code}" -u admin:admin -T $FOLDER_PREFIX/pup.exe http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.pup.exe)
+    RESULT=$(curl --silent -w "%{http_code}" -u admin:admin -T $FOLDER_PREFIX/pup.exe http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.pup.exe)
     echo "Actual: $RESULT"
-    curl --silent -q -u admin:admin -X DELETE http://127.0.0.1/remote.php/dav/files/admin/functionality-parallel.pup.exe || echo "file not found"
+    curl --silent -q -u admin:admin -X DELETE http://$HOSTNAME/remote.php/dav/files/admin/functionality-parallel.pup.exe || echo "file not found"
     [[ $RESULT -ge 200 && $RESULT -lt 300 ]] 
 }
 
 @test "test testuser eicar Upload" {
-    RESULT=$(echo $EICAR_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt)
+    RESULT=$(echo $EICAR_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt)
     echo "Actual: $RESULT"
     $DOCKER_EXEC_WITH_USER -i nextcloud-container php occ config:app:get gdatavaas clientSecret
-    curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt || echo "file not found"
+    curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.eicar.com.txt || echo "file not found"
     [[ "$RESULT" =~ "Upload cannot be completed." ]]
 }
 
 @test "test testuser clean Upload" {
-    STATUS_CODE=$(echo $CLEAN_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt)
+    STATUS_CODE=$(echo $CLEAN_STRING | curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T - http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt)
     echo "Actual: $RESULT"
-    curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt || echo "file not found"
+    curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.clean.txt || echo "file not found"
     [[ $STATUS_CODE -ge 200 && $STATUS_CODE -lt 300 ]] || exit 1
 }
 
 @test "test testuser pup Upload" {
-    RESULT=$(curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T $FOLDER_PREFIX/pup.exe http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe)
+    RESULT=$(curl --silent -w "%{http_code}" -u $TESTUSER:$TESTUSER_PASSWORD -T $FOLDER_PREFIX/pup.exe http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe)
     echo "Actual: $RESULT"
-    curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://127.0.0.1/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe || echo "file not found"
+    curl --silent -q -u $TESTUSER:$TESTUSER_PASSWORD -X DELETE http://$HOSTNAME/remote.php/dav/files/$TESTUSER/functionality-parallel.pup.exe || echo "file not found"
     [[ $RESULT -ge 200 && $RESULT -lt 300 ]] || exit 1
 }