diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a028b0..9559851 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,9 +14,6 @@ variables: # Prefix created compose services so we can act on them (volumes, network, not container because they are named by container_name), should be same as package.json name # Also consider that all docker relevant prefixes should also contain the CI_COMMIT_REF_SLUG environment variable COMPOSE_PROJECT_NAME: wp-reactjs-multi-starter - # Define cache folders for yarn and cypress so they can be cached - CYPRESS_CACHE_FOLDER: $CI_PROJECT_DIR/.cypress - YARN_CACHE_FOLDER: $CI_PROJECT_DIR/.yarn COMPOSER_HOME: $CI_PROJECT_DIR/.composer # This is set to 1 by your project variables so jobs like E2E and review apps gets activated. Currently I do not know another way to detect that, feel free to contribute... # DOCKER_DAEMON_ALLOW_UP: 1 @@ -30,31 +27,29 @@ variables: GIT_AUTHOR_EMAIL: $GITLAB_USER_EMAIL GIT_COMMITTER_NAME: $GITLAB_USER_NAME GIT_COMMITTER_EMAIL: $GITLAB_USER_EMAIL - -# Cache installation of yarn, cypress and co -cache: &cache # see https://gitlab.com/gitlab-org/gitlab-runner/issues/2838#note_95134689 - key: "$CI_COMMIT_REF_SLUG" - untracked: true - policy: pull # As we have an install job, all next steps do not need to push again - paths: - - .yarn/ - - .cypress/ - - .composer/ - - node_modules/ - - plugins/*/node_modules/* - - plugins/*/vendor/* - - packages/*/node_modules/* - - packages/*/vendor/* + # List of install-dependent files - generally, we built a mechanism to hold a hot cache in our gitlab-ci image + # If you add files to this list AND you are using shared-runners you need to add the glob to `gitlab-ci shared#only.changes`, too + INSTALL_FILES: package.json yarn.lock common/patch-package/* packages/*/composer.* plugins/*/composer.* packages/*/package.json plugins/*/package.json + INSTALL_VENDOR_FOLDERS: "{packages,plugins}/*/{vendor,node_modules,.yarn} node_modules .yarn .cypress" # Install all needed node and composer modules as extendable job .install: before_script: - # Set composer github token to avoid API rate limit (https://getcomposer.org/doc/articles/troubleshooting.md#api-rate-limit-and-oauth-tokens) - - test $PHP_COMPOSER_GITHUB_TOKEN && composer config -g github-oauth.github.com $PHP_COMPOSER_GITHUB_TOKEN + # Copy temporary installation files from Dockerfile gitlab-ci... + - export TMP_CI_PROJECT_DIR=/tmp$(realpath $CI_PROJECT_DIR) + - export TMP_FILES=$(cd $TMP_CI_PROJECT_DIR && eval find $INSTALL_VENDOR_FOLDERS -maxdepth 0 2>/dev/null) + - time for dirs in $TMP_FILES; do ln -s $TMP_CI_PROJECT_DIR/$dirs $dirs; done # Make sure all dependencies are installed correctly - yarn bootstrap # Make sure cypress is installed correctly - test $DOCKER_DAEMON_ALLOW_UP && yarn cypress install + # Recreate our local package symlinks + - >- + for sym in $(find + node_modules/@$COMPOSE_PROJECT_NAME/ + {plugins,packages}/*/node_modules/@$COMPOSE_PROJECT_NAME + {plugins,packages}/*/vendor/$COMPOSE_PROJECT_NAME + -maxdepth 1 -type l 2>/dev/null); do ln -sf "$(realpath $sym | cut -c5-)" "$(dirname $sym)"; done # Run a job only in production branch .only production: @@ -66,15 +61,34 @@ cache: &cache # see https://gitlab.com/gitlab-org/gitlab-runner/issues/2838#note before_script: - "[ ! -f $JOB_PACKAGE_FOLDER/$JOB_PACKAGE_NAME/.publish ] && [ ! $LERNA_CHANGES_FORCE ] && echo $LERNA_SKIP_MESSAGE && exit 0" -# Initial installation so the cache gets filled install: - extends: .install stage: install - cache: - <<: *cache - policy: pull-push + variables: + TEMP_CONTAINER_NAME: $COMPOSE_PROJECT_NAME-install script: - - echo Installation done, make cache for the next jobs... + - echo This container just makes sure the built image is correctly published with newest dependencies. + - test ! $DOCKER_DAEMON_ALLOW_UP && exit 0 + # See `stage-containerize.yml` job for more information about this mechanism + - export INSTALL_FILES=$(ls -t $INSTALL_FILES 2>/dev/null) + # Reset times for cache consistency + - touch -a -m -t 201501010000.00 $INSTALL_FILES + - tar -cvf install.tar $INSTALL_FILES + # Create temporary container where we make sure installing current dependencies + - export DOCKER_CONTAINER_FQN=$CI_REGISTRY_IMAGE/gitlab-ci:$CI_COMMIT_REF_SLUG + - (docker rm -f $TEMP_CONTAINER_NAME || :) + - docker run --name $TEMP_CONTAINER_NAME -it -d $DOCKER_CONTAINER_FQN + - docker cp install.tar $TEMP_CONTAINER_NAME:/tmp/builds/devowlio/wp-reactjs-starter + - docker exec -t $TEMP_CONTAINER_NAME /bin/bash -c "tar -xvf install.tar && yarn bootstrap && yarn cypress install" + - docker commit --author "GitLab CI" --message "Reinstall dependencies" $TEMP_CONTAINER_NAME $DOCKER_CONTAINER_FQN + # Push to registry + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker history $DOCKER_CONTAINER_FQN + - docker push $DOCKER_CONTAINER_FQN + after_script: + # Purge + - docker rm -f $TEMP_CONTAINER_NAME + - export DANGLING_IMAGES=$((docker images -f "dangling=true" | grep ^$CI_REGISTRY_IMAGE/gitlab-ci:$CI_COMMIT_REF_SLUG | awk '{ print $3 }') || :) + - test $DANGLING_IMAGES && docker rmi $DANGLING_IMAGES include: - /devops/.gitlab/stage-containerize.yml diff --git a/devops/.gitlab/stage-containerize.yml b/devops/.gitlab/stage-containerize.yml index f0f812f..fa9ce81 100755 --- a/devops/.gitlab/stage-containerize.yml +++ b/devops/.gitlab/stage-containerize.yml @@ -1,23 +1,59 @@ # Containerize a Dockerfile to the GitLab CI container registry. -# The job name represents the folder in devops/docker/$CI_JOB_NAME/* +# The job name represents the folder in devops/docker/$IMAGE_NAME/* # "only.changes" can not be extendable: https://gitlab.com/gitlab-org/gitlab/issues/8177 .containerize: stage: containerize image: # https://github.com/GoogleContainerTools/kaniko/issues/1002#issuecomment-578856209 # > This issue is only affecting people who are using the debug tag which is built on every commit to master. But still, use an explicit version! - name: gcr.io/kaniko-project/executor:debug-v0.16.0 + name: gcr.io/kaniko-project/executor:debug-v0.22.0 entrypoint: [""] cache: {} script: - - export DOCKER_CONTAINER_FQN=$CI_REGISTRY_IMAGE/$CI_JOB_NAME:$CI_COMMIT_REF_SLUG - - echo $DOCKER_CONTAINER_FQN + # Always create a install.tar archive which contains all files relevant for installation + - export INSTALL_FILES=$(ls -t $INSTALL_FILES 2>/dev/null) + # Reset times for cache consistency + - touch -a -m -t 201501010000.00 $INSTALL_FILES + - tar -cvf install.tar $INSTALL_FILES + # Create image name + - export DOCKER_CONTAINER_FQN=$CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_REF_SLUG + - echo Build as $DOCKER_CONTAINER_FQN - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --cache=true --build-arg GL_CI_WORKDIR=$(realpath $CI_PROJECT_DIR) --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/devops/docker/$CI_JOB_NAME/Dockerfile --destination $DOCKER_CONTAINER_FQN + - /kaniko/executor --cache=true --build-arg GL_CI_WORKDIR=$(realpath $CI_PROJECT_DIR) --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/devops/docker/$IMAGE_NAME/Dockerfile --destination $DOCKER_CONTAINER_FQN -gitlab-ci: +# Build an hot-cache image in a shared-runner environment +gitlab-ci shared: extends: [.containerize] + variables: + IMAGE_NAME: gitlab-ci + before_script: + - "echo ########################" + - echo Building and publishing docker images via GitLab shared runners is very slow. Please checkout this and setup your own runner + - echo https://devowlio.gitbook.io/wp-react-starter/gitlab-integration/predefined-pipeline#disadvantages + - "echo ########################" + except: + variables: + - $DOCKER_DAEMON_ALLOW_UP only: changes: + - package.json + - yarn.lock + - common/patch-package/* + - "{packages,plugins}/*/{composer,package}.*" - devops/.gitlab/stage-containerize.yml - devops/docker/gitlab-ci/* + +# Build an hot-cache image in an own-runner environment +# What's happening here? We are building the image only when changes +# to non-dependency files are done. The dependencies are created and commit +# through the `install` job. +gitlab-ci hosted: + extends: [.containerize] + variables: + IMAGE_NAME: gitlab-ci + only: + changes: + - devops/.gitlab/stage-containerize.yml + - devops/docker/gitlab-ci/* + variables: + - $DOCKER_DAEMON_ALLOW_UP diff --git a/devops/.gitlab/stage-release.yml b/devops/.gitlab/stage-release.yml index 43b1edb..fcafba5 100755 --- a/devops/.gitlab/stage-release.yml +++ b/devops/.gitlab/stage-release.yml @@ -21,6 +21,7 @@ release: docker review start: stage: release dependencies: [collect all artifacts] + needs: [collect all artifacts] variables: COMPOSE_PROJECT_NAME_SUFFIX: -traefik script: @@ -50,6 +51,7 @@ docker review start: on_stop: docker review stop docker review stop: + extends: [.install] stage: release variables: COMPOSE_PROJECT_NAME_SUFFIX: -traefik diff --git a/devops/.gitlab/stage-test.yml b/devops/.gitlab/stage-test.yml index ae6e48e..9c4073e 100755 --- a/devops/.gitlab/stage-test.yml +++ b/devops/.gitlab/stage-test.yml @@ -48,7 +48,7 @@ collect all artifacts: # Start the cypress e2e test .docker e2e cypress: - when: always # Always try the e2e test because it must be stopped + extends: [.install] stage: test script: # Cleanup previous environments diff --git a/devops/docker/gitlab-ci/Dockerfile b/devops/docker/gitlab-ci/Dockerfile index 231dc1f..0466595 100755 --- a/devops/docker/gitlab-ci/Dockerfile +++ b/devops/docker/gitlab-ci/Dockerfile @@ -1,80 +1,23 @@ -FROM php:7.3-cli-stretch +FROM devowliode/wp-react-starter-gitlab-ci:php-7.3-cli-stretch -# Avoid too many progress messages -ENV CI=1 +# Prepare our dependencies and cache +ARG GL_CI_WORKDIR +ENV CYPRESS_CACHE_FOLDER=/tmp$GL_CI_WORKDIR/.cypress +ENV YARN_CACHE_FOLDER=/tmp$GL_CI_WORKDIR/.yarn + +# Set composer github token to avoid API rate limit (https://getcomposer.org/doc/articles/troubleshooting.md#api-rate-limit-and-oauth-tokens) +ARG PHP_COMPOSER_GITHUB_TOKEN +RUN (test $PHP_COMPOSER_GITHUB_TOKEN && \ + composer config -g github-oauth.github.com $PHP_COMPOSER_GITHUB_TOKEN) || : + +# Install our dependencies into our gitlab runner +WORKDIR /tmp$GL_CI_WORKDIR -# Apt and common dependencies -RUN apt-get update && apt-get install -y \ - # Commons - sudo \ - autogen \ - wget \ - zip \ - unzip \ - tar \ - git \ - subversion \ - build-essential \ - apt-utils \ - software-properties-common \ - nasm \ - libjpeg-dev \ - libpng-dev \ - libpng16-16 \ - # Allow simple FTP uploads - lftp \ - wput \ - # Pyhton is needed for Docker-Compose - python \ - python-pip \ - python-dev \ - # Cypress.io dependencies - xvfb \ - libgtk2.0-0 \ - libgtk-3-0 \ - libnotify-dev \ - libgconf-2-4 \ - libnss3 \ - libxss1 \ - libasound2 \ - libxtst6 \ - xauth \ - # libssl-dev is needed for Phar (https://raspberrypi.stackexchange.com/a/36379) - libssl-dev && \ - # PHP dependencies (https://git.io/JvAFa) - docker-php-ext-install -j4 phar json mbstring tokenizer && \ - # xdebug (can be activated with `docker-php-ext-enable xdebug`, not by default because it slows down PHP) - pecl install xdebug && \ - # Composer - curl -sS https://getcomposer.org/installer | php && \ - mv composer.phar /usr/local/bin/composer && \ - chmod +x /usr/local/bin/composer && \ - composer self-update --preview && \ - # Node - curl -sL https://deb.nodesource.com/setup_12.x -o nodesource_setup.sh && \ - bash nodesource_setup.sh && \ - apt-get install -y nodejs && \ - # Yarn - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ - apt-get update && apt-get install -y yarn && \ - # Docker and Docker-Compose (via Python, PiP) - curl -sSL https://get.docker.com/ | sh && \ - pip install docker-compose && \ - # WP CLI - curl -O https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar && \ - php wp-cli.phar --info && \ - chmod +x wp-cli.phar && \ - mv wp-cli.phar /usr/local/bin/wp && \ - # PHP Scoper - curl -O -L https://github.com/humbug/php-scoper/releases/download/0.13.1/php-scoper.phar && \ - chmod +x php-scoper.phar && \ - mv php-scoper.phar /usr/local/bin/php-scoper +COPY install.tar . -RUN npm install -g \ - # Global installations of npm packages - wait-on - # && \ - # composer global require \ - # Global installations of composer packages - # hirak/prestissimo Prestissimo is currently not possible together with Lerna and CI due to a RC (https://git.io/JvPkZ)... wait for Composer v2 +RUN tar -xvf install.tar && \ + yarn bootstrap && \ + yarn cypress install + +# Avoid too many progress messages +ENV CI=1 \ No newline at end of file diff --git a/package.json b/package.json index 89a7ea0..ca9a3d0 100755 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "workspace:compose-files": "find plugins/*/devops packages/*/devops | grep \"^\\(plugins\\|packages\\)/.*/devops/docker-compose/docker-compose\\.\\(yml\\|$COMPOSE_CONTEXT\\)\" | sed -n -e 'H;${x;s/\\n/:.\\//g;s/^ //;p;}' | sed -e 's/:*$//' | sed -e 's/^://' ", "workspace:slugs": "find plugins/* -maxdepth 0 2>/dev/null | sed -n -e 'H;${x;s/\\n/ /g;s/^ //;p;}' | sed 's/plugins\\///g'", "exposedotenv": "env $(cat ./common/.env-default | xargs) env $(cat ./.env 2>/dev/null | xargs)", - "preversion": "WORKSPACE_COMMAND=\"yarn --silent grunt composer:disclaimer\" yarn --silent workspace:concurrently && lerna exec --concurrency 1 -- yarn grunt yarn:disclaimer && git add \"*LICENSE_3RD_PARTY*\"" + "preversion": "(test \"${CI_JOB_NAME}\" != \"semver\" && WORKSPACE_COMMAND=\"yarn --silent grunt composer:disclaimer\" yarn --silent workspace:concurrently && lerna exec --concurrency 1 -- yarn grunt yarn:disclaimer && git add \"*LICENSE_3RD_PARTY*\") || :" }, "lerna": { "npmClient": "yarn",