Merge pull request #16023 from CartoDB/2020-12-17-test-different-build

moving to a new build workflow

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "lib/sql"]
 	path = lib/sql
 	url = git://github.com/CartoDB/cartodb-postgresql.git
+[submodule "private"]
+	path = private
+	url = [email protected]:CartoDB/cartodb-private.git

NEWS.md

@@ -31,6 +31,7 @@ sudo make install
 - Add a default connection timeout of 30 seconds to Carto::Http::Client [#16020](https://github.com/CartoDB/cartodb/pull/16020)
 - Propagate request_id in MessageBroker logs [#16006](https://github.com/CartoDB/cartodb/pull/16006)
 - Don't report Coverband errors to Rollbar [#16021](https://github.com/CartoDB/cartodb/pull/16021)
+- Add private submodule [#16023](https://github.com/CartoDB/cartodb/pull/16023)
 - Maps API client now honors 429 Too Many Requests error [#16025](https://github.com/CartoDB/cartodb/pull/16025)
 - Fix a loop between our logger and rollbar [#16026](https://github.com/CartoDB/cartodb/pull/16026)
 - Make the MessageBroker subscriber PIDFILE check more resilient [#16022](https://github.com/CartoDB/cartodb/pull/16022)

script/ci/cloudbuild-new-build-master.yaml

@@ -0,0 +1,89 @@
+steps:
+
+# Cancel previous job on the same branch
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - 'gcloud builds list --ongoing --filter="buildTriggerId=70299e98-eed0-4fc6-943b-b8a5e0cf2aca AND substitutions.BRANCH_NAME=${BRANCH_NAME} AND id!=${BUILD_ID}" --format="get(ID)" > jobs_to_cancel'
+
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - 'gcloud builds cancel $(cat jobs_to_cancel | xargs) || true'
+
+# Decrypt github key
+- name: gcr.io/cloud-builders/gcloud
+  args:
+  - kms
+  - decrypt
+  - --ciphertext-file=script/ci/.id_github_cartofante.enc
+  - --plaintext-file=/root/.ssh/id_rsa
+  - --location=global
+  - --keyring=keyring-github-ci
+  - --key=github-key
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Set up git with key and domain
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    chmod 600 /root/.ssh/id_rsa
+    cat <<EOF >/root/.ssh/config
+    Hostname github.com
+    IdentityFile /root/.ssh/id_rsa
+    EOF
+    mv script/ci/known_hosts /root/.ssh/known_hosts
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Clean working directory
+- name: alpine
+  args: ['sh', '-c', 'rm -rf .* | true && rm -rf *']
+
+# Checkout repo with submodules
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'sh'
+  args: ['-c', 'git clone [email protected]:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .']
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Copy files to the right directories to run the tests
+- name: ubuntu
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - |
+      cp private/script/ci/* .
+      cp private/Dockerfil* .
+      cp -r private/build_resources/ .
+      cp script/ci/* .
+      cp config/app_config.yml.sample config/app_config.yml
+      cp config/database.yml.sample config/database.yml
+      cp lib/assets/javascripts/cdb/secrets.example.json lib/assets/javascripts/cdb/secrets.json
+
+# Build and push image
+- name: gcr.io/cloud-builders/docker
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - |
+      docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest
+      docker build --build-arg BUNDLE_JOBS=16 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA} --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:latest .
+      docker build -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA}  -f Dockerfile.resque .
+      docker build -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:latest -t gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA}  -f Dockerfile.subscriber .
+      docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:${SHORT_SHA}
+      docker push gcr.io/cartodb-on-gcp-main-artifacts/builder:latest
+      docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:${SHORT_SHA}
+      docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-resque:latest
+      docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:${SHORT_SHA}
+      docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-subscriber:latest
+
+timeout: 1800s

script/ci/cloudbuild-new-pr-pg11.yaml

@@ -0,0 +1,92 @@
+steps:
+
+# Cancel previous job on the same branch
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - 'gcloud builds list --ongoing --filter="buildTriggerId=056221c7-f5ef-4857-9706-3bc1054799a3 AND substitutions.BRANCH_NAME=${BRANCH_NAME} AND id!=${BUILD_ID}" --format="get(ID)" > jobs_to_cancel'
+
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - 'gcloud builds cancel $(cat jobs_to_cancel | xargs) || true'
+
+# Decrypt github key
+- name: gcr.io/cloud-builders/gcloud
+  args:
+  - kms
+  - decrypt
+  - --ciphertext-file=script/ci/.id_github_cartofante.enc
+  - --plaintext-file=/root/.ssh/id_rsa
+  - --location=global
+  - --keyring=keyring-github-ci
+  - --key=github-key
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Set up git with key and domain
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    chmod 600 /root/.ssh/id_rsa
+    cat <<EOF >/root/.ssh/config
+    Hostname github.com
+    IdentityFile /root/.ssh/id_rsa
+    EOF
+    mv script/ci/known_hosts /root/.ssh/known_hosts
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Clean working directory
+- name: alpine
+  args: ['sh', '-c', 'rm -rf .* | true && rm -rf *']
+
+# Checkout repo with submodules
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'sh'
+  args: ['-c', 'git clone [email protected]:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .']
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Copy files to the right directories to run the tests
+- name: ubuntu
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - |
+      cp private/script/ci/* .
+      cp private/Dockerfile .
+      cp -r private/build_resources/ .
+      cp script/ci/* .
+      cp config/app_config.yml.sample config/app_config.yml
+      cp config/database.yml.sample config/database.yml
+      cp lib/assets/javascripts/cdb/secrets.example.json lib/assets/javascripts/cdb/secrets.json
+
+# Build image
+- name: gcr.io/cloud-builders/docker
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - |
+      docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest
+      docker build --build-arg BUNDLE_JOBS=16 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:current --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:latest .
+
+# Run tests
+- name: 'docker/compose:1.22.0'
+  args: ['-f', 'docker-compose-pg11.yml', 'up', '--build', '-d']
+  timeout: 900s
+
+- name: gcr.io/cloud-builders/docker
+  args: ['exec', '-i', 'builder_1', 'bash', '-c', '/cartodb/runParallelTests.sh 24' ]
+  timeout: 2700s
+
+options:
+    machineType: 'N1_HIGHCPU_32'
+timeout: 3600s

script/ci/cloudbuild-new-pr-pg12.yaml

@@ -0,0 +1,99 @@
+steps:
+
+# Cancel previous job on the same branch
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - 'gcloud builds list --ongoing --filter="buildTriggerId=e323286e-7003-4480-ae34-677e597ce4dc AND substitutions.BRANCH_NAME=${BRANCH_NAME} AND id!=${BUILD_ID}" --format="get(ID)" > jobs_to_cancel'
+
+- name: gcr.io/cloud-builders/gcloud
+  entrypoint: /bin/bash
+  args:
+  - '-c'
+  - 'gcloud builds cancel $(cat jobs_to_cancel | xargs) || true'
+
+# Decrypt github key
+- name: gcr.io/cloud-builders/gcloud
+  args:
+  - kms
+  - decrypt
+  - --ciphertext-file=script/ci/.id_github_cartofante.enc
+  - --plaintext-file=/root/.ssh/id_rsa
+  - --location=global
+  - --keyring=keyring-github-ci
+  - --key=github-key
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Set up git with key and domain
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'bash'
+  args:
+  - '-c'
+  - |
+    chmod 600 /root/.ssh/id_rsa
+    cat <<EOF >/root/.ssh/config
+    Hostname github.com
+    IdentityFile /root/.ssh/id_rsa
+    EOF
+    mv script/ci/known_hosts /root/.ssh/known_hosts
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Clean working directory
+- name: alpine
+  args: ['sh', '-c', 'rm -rf .* | true && rm -rf *']
+
+# Checkout repo with submodules
+- name: 'gcr.io/cloud-builders/git'
+  entrypoint: 'sh'
+  args: ['-c', 'git clone [email protected]:CartoDB/cartodb -b "${BRANCH_NAME}" --depth 1 --recursive .']
+  volumes:
+  - name: 'ssh'
+    path: /root/.ssh
+
+# Copy files to the right directories to run the tests
+- name: ubuntu
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - |
+      cp private/script/ci/* .
+      cp private/Dockerfile .
+      cp -r private/build_resources/ .
+      cp script/ci/* .
+      cp config/app_config.yml.sample config/app_config.yml
+      cp config/database.yml.sample config/database.yml
+      cp lib/assets/javascripts/cdb/secrets.example.json lib/assets/javascripts/cdb/secrets.json
+
+# Build image
+- name: gcr.io/cloud-builders/docker
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - |
+      docker pull gcr.io/cartodb-on-gcp-main-artifacts/builder:latest
+      docker build --build-arg BUNDLE_JOBS=16 -t gcr.io/cartodb-on-gcp-main-artifacts/builder:current --cache-from gcr.io/cartodb-on-gcp-main-artifacts/builder:latest .
+
+# Run tests
+- name: 'docker/compose:1.22.0'
+  args: ['-f', 'docker-compose-pg12.yml', 'up', '--build', '-d']
+  timeout: 900s
+
+# Push builder-tests image to registry to use it as cache on next builds
+- name: gcr.io/cloud-builders/docker
+  entrypoint: /bin/bash
+  args:
+    - -c
+    - 'docker push gcr.io/cartodb-on-gcp-main-artifacts/builder-tests:latest'
+
+- name: gcr.io/cloud-builders/docker
+  args: ['exec', '-i', 'builder_1', 'bash', '-c', '/cartodb/runParallelTests.sh 24' ]
+  timeout: 2700s
+
+options:
+    machineType: 'N1_HIGHCPU_32'
+timeout: 3600s

script/ci/cloudbuild-pg11.yaml

@@ -18,7 +18,7 @@ steps:
   args:
   - kms
   - decrypt
-  - --ciphertext-file=script/ci/.gh_helper.enc
+  - --ciphertext-file=script/ci/.id_github_cartofante.enc
   - --plaintext-file=/root/.ssh/id_rsa
   - --location=global
   - --keyring=keyring-github-ci

script/ci/cloudbuild-pg12.yaml

@@ -18,7 +18,7 @@ steps:
   args:
   - kms
   - decrypt
-  - --ciphertext-file=script/ci/.gh_helper.enc
+  - --ciphertext-file=script/ci/.id_github_cartofante.enc
   - --plaintext-file=/root/.ssh/id_rsa
   - --location=global
   - --keyring=keyring-github-ci