From bf2210196b32144f1f89dcf38e4577beb2822d85 Mon Sep 17 00:00:00 2001 From: Andreas Braun Date: Wed, 18 Oct 2023 08:52:34 +0200 Subject: [PATCH] PHPLIB-1260: Revamp evergreen config (#1179) * Split up evergreen config and create single build variant * Upload built extension * Add test variant * Build on all platforms * Run tests on all MongoDB versions * Add remaining tests * Build against latest extension branch * Run no-auth tests * Set PHP_VERSION expansion in test variants * Remove sudo to fix RHEL builds * Clean up legacy variants to account for progress * Move Atlas Tests to new format * Move serverless tests to new format * Move more tests to new format * Fix wrong output in test configuration * Move CSFLE tests to new format * Move ADL tests to new format * Move lowest dependency tests to new format * Define aliases for PR builds * Prevent generated build files from showing diffs in pull requests * Apply recent changes from PHPC config This moves templates and generated files to separate folders and changes the loadbalanced tests to use its own topology instead of "sharded". * Remove unused YAML tags * Supply makeflags for pecl install * Remove irrelevant functions * Clean up variables when running scripts * Skip CSFLE tests for all topologies * Document environment variables for install-composer.sh * Add missing CSFLE tag to no-aws-creds tests --- .evergreen/compile-extension.sh | 42 + .evergreen/config.yml | 1073 +---------------- .evergreen/config/build-task-groups.yml | 32 + .evergreen/config/build-variants.yml | 87 ++ .evergreen/config/functions.yml | 484 ++++++++ .evergreen/config/generate-config.php | 88 ++ .../config/generated/build/build-php-7.4.yml | 44 + .../config/generated/build/build-php-8.0.yml | 44 + .../config/generated/build/build-php-8.1.yml | 44 + .../config/generated/build/build-php-8.2.yml | 44 + .../config/generated/test/csfle-4.2.yml | 42 + .../config/generated/test/csfle-4.4.yml | 42 + .../config/generated/test/csfle-5.0.yml | 42 + .../config/generated/test/csfle-6.0.yml | 42 + .../config/generated/test/csfle-7.0.yml | 42 + .../config/generated/test/csfle-latest.yml | 42 + .../config/generated/test/csfle-rapid.yml | 42 + .../generated/test/load-balanced-5.0.yml | 18 + .../generated/test/load-balanced-6.0.yml | 18 + .../generated/test/load-balanced-7.0.yml | 18 + .../generated/test/load-balanced-latest.yml | 18 + .../generated/test/load-balanced-rapid.yml | 18 + .../config/generated/test/local-3.6.yml | 34 + .../config/generated/test/local-4.0.yml | 34 + .../config/generated/test/local-4.2.yml | 34 + .../config/generated/test/local-4.4.yml | 34 + .../config/generated/test/local-5.0.yml | 34 + .../config/generated/test/local-6.0.yml | 34 + .../config/generated/test/local-7.0.yml | 34 + .../config/generated/test/local-latest.yml | 34 + .../config/generated/test/local-rapid.yml | 34 + .../test/require-api-version-5.0.yml | 30 + .../test/require-api-version-6.0.yml | 30 + .../test/require-api-version-7.0.yml | 30 + .../test/require-api-version-latest.yml | 30 + .../test/require-api-version-rapid.yml | 30 + .../templates/build/build-extension.yml | 43 + .evergreen/config/templates/test/csfle.yml | 41 + .../config/templates/test/load-balanced.yml | 17 + .evergreen/config/templates/test/local.yml | 33 + .../templates/test/require-api-version.yml | 29 + .evergreen/config/test-task-groups.yml | 36 + .evergreen/config/test-tasks.yml | 20 + .evergreen/config/test-variants.yml | 116 ++ .evergreen/install-composer.sh | 41 + .evergreen/install-dependencies.sh | 93 -- .gitattributes | 3 + tests/TestCase.php | 2 +- 48 files changed, 2190 insertions(+), 1106 deletions(-) create mode 100644 .evergreen/compile-extension.sh create mode 100644 .evergreen/config/build-task-groups.yml create mode 100644 .evergreen/config/build-variants.yml create mode 100644 .evergreen/config/functions.yml create mode 100644 .evergreen/config/generate-config.php create mode 100644 .evergreen/config/generated/build/build-php-7.4.yml create mode 100644 .evergreen/config/generated/build/build-php-8.0.yml create mode 100644 .evergreen/config/generated/build/build-php-8.1.yml create mode 100644 .evergreen/config/generated/build/build-php-8.2.yml create mode 100644 .evergreen/config/generated/test/csfle-4.2.yml create mode 100644 .evergreen/config/generated/test/csfle-4.4.yml create mode 100644 .evergreen/config/generated/test/csfle-5.0.yml create mode 100644 .evergreen/config/generated/test/csfle-6.0.yml create mode 100644 .evergreen/config/generated/test/csfle-7.0.yml create mode 100644 .evergreen/config/generated/test/csfle-latest.yml create mode 100644 .evergreen/config/generated/test/csfle-rapid.yml create mode 100644 .evergreen/config/generated/test/load-balanced-5.0.yml create mode 100644 .evergreen/config/generated/test/load-balanced-6.0.yml create mode 100644 .evergreen/config/generated/test/load-balanced-7.0.yml create mode 100644 .evergreen/config/generated/test/load-balanced-latest.yml create mode 100644 .evergreen/config/generated/test/load-balanced-rapid.yml create mode 100644 .evergreen/config/generated/test/local-3.6.yml create mode 100644 .evergreen/config/generated/test/local-4.0.yml create mode 100644 .evergreen/config/generated/test/local-4.2.yml create mode 100644 .evergreen/config/generated/test/local-4.4.yml create mode 100644 .evergreen/config/generated/test/local-5.0.yml create mode 100644 .evergreen/config/generated/test/local-6.0.yml create mode 100644 .evergreen/config/generated/test/local-7.0.yml create mode 100644 .evergreen/config/generated/test/local-latest.yml create mode 100644 .evergreen/config/generated/test/local-rapid.yml create mode 100644 .evergreen/config/generated/test/require-api-version-5.0.yml create mode 100644 .evergreen/config/generated/test/require-api-version-6.0.yml create mode 100644 .evergreen/config/generated/test/require-api-version-7.0.yml create mode 100644 .evergreen/config/generated/test/require-api-version-latest.yml create mode 100644 .evergreen/config/generated/test/require-api-version-rapid.yml create mode 100644 .evergreen/config/templates/build/build-extension.yml create mode 100644 .evergreen/config/templates/test/csfle.yml create mode 100644 .evergreen/config/templates/test/load-balanced.yml create mode 100644 .evergreen/config/templates/test/local.yml create mode 100644 .evergreen/config/templates/test/require-api-version.yml create mode 100644 .evergreen/config/test-task-groups.yml create mode 100644 .evergreen/config/test-tasks.yml create mode 100644 .evergreen/config/test-variants.yml create mode 100644 .evergreen/install-composer.sh diff --git a/.evergreen/compile-extension.sh b/.evergreen/compile-extension.sh new file mode 100644 index 000000000..1d8010994 --- /dev/null +++ b/.evergreen/compile-extension.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -o errexit # Exit the script with error if any of the commands fail + +PATH="$PHP_PATH/bin:$PATH" + +install_extension () +{ + rm -f ${PHP_PATH}/lib/php.ini + + if [ "x${EXTENSION_BRANCH}" != "x" ] || [ "x${EXTENSION_REPO}" != "x" ]; then + CLONE_REPO=${EXTENSION_REPO:-https://github.com/mongodb/mongo-php-driver} + CHECKOUT_BRANCH=${EXTENSION_BRANCH:-master} + + echo "Compiling driver branch ${CHECKOUT_BRANCH} from repository ${CLONE_REPO}" + + mkdir -p /tmp/compile + rm -rf /tmp/compile/mongo-php-driver + git clone ${CLONE_REPO} /tmp/compile/mongo-php-driver + cd /tmp/compile/mongo-php-driver + + git checkout ${CHECKOUT_BRANCH} + git submodule update --init + phpize + ./configure --enable-mongodb-developer-flags + make all -j20 > /dev/null + make install + + cd ${PROJECT_DIRECTORY} + elif [ "${EXTENSION_VERSION}" != "" ]; then + echo "Installing driver version ${EXTENSION_VERSION} from PECL" + MAKEFLAGS=-j20 pecl install -f mongodb-${EXTENSION_VERSION} + else + echo "Installing latest driver version from PECL" + MAKEFLAGS=-j20 pecl install -f mongodb + fi + + cp ${PROJECT_DIRECTORY}/.evergreen/config/php.ini ${PHP_PATH}/lib/php.ini + + php --ri mongodb +} + +install_extension diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9e6027b37..bc92d4798 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1,1031 +1,80 @@ -######################################## -# Evergreen Template for MongoDB Drivers -######################################## - -# When a task that used to pass starts to fail -# Go through all versions that may have been skipped to detect -# when the task started failing +# When a task that used to pass starts to fail, go through all versions that may have been skipped to detect when the +# task started failing. stepback: true -# Mark a failure as a system/bootstrap failure (purple box) rather then a task -# failure by default. +# Mark a failure as a system/bootstrap failure (purple box) rather than a task failure by default. # Actual testing tasks are marked with `type: test` command_type: system -# Protect ourself against rogue test case, or curl gone wild, that runs forever -# Good rule of thumb: the averageish length a task takes, times 5 -# That roughly accounts for variable system performance for various buildvariants -exec_timeout_secs: 1800 # 6 minutes is the longest we'll ever run - -# What to do when evergreen hits the timeout (`post:` tasks are run automatically) -timeout: - - command: shell.exec - params: - script: | - ls -la - -functions: - "fetch source": - # Executes git clone and applies the submitted patch, if any - - command: git.get_project - params: - directory: "src" - # Make an evergreen exapanstion file with dynamic values - - command: shell.exec - params: - working_dir: "src" - script: | - # Get the current unique version of this checkout - if [ "${is_patch}" = "true" ]; then - CURRENT_VERSION=$(git describe)-patch-${version_id} - else - CURRENT_VERSION=latest - fi - - export DRIVERS_TOOLS="$(pwd)/../drivers-tools" - export PROJECT_DIRECTORY="$(pwd)" - - # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory - if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) - export PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) - fi - - export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" - export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" - - cat < expansion.yml - CURRENT_VERSION: "$CURRENT_VERSION" - DRIVERS_TOOLS: "$DRIVERS_TOOLS" - MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" - MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" - PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" - PREPARE_SHELL: | - set -o errexit - export DRIVERS_TOOLS="$DRIVERS_TOOLS" - export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" - export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" - export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$MONGODB_BINARIES:$PATH" - export PROJECT="${project}" - EOT - # See what we've done - cat expansion.yml - - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update - params: - file: src/expansion.yml - - "prepare resources": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - rm -rf $DRIVERS_TOOLS - if [ "${project}" = "drivers-tools" ]; then - # If this was a patch build, doing a fresh clone would not actually test the patch - cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS - else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git --depth 1 $DRIVERS_TOOLS - fi - echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - - "upload mo artifacts": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: mongodb-logs.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "mongodb-logs.tar.gz" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log - bucket: mciuploads - permissions: public-read - content_type: ${content_type|text/plain} - display_name: "orchestration.log" - - "upload working dir": - - command: archive.targz_pack - params: - target: "working-dir.tar.gz" - source_dir: ${PROJECT_DIRECTORY}/ - include: - - "./**" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: working-dir.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "working-dir.tar.gz" - - command: archive.targz_pack - params: - target: "drivers-dir.tar.gz" - source_dir: ${DRIVERS_TOOLS} - include: - - "./**" - exclude_files: - # Windows cannot read the mongod *.lock files because they are locked. - - "*.lock" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: drivers-dir.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "drivers-dir.tar.gz" - - "upload test results": - - command: attach.xunit_results - params: - # Uploading test results does not work when using ${PROJECT_DIRECTORY}, - # so we use an absolute path to work around this - file: "src/test-results.xml" - - command: attach.results - params: - file_location: "${DRIVERS_TOOLS}/results.json" - - "bootstrap mongo-orchestration": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - SKIP_CRYPT_SHARED=${SKIP_CRYPT_SHARED} SKIP_LEGACY_SHELL=true MONGODB_VERSION=${MONGODB_VERSION} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} LOAD_BALANCER=${LOAD_BALANCER} REQUIRE_API_VERSION=${REQUIRE_API_VERSION} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - # run-orchestration generates expansion file with MONGODB_URI and CRYPT_SHARED_LIB_PATH - - command: expansions.update - params: - file: mo-expansion.yml - - "stop mongo-orchestration": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh - - "bootstrap mongohoused": - - command: shell.exec - params: - script: | - VARIANT=${VARIANT} DRIVERS_TOOLS="${DRIVERS_TOOLS}" sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh - - command: shell.exec - params: - background: true - script: | - DRIVERS_TOOLS="${DRIVERS_TOOLS}" sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh - - "create serverless instance": - - command: shell.exec - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ - SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ - SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ - bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh - - command: expansions.update - params: - file: src/serverless-expansion.yml - - command: shell.exec - params: - shell: bash - script: | - ${PREPARE_SHELL} - - if [ -z "${SERVERLESS_MONGODB_VERSION}" ]; then - echo "expected SERVERLESS_MONGODB_VERSION to be set" - exit 1 - fi - - . ${DRIVERS_TOOLS}/.evergreen/find-python3.sh - PYTHON_BINARY="$(find_python3)" || exit 1 - - # Download the enterprise server download for current platform to $MONGODB_BINARIES. - # This is required for tests that need mongocryptd. - # $MONGODB_BINARIES is added to the $PATH in fetch-source. - ${PYTHON_BINARY} ${DRIVERS_TOOLS}/.evergreen/mongodl.py \ - --component archive \ - --version ${SERVERLESS_MONGODB_VERSION} \ - --edition enterprise \ - --out $MONGODB_BINARIES \ - --strip-path-components 2 - - # Download the crypt_shared dynamic library for the current platform. - ${PYTHON_BINARY} ${DRIVERS_TOOLS}/.evergreen/mongodl.py \ - --component crypt_shared \ - --version ${SERVERLESS_MONGODB_VERSION} \ - --edition enterprise \ - --out . \ - --only "**/mongo_crypt_v1.*" \ - --strip-path-components 1 - - # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to - # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other - # downloaded files. - CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \ - -name 'mongo_crypt_v1.so' -o \ - -name 'mongo_crypt_v1.dll' -o \ - -name 'mongo_crypt_v1.dylib')" - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" >> crypt-expansion.yml - - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update - params: - file: crypt-expansion.yml - - "delete serverless instance": - - command: shell.exec - params: - script: | - # Only run if a serverless instance was started - if [ -n "${SERVERLESS_INSTANCE_NAME}" ]; then - SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ - SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ - SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ - SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ - bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - fi - - "run tests": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}" - export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}" - export AWS_TEMP_ACCESS_KEY_ID="${client_side_encryption_aws_temp_access_key_id}" - export AWS_TEMP_SECRET_ACCESS_KEY="${client_side_encryption_aws_temp_secret_access_key_key}" - export AWS_TEMP_SESSION_TOKEN="${client_side_encryption_aws_temp_session_token}" - export AZURE_TENANT_ID="${client_side_encryption_azure_tenant_id}" - export AZURE_CLIENT_ID="${client_side_encryption_azure_client_id}" - export AZURE_CLIENT_SECRET="${client_side_encryption_azure_client_secret}" - export GCP_EMAIL="${client_side_encryption_gcp_email}" - export GCP_PRIVATE_KEY="${client_side_encryption_gcp_privatekey}" - export KMIP_ENDPOINT="${client_side_encryption_kmip_endpoint}" - export KMS_ENDPOINT_EXPIRED="${client_side_encryption_kms_endpoint_expired}" - export KMS_ENDPOINT_WRONG_HOST="${client_side_encryption_kms_endpoint_wrong_host}" - export KMS_ENDPOINT_REQUIRE_CLIENT_CERT="${client_side_encryption_kms_endpoint_require_client_cert}" - export KMS_TLS_CA_FILE="${client_side_encryption_kms_tls_ca_file}" - export KMS_TLS_CERTIFICATE_KEY_FILE="${client_side_encryption_kms_tls_certificate_key_file}" - export PATH="${PHP_PATH}/bin:$PATH" - - API_VERSION=${API_VERSION} \ - CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ - MONGODB_URI="${MONGODB_URI}" \ - MONGODB_SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ - MONGODB_MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ - PHP_VERSION=${PHP_VERSION} \ - SSL=${SSL} \ - TESTS=${TESTS} \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh - - "run atlas data lake test": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - export PATH="${PHP_PATH}/bin:$PATH" - - MONGODB_URI="mongodb://mhuser:pencil@127.0.0.1:27017" \ - TESTS="atlas-data-lake" \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh - - "run serverless tests": - - command: shell.exec - type: test - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}" - export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}" - export AWS_TEMP_ACCESS_KEY_ID="${client_side_encryption_aws_temp_access_key_id}" - export AWS_TEMP_SECRET_ACCESS_KEY="${client_side_encryption_aws_temp_secret_access_key_key}" - export AWS_TEMP_SESSION_TOKEN="${client_side_encryption_aws_temp_session_token}" - export AZURE_TENANT_ID="${client_side_encryption_azure_tenant_id}" - export AZURE_CLIENT_ID="${client_side_encryption_azure_client_id}" - export AZURE_CLIENT_SECRET="${client_side_encryption_azure_client_secret}" - export GCP_EMAIL="${client_side_encryption_gcp_email}" - export GCP_PRIVATE_KEY="${client_side_encryption_gcp_privatekey}" - export KMIP_ENDPOINT="${client_side_encryption_kmip_endpoint}" - export KMS_ENDPOINT_EXPIRED="${client_side_encryption_kms_endpoint_expired}" - export KMS_ENDPOINT_WRONG_HOST="${client_side_encryption_kms_endpoint_wrong_host}" - export KMS_ENDPOINT_REQUIRE_CLIENT_CERT="${client_side_encryption_kms_endpoint_require_client_cert}" - export KMS_TLS_CA_FILE="${client_side_encryption_kms_tls_ca_file}" - export KMS_TLS_CERTIFICATE_KEY_FILE="${client_side_encryption_kms_tls_certificate_key_file}" - export MONGODB_IS_SERVERLESS=on - export MONGODB_USERNAME=${SERVERLESS_ATLAS_USER} - export MONGODB_PASSWORD=${SERVERLESS_ATLAS_PASSWORD} - export PATH="${PHP_PATH}/bin:$PATH" - - CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ - MONGODB_URI="${SERVERLESS_URI}" \ - TESTS="serverless" \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh - - "cleanup": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - rm -rf $DRIVERS_TOOLS || true - - "fix absolute paths": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename - done - - "windows fix": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - cat $i | tr -d '\r' > $i.new - mv $i.new $i - done - # Copy client certificate because symlinks do not work on Windows. - cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem - - "make files executable": - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/.evergreen -name \*.sh); do - chmod +x $i - done - - "install dependencies": - - command: shell.exec - params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" - # Don't use ${file} syntax here because evergreen treats it as an empty expansion. - [ -f "$file" ] && PHP_VERSION=${PHP_VERSION} EXTENSION_VERSION=${EXTENSION_VERSION} EXTENSION_REPO=${EXTENSION_REPO} EXTENSION_BRANCH=${EXTENSION_BRANCH} DEPENDENCIES=${DEPENDENCIES} sh $file || echo "$file not available, skipping" - # install-dependencies generates expansion file with the PHP_PATH for the chosen PHP version - - command: expansions.update - params: - file: src/php-expansion.yml - - "start load balancer": - - command: shell.exec - params: - script: | - MONGODB_URI="${MONGODB_URI}" ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start - - command: expansions.update - params: - file: lb-expansion.yml - - "stop load balancer": - - command: shell.exec - params: - script: | - # Only run if a load balancer was started - if [ -n "${SINGLE_MONGOS_LB_URI}" ]; then - ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop - fi - - "start kms servers": - - command: shell.exec - # Init venv without background:true to install dependencies - params: - shell: bash - script: |- - set -o errexit - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - - command: shell.exec - params: - background: true - shell: bash - # Use different ports for KMS HTTP servers to avoid conflicts with load balancers - script: |- - set -o errexit - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8100 & - python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8101 & - python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8102 --require_client_cert & - python -u kms_kmip_server.py --port 5698 & - - command: expansions.update - params: - updates: - - key: client_side_encryption_kms_tls_ca_file - value: ${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem - - key: client_side_encryption_kms_tls_certificate_key_file - value: ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem - - key: client_side_encryption_kms_endpoint_expired - value: 127.0.0.1:8100 - - key: client_side_encryption_kms_endpoint_wrong_host - value: 127.0.0.1:8101 - - key: client_side_encryption_kms_endpoint_require_client_cert - value: 127.0.0.1:8102 - - key: client_side_encryption_kmip_endpoint - value: localhost:5698 - - "set aws temp creds": - - command: shell.exec - params: - shell: bash - script: |- - set -o errexit - - export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}" - export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}" - export AWS_DEFAULT_REGION="us-east-1" - - pushd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - . ./set-temp-creds.sh - popd - - if [ -z "$CSFLE_AWS_TEMP_ACCESS_KEY_ID" ]; then - echo "Failed to set AWS temporary credentials!" - exit 1 - fi +# Fail builds when pre tasks fail. +pre_error_fails_task: true - cat < aws-expansion.yml - client_side_encryption_aws_temp_access_key_id: "$CSFLE_AWS_TEMP_ACCESS_KEY_ID" - client_side_encryption_aws_temp_secret_access_key_key: "$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" - client_side_encryption_aws_temp_session_token: "$CSFLE_AWS_TEMP_SESSION_TOKEN" - EOT - - command: expansions.update - params: - file: aws-expansion.yml +# Protect ourselves against rogue test case that runs forever. Tasks are killed after 10 minutes, which shouldn't occur +# under normal circumstances. +exec_timeout_secs: 600 +# These pre and post rules apply to all tasks not part of a task group, which should only ever be tests against local +# MongoDB instances. All other tasks that require special preparation should be run from within a task group pre: - func: "fetch source" - func: "prepare resources" - - func: "windows fix" - func: "fix absolute paths" - - func: "make files executable" - func: "install dependencies" - + - func: "locate PHP binaries" + - func: "fetch extension" + - func: "install composer" post: - # Skip: uploading the full working directory is not needed by drivers-evergreen-tools. - # - func: "upload working dir" - - func: "upload mo artifacts" - func: "upload test results" - func: "delete serverless instance" - func: "stop load balancer" - func: "stop mongo-orchestration" - func: "cleanup" -tasks: - - # Wildcard task. Do you need to find out what tools are available and where? - # Throw it here, and execute this task on all buildvariants - - name: getdata - commands: - - command: shell.exec - type: test - params: - script: | - . ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true - get_distro || true - echo $DISTRO - echo $MARCH - echo $OS - uname -a || true - ls /etc/*release* || true - cc --version || true - gcc --version || true - clang --version || true - gcov --version || true - lcov --version || true - llvm-cov --version || true - echo $PATH - ls -la /usr/local/Cellar/llvm/*/bin/ || true - ls -la /usr/local/Cellar/ || true - scan-build --version || true - genhtml --version || true - valgrind --version || true - - -# Standard test tasks {{{ - - - name: "test-standalone" - tags: ["standalone"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "server" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - - - name: "test-replica_set" - tags: ["replica_set"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "replica_set" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - - - name: "test-sharded_cluster" - tags: ["sharded_cluster"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "sharded_cluster" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - - - name: "test-atlas-data-lake" - commands: - - func: "bootstrap mongohoused" - - func: "run atlas data lake test" - - - name: "test-requireApiVersion" - tags: ["versioned-api"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "server" - AUTH: "auth" - REQUIRE_API_VERSION: "yes" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - vars: - API_VERSION: "1" - - - name: "test-acceptApiVersion2" - tags: ["versioned-api"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "server" - ORCHESTRATION_FILE: "versioned-api-testing.json" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - vars: - TESTS: "versioned-api" - - - name: "test-serverless" - tags: ["serverless"] - commands: - - func: "create serverless instance" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run serverless tests" - - - name: "test-loadBalanced" - tags: ["loadbalanced"] - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "sharded_cluster" - LOAD_BALANCER: "true" - SSL: "yes" - - func: "start load balancer" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - vars: - # Note: loadBalanced=true should already be appended to SINGLE_MONGOS_LB_URI - MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" - SSL: "yes" - # Note: "stop load balancer" will be called from "post" - - - name: "test-csfle-crypt_shared" - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "replica_set" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - vars: - TESTS: "csfle" - - - name: "test-csfle-mongocryptd" - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "replica_set" - SKIP_CRYPT_SHARED: "yes" - - func: "start kms servers" - - func: "set aws temp creds" - - func: "run tests" - vars: - TESTS: "csfle" - - - name: "test-without_aws_creds" - commands: - - func: "bootstrap mongo-orchestration" - vars: - TOPOLOGY: "replica_set" - - func: "start kms servers" - - func: "run tests" - vars: - client_side_encryption_aws_access_key_id: "" - client_side_encryption_aws_secret_access_key: "" - TESTS: "csfle-without-aws-creds" - - - name: "test-atlas" - commands: - - func: "start kms servers" - - func: "run tests" - vars: - TESTS: "atlas" - -# }}} - - -axes: - # Note: install-dependencies.sh will search for the latest minor version - # matching the PHP_VERSION constant - - id: php-versions - display_name: PHP Version - values: - - id: "8.2" - display_name: "PHP 8.2" - variables: - PHP_VERSION: "8.2" - - id: "8.1" - display_name: "PHP 8.1" - variables: - PHP_VERSION: "8.1" - - id: "8.0" - display_name: "PHP 8.0" - variables: - PHP_VERSION: "8.0" - - id: "7.4" - display_name: "PHP 7.4" - variables: - PHP_VERSION: "7.4" - - - id: php-edge-versions - display_name: PHP Version - values: - - id: "latest-stable" - display_name: "PHP 8.2" - variables: - PHP_VERSION: "8.2" - - id: "oldest-supported" - display_name: "PHP 7.4" - variables: - PHP_VERSION: "7.4" - - - id: mongodb-versions - display_name: MongoDB Version - values: - - id: "latest" - display_name: "MongoDB latest" - variables: - MONGODB_VERSION: "latest" - - id: "rapid" - display_name: "MongoDB rapid" - variables: - MONGODB_VERSION: "rapid" - - id: "7.0" - display_name: "MongoDB 7.0" - variables: - MONGODB_VERSION: "7.0" - - id: "6.0" - display_name: "MongoDB 6.0" - variables: - MONGODB_VERSION: "6.0" - - id: "5.0" - display_name: "MongoDB 5.0" - variables: - MONGODB_VERSION: "5.0" - - id: "4.4" - display_name: "MongoDB 4.4" - variables: - MONGODB_VERSION: "4.4" - - id: "4.2" - display_name: "MongoDB 4.2" - variables: - MONGODB_VERSION: "4.2" - - id: "4.0" - display_name: "MongoDB 4.0" - variables: - MONGODB_VERSION: "4.0" - - id: "3.6" - display_name: "MongoDB 3.6" - variables: - MONGODB_VERSION: "3.6" - - - id: mongodb-edge-versions - display_name: MongoDB Version - values: - - id: "latest-stable" - display_name: "MongoDB 6.0" - variables: - MONGODB_VERSION: "6.0" - - id: "oldest-supported" - display_name: "MongoDB 3.6" - variables: - MONGODB_VERSION: "3.6" - - - id: driver-versions - display_name: Driver Version - values: - - id: "oldest-supported" - display_name: "PHPC 1.17.0" - variables: - EXTENSION_BRANCH: "master" - - id: "latest-stable" - display_name: "PHPC (1.17.x)" - variables: - EXTENSION_BRANCH: "master" - - id: "latest-dev" - display_name: "PHPC (1.17-dev)" - variables: - EXTENSION_BRANCH: "master" - - - id: os - display_name: OS - values: - # Debian - - id: debian11 - display_name: "Debian 11" - run_on: debian11-small - - id: debian10 - display_name: "Debian 10" - run_on: debian10-small - - id: debian92 - display_name: "Debian 9.2" - run_on: debian92-small - - # RHEL - - id: rhel90 - display_name: "RHEL 9.0" - run_on: rhel90-small - - id: rhel80 - display_name: "RHEL 8.0" - run_on: rhel80-small - - # Ubuntu LTS - - id: ubuntu2204 - display_name: "Ubuntu 22.04 x64" - run_on: ubuntu2204-small - - id: ubuntu2204-arm64 - display_name: "Ubuntu 22.04 ARM64" - run_on: ubuntu2204-arm64-small - - id: ubuntu2004 - display_name: "Ubuntu 20.04 x64" - run_on: ubuntu2004-small - - id: ubuntu2004-arm64 - display_name: "Ubuntu 20.04 ARM64" - run_on: ubuntu2204-arm64-small - - - id: topology - display_name: Topology - values: - - id: standalone - display_name: Standalone - variables: - TOPOLOGY: "server" - - id: replicaset - display_name: Replica Set - variables: - TOPOLOGY: "replica_set" - - id: sharded-cluster - display_name: Sharded Cluster - variables: - TOPOLOGY: "sharded_cluster" - - - id: auth - display_name: Authentication - values: - - id: auth - display_name: Auth - variables: - AUTH: "auth" - - id: noauth - display_name: NoAuth - variables: - AUTH: "noauth" - - - id: ssl - display_name: SSL - values: - - id: ssl - display_name: SSL - variables: - SSL: "ssl" - - id: nossl - display_name: NoSSL - variables: - SSL: "nossl" - - - id: storage-engine - display_name: Storage - values: - - id: mmapv1 - display_name: MMAPv1 - variables: - STORAGE_ENGINE: "mmapv1" - - id: wiredtiger - display_name: WiredTiger - variables: - STORAGE_ENGINE: "wiredtiger" - - id: inmemory - display_name: InMemory - variables: - STORAGE_ENGINE: "inmemory" - - - id: dependencies - display_name: Dependencies - values: - - id: lowest - display_name: Lowest - variables: - DEPENDENCIES: "lowest" - -task_groups: - - name: test_atlas_task_group - setup_group: - - func: "fetch source" - - func: "prepare resources" - - func: "fix absolute paths" - - func: "make files executable" - - func: "install dependencies" - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - env: - MONGODB_VERSION: '7.0' - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: src/atlas-expansion.yml - teardown_group: - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh - - func: "upload test results" - - func: "cleanup" - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 - tasks: - - test-atlas - -buildvariants: -# Test all PHP versions with latest-stable MongoDB and PHPC on all platforms -- matrix_name: "test-php-versions" - matrix_spec: - os: - - debian11 - - debian10 - - rhel90 - - ubuntu2204 - - ubuntu2204-arm64 - - ubuntu2004 - mongodb-edge-versions: "latest-stable" - php-versions: "*" - driver-versions: "latest-stable" - display_name: "${os}, ${mongodb-edge-versions}, ${php-versions}, ${driver-versions}" - exclude_spec: - # Exclude "latest-stable" PHP version for Debian 11 (see: test-mongodb-versions matrix) - - { "os": "debian11", "mongodb-edge-versions": "latest-stable", "php-versions": "8.2", "driver-versions": "latest-stable" } - # Exclude PHP versions older than 8.1 on RHEL 9 and Ubuntu 22.04 (OpenSSL 3 is only supported on PHP 8.1+) - - { "os": ["rhel90", "ubuntu2204-arm64", "ubuntu2204"], "php-versions": ["7.4", "8.0"], "mongodb-edge-versions": "*", "driver-versions": "*" } - tasks: - - name: "test-standalone" - - name: "test-replica_set" - - name: "test-sharded_cluster" - -# Test all topologies and MongoDB versions with latest-stable PHP and PHPC on Debian -- matrix_name: "test-mongodb-versions" - matrix_spec: { "os": ["debian92", "debian11"], "mongodb-versions": "*", "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: "${os}, ${mongodb-versions}, ${php-edge-versions}, ${driver-versions}" - exclude_spec: - # Debian 9.2 only supports up to MongoDB 5.0 - - { "os": "debian92", "mongodb-versions": ["6.0", "7.0", "rapid", "latest"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - - { "os": "debian11", "mongodb-versions": ["3.6", "4.0", "4.2", "4.4", "5.0"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - tasks: - - name: "test-standalone" - - name: "test-replica_set" - - name: "test-sharded_cluster" - -# Test oldest-supported PHP, MongoDB, and driver versions with lowest dependencies on Debian -- matrix_name: "test-oldest-supported" - matrix_spec: { "os": "debian92", "mongodb-edge-versions": "oldest-supported", "php-edge-versions": "oldest-supported", "driver-versions": "oldest-supported", "dependencies": "lowest" } - display_name: "Lowest Dependencies: ${os}, ${mongodb-edge-versions}, ${php-edge-versions}, ${driver-versions}" - tasks: - - name: "test-standalone" - - name: "test-replica_set" - - name: "test-sharded_cluster" - -- matrix_name: "atlas-data-lake-test" - matrix_spec: { "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: "Atlas Data Lake" - run_on: debian11 - expansions: - VARIANT: debian11 # Referenced by ADL build script for downloading MQLRun - tasks: - - name: "test-atlas-data-lake" - -- matrix_name: "test-requireApiVersion" - matrix_spec: { "os": "debian11", "mongodb-versions": "*", "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: "Versioned API - ${mongodb-versions}" - exclude_spec: - # Stable API is available from MongoDB 5.0+ - - { "os": "debian11", "mongodb-versions": ["3.6", "4.0", "4.2", "4.4"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - tasks: - - .versioned-api - -- matrix_name: "serverless" - matrix_spec: { "os": "debian11", "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: "Serverless" - tasks: - - .serverless - -- matrix_name: "test-loadBalanced" - matrix_spec: { "os": "debian11", "mongodb-versions": "*", "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: "Load balanced - ${mongodb-versions}" - exclude_spec: - # Load balancer is available from MongoDB 5.0+ - - { "os": "debian11", "mongodb-versions": ["3.6", "4.0", "4.2", "4.4"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - tasks: - - name: "test-loadBalanced" - -# CSFLE tests (crypt_shared and mongocryptd) are tested on RHEL 8 as it's the only version that supports -# 4.2 AND 6.0 (which is required for crypt_shared), as well as a somewhat relevant PHP version. -# Newer MongoDB version may require adding different operating systems here -- matrix_name: "test-csfle-crypt_shared" - matrix_spec: { "os": "rhel80", "mongodb-versions": "*", "php-versions": "8.0", "driver-versions": "latest-stable" } - display_name: "CSFLE crypt_shared - ${mongodb-versions}" - exclude_spec: - # MongoDB < 4.2 does not need to be tested. - - { "os": "rhel80", "mongodb-versions": ["3.6", "4.0"], "php-versions": "8.0", "driver-versions": "latest-stable" } - tasks: - - name: "test-csfle-crypt_shared" -- matrix_name: "test-csfle-mongocryptd" - matrix_spec: { "os": "rhel80", "mongodb-versions": "*", "php-versions": "8.0", "driver-versions": "latest-stable" } - display_name: "CSFLE mongocryptd - ${mongodb-versions}" - exclude_spec: - # MongoDB < 4.2 does not need to be tested. - - { "os": "rhel80", "mongodb-versions": ["3.6", "4.0"], "php-versions": "8.0", "driver-versions": "latest-stable" } - tasks: - - name: "test-csfle-mongocryptd" - -# Run CSFLE tests without AWS credentials (for "On-demand AWS Credentials" prose test) -- matrix_name: "test-csfle-without_aws_creds" - matrix_spec: { "os": ["debian92", "debian11"], "mongodb-versions": "*", "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: "CSFLE without_aws_creds - ${mongodb-versions}" - exclude_spec: - # CSFLE is available from MongoDB 4.2+ and Debian 9.2 only supports up to MongoDB 5.0 - - { "os": "debian92", "mongodb-versions": ["3.6", "4.0", "6.0", "rapid", "latest"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - - { "os": "debian11", "mongodb-versions": ["3.6", "4.0", "4.2", "4.4", "5.0"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - tasks: - - name: "test-without_aws_creds" - -- matrix_name: rhel8-test-atlas - matrix_spec: { "os": ["rhel80"], "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" } - display_name: Atlas Tests - tasks: - - test_atlas_task_group +# Define aliases for patch builds and PR builds. These only apply if no aliases are defined in project settings +github_pr_aliases: + # Run tests on debian, only testing local replicasets + - variant: "test-.*" + task: "test-mongodb-.*-replicaset" + +# Include files that contain various tasks, task groups, and build variant definitions +include: + - filename: .evergreen/config/functions.yml + + - filename: .evergreen/config/build-task-groups.yml + - filename: .evergreen/config/build-variants.yml + + - filename: .evergreen/config/test-tasks.yml + - filename: .evergreen/config/test-task-groups.yml + - filename: .evergreen/config/test-variants.yml + + # Automatically generated files + - filename: .evergreen/config/generated/build/build-php-8.2.yml + - filename: .evergreen/config/generated/build/build-php-8.1.yml + - filename: .evergreen/config/generated/build/build-php-8.0.yml + - filename: .evergreen/config/generated/build/build-php-7.4.yml + - filename: .evergreen/config/generated/test/local-latest.yml + - filename: .evergreen/config/generated/test/local-rapid.yml + - filename: .evergreen/config/generated/test/local-7.0.yml + - filename: .evergreen/config/generated/test/local-6.0.yml + - filename: .evergreen/config/generated/test/local-5.0.yml + - filename: .evergreen/config/generated/test/local-4.4.yml + - filename: .evergreen/config/generated/test/local-4.2.yml + - filename: .evergreen/config/generated/test/local-4.0.yml + - filename: .evergreen/config/generated/test/local-3.6.yml + - filename: .evergreen/config/generated/test/load-balanced-latest.yml + - filename: .evergreen/config/generated/test/load-balanced-rapid.yml + - filename: .evergreen/config/generated/test/load-balanced-7.0.yml + - filename: .evergreen/config/generated/test/load-balanced-6.0.yml + - filename: .evergreen/config/generated/test/load-balanced-5.0.yml + - filename: .evergreen/config/generated/test/require-api-version-latest.yml + - filename: .evergreen/config/generated/test/require-api-version-rapid.yml + - filename: .evergreen/config/generated/test/require-api-version-7.0.yml + - filename: .evergreen/config/generated/test/require-api-version-6.0.yml + - filename: .evergreen/config/generated/test/require-api-version-5.0.yml + - filename: .evergreen/config/generated/test/csfle-latest.yml + - filename: .evergreen/config/generated/test/csfle-rapid.yml + - filename: .evergreen/config/generated/test/csfle-7.0.yml + - filename: .evergreen/config/generated/test/csfle-6.0.yml + - filename: .evergreen/config/generated/test/csfle-5.0.yml + - filename: .evergreen/config/generated/test/csfle-4.4.yml + - filename: .evergreen/config/generated/test/csfle-4.2.yml diff --git a/.evergreen/config/build-task-groups.yml b/.evergreen/config/build-task-groups.yml new file mode 100644 index 000000000..892548181 --- /dev/null +++ b/.evergreen/config/build-task-groups.yml @@ -0,0 +1,32 @@ +variables: + build_setup: &build_setup + - func: "fetch source" + - func: "prepare resources" + - func: "fix absolute paths" + - func: "install dependencies" + build_teardown: &build_teardown + - func: "cleanup" + +task_groups: + # Builds all versions of PHP + - name: "build-all-php" + # Keep this number in sync with the number of PHP versions to allow for parallel builds + max_hosts: 4 + setup_task: *build_setup + setup_task_can_fail_task: true + setup_task_timeout_secs: 1800 + teardown_task: *build_teardown + tasks: + - ".build" + + # Builds all versions of PHP that support OpenSSL 3 (PHP 8.1+) + - name: "build-php-openssl3" + # Keep this number in sync with the number of PHP versions to allow for parallel builds + # Subtract 2 versions as PHP 7.4 and 8.0 are not built with OpenSSL 3 + max_hosts: 2 + setup_task: *build_setup + setup_task_can_fail_task: true + setup_task_timeout_secs: 1800 + teardown_task: *build_teardown + tasks: + - ".build !.php7.4 !.php8.0" diff --git a/.evergreen/config/build-variants.yml b/.evergreen/config/build-variants.yml new file mode 100644 index 000000000..189abbe1d --- /dev/null +++ b/.evergreen/config/build-variants.yml @@ -0,0 +1,87 @@ +# +# Build variants to build the driver - these are run for all operating systems and PHP versions we support +# +buildvariants: + # Debian + - name: build-debian11 + display_name: "Build: Debian 11" + tags: ["build", "debian", "x64"] + run_on: debian11-small + tasks: + - name: "build-all-php" + - name: build-debian10 + display_name: "Build: Debian 10" + tags: ["build", "debian", "x64"] + run_on: debian10-small + tasks: + - name: "build-all-php" + - name: build-debian92 + display_name: "Build: Debian 9.2" + tags: ["build", "debian", "x64"] + run_on: debian92-small + tasks: + - name: "build-all-php" + + # RHEL + - name: build-rhel90 + display_name: "Build: RHEL 9.0" + tags: ["build", "rhel", "x64"] + run_on: rhel90-small + tasks: + - name: "build-php-openssl3" + - name: build-rhel83-zseries + display_name: "Build: RHEL 8.3 Zseries" + tags: ["build", "rhel", "zseries"] + run_on: rhel83-zseries-small + tasks: + - name: "build-all-php" + - name: build-rhel82-arm64 + display_name: "Build: RHEL 8.2 ARM64" + tags: ["build", "rhel", "arm64"] + run_on: rhel82-arm64 + tasks: + - name: "build-all-php" + - name: build-rhel81-power8 + display_name: "Build: RHEL 8.1 Power8" + tags: ["build", "rhel", "power8"] + run_on: rhel81-power8-large + tasks: + - name: "build-all-php" + - name: build-rhel80 + display_name: "Build: RHEL 8.0" + tags: ["build", "rhel", "x64"] + run_on: rhel80-small + tasks: + - name: "build-all-php" + - name: build-rhel76 + display_name: "Build: RHEL 7.6" + tags: ["build", "rhel", "x64"] + run_on: rhel76-small + tasks: + - name: "build-all-php" + + # Ubuntu LTS + - name: build-ubuntu2204 + display_name: "Build: Ubuntu 22.04 x64" + tags: ["build", "ubuntu", "x64"] + run_on: ubuntu2204-small + tasks: + - name: "build-php-openssl3" + - name: build-ubuntu2204-arm64 + display_name: "Build: Ubuntu 22.04 ARM64" + tags: ["build", "ubuntu", "arm64"] + run_on: ubuntu2204-arm64-small + tasks: + - name: "build-php-openssl3" + - name: build-ubuntu2004 + display_name: "Build: Ubuntu 20.04 x64" + tags: ["build", "ubuntu", "x64"] + run_on: ubuntu2004-small + tasks: + - name: "build-all-php" + - name: build-ubuntu2004-arm64 + display_name: "Build: Ubuntu 20.04 ARM64" + tags: ["build", "ubuntu", "arm64"] + run_on: ubuntu2004-arm64-small + tasks: + - name: "build-all-php" diff --git a/.evergreen/config/functions.yml b/.evergreen/config/functions.yml new file mode 100644 index 000000000..4df8d3edf --- /dev/null +++ b/.evergreen/config/functions.yml @@ -0,0 +1,484 @@ +functions: + "fetch source": + # Executes git clone and applies the submitted patch, if any + - command: git.get_project + params: + directory: "src" + # Make an evergreen exapanstion file with dynamic values + - command: shell.exec + params: + working_dir: "src" + script: | + # Get the current unique version of this checkout + if [ "${is_patch}" = "true" ]; then + CURRENT_VERSION=$(git describe)-patch-${version_id} + else + CURRENT_VERSION=latest + fi + + export DRIVERS_TOOLS="$(pwd)/../drivers-tools" + export PROJECT_DIRECTORY="$(pwd)" + + # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory + if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) + export PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) + fi + + export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" + export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" + export UPLOAD_BUCKET="${project}" + + cat < expansion.yml + CURRENT_VERSION: "$CURRENT_VERSION" + DRIVERS_TOOLS: "$DRIVERS_TOOLS" + MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" + MONGODB_BINARIES: "$MONGODB_BINARIES" + UPLOAD_BUCKET: "$UPLOAD_BUCKET" + PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + PREPARE_SHELL: | + set -o errexit + export DRIVERS_TOOLS="$DRIVERS_TOOLS" + export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" + export MONGODB_BINARIES="$MONGODB_BINARIES" + export UPLOAD_BUCKET="$UPLOAD_BUCKET" + export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" + + export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" + export PATH="$MONGODB_BINARIES:$PATH" + export PROJECT="${project}" + EOT + # See what we've done + cat expansion.yml + + # Load the expansion file to make an evergreen variable with the current unique version + - command: expansions.update + params: + file: src/expansion.yml + + # Upload build artifacts that other tasks may depend on + # Note this URL needs to be totally unique, while predictable for the next task + # so it can automatically download the artifacts + "upload extension": + # Copy compiled extension to source directory for archiving + - command: subprocess.exec + type: setup + params: + working_dir: "src" + binary: bash + args: + - -c + - cp `${PHP_PATH}/bin/php -r "echo ini_get('extension_dir');"`/mongodb.so . + # Compress and upload the entire build directory + - command: archive.targz_pack + params: + target: "${build_id}.tar.gz" + source_dir: src + include: + - "mongodb.so" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: ${content_type|application/x-gzip} + permissions: public-read + local_file: ${build_id}.tar.gz + remote_file: mongo-php-driver/${build_variant}/${revision}/${task_name}/${version}.tar.gz + # TODO: Use separate folder for the library once it exists +# remote_file: ${project}/${build_variant}/${revision}/${task_name}/${version}.tar.gz + + "fetch extension": + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + # TODO: Use separate folder for the library once it exists + remote_file: mongo-php-driver/${FETCH_BUILD_VARIANT}/${revision}/${FETCH_BUILD_TASK}/${version}.tar.gz +# remote_file: ${project}/${FETCH_BUILD_VARIANT}/${revision}/${FETCH_BUILD_TASK}/${version}.tar.gz + local_file: build.tar.gz + - command: archive.targz_extract + params: + destination: src + path: build.tar.gz + # Move compiled extension to correct ini path + - command: subprocess.exec + type: setup + params: + working_dir: "src" + binary: bash + args: + - -c + - mv mongodb.so `${PHP_PATH}/bin/php -r "echo ini_get('extension_dir');"` + + "prepare resources": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + rm -rf $DRIVERS_TOOLS + if [ "${project}" = "drivers-tools" ]; then + # If this was a patch build, doing a fresh clone would not actually test the patch + cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS + else + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git --depth 1 $DRIVERS_TOOLS + fi + echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config + + "upload test results": + - command: attach.xunit_results + params: + # Uploading test results does not work when using ${PROJECT_DIRECTORY}, + # so we use an absolute path to work around this + file: "src/test-results.xml" + - command: attach.results + params: + file_location: "${DRIVERS_TOOLS}/results.json" + + "bootstrap mongo-orchestration": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + SKIP_CRYPT_SHARED=${SKIP_CRYPT_SHARED} SKIP_LEGACY_SHELL=true MONGODB_VERSION=${MONGODB_VERSION} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} TOPOLOGY=${TOPOLOGY} AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} LOAD_BALANCER=${LOAD_BALANCER} REQUIRE_API_VERSION=${REQUIRE_API_VERSION} sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + # run-orchestration generates expansion file with MONGODB_URI and CRYPT_SHARED_LIB_PATH + - command: expansions.update + params: + file: mo-expansion.yml + + "stop mongo-orchestration": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh + + "bootstrap mongohoused": + - command: shell.exec + params: + script: | + VARIANT=${VARIANT} DRIVERS_TOOLS="${DRIVERS_TOOLS}" sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh + - command: shell.exec + params: + background: true + script: | + DRIVERS_TOOLS="${DRIVERS_TOOLS}" sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh + + "create serverless instance": + - command: shell.exec + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: src/serverless-expansion.yml + - command: shell.exec + params: + shell: bash + script: | + ${PREPARE_SHELL} + + if [ -z "${SERVERLESS_MONGODB_VERSION}" ]; then + echo "expected SERVERLESS_MONGODB_VERSION to be set" + exit 1 + fi + + . ${DRIVERS_TOOLS}/.evergreen/find-python3.sh + PYTHON_BINARY="$(find_python3)" || exit 1 + + # Download the enterprise server download for current platform to $MONGODB_BINARIES. + # This is required for tests that need mongocryptd. + # $MONGODB_BINARIES is added to the $PATH in fetch-source. + ${PYTHON_BINARY} ${DRIVERS_TOOLS}/.evergreen/mongodl.py \ + --component archive \ + --version ${SERVERLESS_MONGODB_VERSION} \ + --edition enterprise \ + --out $MONGODB_BINARIES \ + --strip-path-components 2 + + # Download the crypt_shared dynamic library for the current platform. + ${PYTHON_BINARY} ${DRIVERS_TOOLS}/.evergreen/mongodl.py \ + --component crypt_shared \ + --version ${SERVERLESS_MONGODB_VERSION} \ + --edition enterprise \ + --out . \ + --only "**/mongo_crypt_v1.*" \ + --strip-path-components 1 + + # Find the crypt_shared library file in the current directory and set the CRYPT_SHARED_LIB_PATH to + # the path of that file. Only look for .so, .dll, or .dylib files to prevent matching any other + # downloaded files. + CRYPT_SHARED_LIB_PATH="$(find $(pwd) -maxdepth 1 -type f \ + -name 'mongo_crypt_v1.so' -o \ + -name 'mongo_crypt_v1.dll' -o \ + -name 'mongo_crypt_v1.dylib')" + + echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" >> crypt-expansion.yml + + # Load the expansion file to make an evergreen variable with the current unique version + - command: expansions.update + params: + file: crypt-expansion.yml + + "delete serverless instance": + - command: shell.exec + params: + script: | + # Only run if a serverless instance was started + if [ -n "${SERVERLESS_INSTANCE_NAME}" ]; then + SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ + SERVERLESS_DRIVERS_GROUP=${SERVERLESS_DRIVERS_GROUP} \ + SERVERLESS_API_PUBLIC_KEY=${SERVERLESS_API_PUBLIC_KEY} \ + SERVERLESS_API_PRIVATE_KEY=${SERVERLESS_API_PRIVATE_KEY} \ + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + fi + + "run tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}" + export AWS_TEMP_ACCESS_KEY_ID="${client_side_encryption_aws_temp_access_key_id}" + export AWS_TEMP_SECRET_ACCESS_KEY="${client_side_encryption_aws_temp_secret_access_key_key}" + export AWS_TEMP_SESSION_TOKEN="${client_side_encryption_aws_temp_session_token}" + export AZURE_TENANT_ID="${client_side_encryption_azure_tenant_id}" + export AZURE_CLIENT_ID="${client_side_encryption_azure_client_id}" + export AZURE_CLIENT_SECRET="${client_side_encryption_azure_client_secret}" + export GCP_EMAIL="${client_side_encryption_gcp_email}" + export GCP_PRIVATE_KEY="${client_side_encryption_gcp_privatekey}" + export KMIP_ENDPOINT="${client_side_encryption_kmip_endpoint}" + export KMS_ENDPOINT_EXPIRED="${client_side_encryption_kms_endpoint_expired}" + export KMS_ENDPOINT_WRONG_HOST="${client_side_encryption_kms_endpoint_wrong_host}" + export KMS_ENDPOINT_REQUIRE_CLIENT_CERT="${client_side_encryption_kms_endpoint_require_client_cert}" + export KMS_TLS_CA_FILE="${client_side_encryption_kms_tls_ca_file}" + export KMS_TLS_CERTIFICATE_KEY_FILE="${client_side_encryption_kms_tls_certificate_key_file}" + export PATH="${PHP_PATH}/bin:$PATH" + + API_VERSION=${API_VERSION} \ + CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ + MONGODB_URI="${MONGODB_URI}" \ + MONGODB_SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ + MONGODB_MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ + PHP_VERSION=${PHP_VERSION} \ + SSL=${SSL} \ + TESTS=${TESTS} \ + sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + + "run atlas data lake test": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + export PATH="${PHP_PATH}/bin:$PATH" + + MONGODB_URI="mongodb://mhuser:pencil@127.0.0.1:27017" \ + TESTS="atlas-data-lake" \ + sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + + "run serverless tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}" + export AWS_TEMP_ACCESS_KEY_ID="${client_side_encryption_aws_temp_access_key_id}" + export AWS_TEMP_SECRET_ACCESS_KEY="${client_side_encryption_aws_temp_secret_access_key_key}" + export AWS_TEMP_SESSION_TOKEN="${client_side_encryption_aws_temp_session_token}" + export AZURE_TENANT_ID="${client_side_encryption_azure_tenant_id}" + export AZURE_CLIENT_ID="${client_side_encryption_azure_client_id}" + export AZURE_CLIENT_SECRET="${client_side_encryption_azure_client_secret}" + export GCP_EMAIL="${client_side_encryption_gcp_email}" + export GCP_PRIVATE_KEY="${client_side_encryption_gcp_privatekey}" + export KMIP_ENDPOINT="${client_side_encryption_kmip_endpoint}" + export KMS_ENDPOINT_EXPIRED="${client_side_encryption_kms_endpoint_expired}" + export KMS_ENDPOINT_WRONG_HOST="${client_side_encryption_kms_endpoint_wrong_host}" + export KMS_ENDPOINT_REQUIRE_CLIENT_CERT="${client_side_encryption_kms_endpoint_require_client_cert}" + export KMS_TLS_CA_FILE="${client_side_encryption_kms_tls_ca_file}" + export KMS_TLS_CERTIFICATE_KEY_FILE="${client_side_encryption_kms_tls_certificate_key_file}" + export MONGODB_IS_SERVERLESS=on + export MONGODB_USERNAME=${SERVERLESS_ATLAS_USER} + export MONGODB_PASSWORD=${SERVERLESS_ATLAS_PASSWORD} + export PATH="${PHP_PATH}/bin:$PATH" + + CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ + MONGODB_URI="${SERVERLESS_URI}" \ + TESTS="serverless" \ + sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + + "cleanup": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + rm -rf $DRIVERS_TOOLS || true + + "fix absolute paths": + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do + perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename + done + + "install dependencies": + - command: shell.exec + params: + working_dir: "src" + script: | + ${PREPARE_SHELL} + file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" + # Don't use ${file} syntax here because evergreen treats it as an empty expansion. + [ -f "$file" ] && sh $file || echo "$file not available, skipping" + + "install composer": + - command: shell.exec + params: + add_expansions_to_env: true + working_dir: "src" + script: | + ${PREPARE_SHELL} + file="${PROJECT_DIRECTORY}/.evergreen/install-composer.sh" + # Don't use ${file} syntax here because evergreen treats it as an empty expansion. + [ -f "$file" ] && DEPENDENCIES=${DEPENDENCIES} sh $file || echo "$file not available, skipping" + + "start load balancer": + - command: shell.exec + params: + script: | + MONGODB_URI="${MONGODB_URI}" ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start + - command: expansions.update + params: + file: lb-expansion.yml + + "stop load balancer": + - command: shell.exec + params: + script: | + # Only run if a load balancer was started + if [ -n "${SINGLE_MONGOS_LB_URI}" ]; then + ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop + fi + + "start kms servers": + - command: shell.exec + # Init venv without background:true to install dependencies + params: + shell: bash + script: |- + set -o errexit + cd ${DRIVERS_TOOLS}/.evergreen/csfle + . ./activate-kmstlsvenv.sh + - command: shell.exec + params: + background: true + shell: bash + # Use different ports for KMS HTTP servers to avoid conflicts with load balancers + script: |- + set -o errexit + cd ${DRIVERS_TOOLS}/.evergreen/csfle + . ./activate-kmstlsvenv.sh + python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8100 & + python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8101 & + python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8102 --require_client_cert & + python -u kms_kmip_server.py --port 5698 & + - command: expansions.update + params: + updates: + - key: client_side_encryption_kms_tls_ca_file + value: ${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem + - key: client_side_encryption_kms_tls_certificate_key_file + value: ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem + - key: client_side_encryption_kms_endpoint_expired + value: 127.0.0.1:8100 + - key: client_side_encryption_kms_endpoint_wrong_host + value: 127.0.0.1:8101 + - key: client_side_encryption_kms_endpoint_require_client_cert + value: 127.0.0.1:8102 + - key: client_side_encryption_kmip_endpoint + value: localhost:5698 + + "set aws temp creds": + - command: shell.exec + params: + shell: bash + script: |- + set -o errexit + + export AWS_ACCESS_KEY_ID="${client_side_encryption_aws_access_key_id}" + export AWS_SECRET_ACCESS_KEY="${client_side_encryption_aws_secret_access_key}" + export AWS_DEFAULT_REGION="us-east-1" + + pushd ${DRIVERS_TOOLS}/.evergreen/csfle + . ./activate-kmstlsvenv.sh + . ./set-temp-creds.sh + popd + + if [ -z "$CSFLE_AWS_TEMP_ACCESS_KEY_ID" ]; then + echo "Failed to set AWS temporary credentials!" + exit 1 + fi + + cat < aws-expansion.yml + client_side_encryption_aws_temp_access_key_id: "$CSFLE_AWS_TEMP_ACCESS_KEY_ID" + client_side_encryption_aws_temp_secret_access_key_key: "$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" + client_side_encryption_aws_temp_session_token: "$CSFLE_AWS_TEMP_SESSION_TOKEN" + EOT + - command: expansions.update + params: + file: aws-expansion.yml + + "locate PHP binaries": + - command: shell.exec + params: + shell: bash + add_expansions_to_env: true + script: | + if [ ! -d "/opt/php" ]; then + echo "PHP is not available" + exit 1 + fi + + if [ -d "/opt/php/${PHP_VERSION}-64bit/bin" ]; then + export PHP_PATH="/opt/php/${PHP_VERSION}-64bit" + else + # Try to find the newest version matching our constant + export PHP_PATH=`find /opt/php/ -maxdepth 1 -type d -name "${PHP_VERSION}*-64bit" -print | sort -V -r | head -n 1` + fi + + if [ ! -d "$PHP_PATH" ]; then + echo "Could not find PHP binaries for version ${PHP_VERSION}. Listing available versions..." + ls -1 /opt/php + exit 1 + fi + + echo "Found PHP: $PHP_PATH" + echo 'PHP_PATH: "'$PHP_PATH'"' > php-expansion.yml + - command: expansions.update + params: + file: php-expansion.yml + + "compile extension": + - command: subprocess.exec + type: test + params: + working_dir: src + add_expansions_to_env: true + binary: bash + args: + - .evergreen/compile-extension.sh diff --git a/.evergreen/config/generate-config.php b/.evergreen/config/generate-config.php new file mode 100644 index 000000000..d1a6c866b --- /dev/null +++ b/.evergreen/config/generate-config.php @@ -0,0 +1,88 @@ +#!/bin/env php + in_array($version, ['latest', 'rapid']) || version_compare($version, '5.0', '>='), +); +$requireApiServerVersions = array_filter( + $supportedMongoDBVersions, + // requireApiVersion supports MongoDB 5.0+ + fn (string $version): bool => in_array($version, ['latest', 'rapid']) || version_compare($version, '5.0', '>='), +); +$csfleServerVersions = array_filter( + $supportedMongoDBVersions, + // Test CSFLE on MongoDB 4.2+ + fn (string $version): bool => in_array($version, ['latest', 'rapid']) || version_compare($version, '4.2', '>='), +); + +$allFiles = []; + +// Build tasks +$allFiles[] = generateConfigs('build', 'phpVersion', 'build-extension.yml', 'build-php-%s', $supportedPhpVersions); + +// Test tasks +$allFiles[] = generateConfigs('test', 'mongodbVersion', 'local.yml', 'local-%s', $localServerVersions); +$allFiles[] = generateConfigs('test', 'mongodbVersion', 'load-balanced.yml', 'load-balanced-%s', $loadBalancedServerVersions); +$allFiles[] = generateConfigs('test', 'mongodbVersion', 'require-api-version.yml', 'require-api-version-%s', $requireApiServerVersions); +$allFiles[] = generateConfigs('test', 'mongodbVersion', 'csfle.yml', 'csfle-%s', $csfleServerVersions); + +echo "Generated config. Use the following list to import files:\n"; +echo implode("\n", array_map('getImportConfig', array_merge(...$allFiles))) . "\n"; + +function getImportConfig(string $filename): string +{ + return '- filename: ' . $filename; +} + +function generateConfigs( + string $directory, + string $replacementName, + string $templateFile, + string $outputFormat, + array $versions, +): array { + $templateRelativePath = 'templates/' . $directory . '/' . $templateFile; + $template = file_get_contents(__DIR__ . '/' . $templateRelativePath); + $header = sprintf( + '# This file is generated automatically - please edit the "%s" template file instead.', + $templateRelativePath + ); + + $files = []; + + foreach ($versions as $version) { + $filename = sprintf('/generated/%s/' . $outputFormat . '.yml', $directory, $version); + $files[] = '.evergreen/config' . $filename; + + $replacements = ['%' . $replacementName . '%' => $version]; + + file_put_contents(__DIR__ . $filename, $header . "\n" . strtr($template, $replacements)); + } + + return $files; +} diff --git a/.evergreen/config/generated/build/build-php-7.4.yml b/.evergreen/config/generated/build/build-php-7.4.yml new file mode 100644 index 000000000..c536ec27b --- /dev/null +++ b/.evergreen/config/generated/build/build-php-7.4.yml @@ -0,0 +1,44 @@ +# This file is generated automatically - please edit the "templates/build/build-extension.yml" template file instead. +tasks: + - name: "build-php-7.4" + tags: ["build", "php7.4", "stable"] + commands: + - func: "locate PHP binaries" + vars: + PHP_VERSION: "7.4" + - func: "compile extension" + # TODO: Remove this variable once PHPC 1.17.0 is released + vars: + EXTENSION_BRANCH: "master" + - func: "upload extension" +# TODO: Re-enable these tasks once PHPC 1.17.0 is released +# - name: "build-php-7.4-lowest" +# tags: ["build", "php7.4", "lowest"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "7.4" +# - func: "compile extension" +# vars: +# EXTENSION_VERSION: "1.17.0" +# - func: "upload extension" +# - name: "build-php-7.4-next-stable" +# tags: ["build", "php7.4", "next-stable"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "7.4" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "v1.17" +# - func: "upload extension" +# - name: "build-php-7.4-next-minor" +# tags: ["build", "php7.4", "next-minor"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "7.4" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "master" +# - func: "upload extension" diff --git a/.evergreen/config/generated/build/build-php-8.0.yml b/.evergreen/config/generated/build/build-php-8.0.yml new file mode 100644 index 000000000..c34b722ec --- /dev/null +++ b/.evergreen/config/generated/build/build-php-8.0.yml @@ -0,0 +1,44 @@ +# This file is generated automatically - please edit the "templates/build/build-extension.yml" template file instead. +tasks: + - name: "build-php-8.0" + tags: ["build", "php8.0", "stable"] + commands: + - func: "locate PHP binaries" + vars: + PHP_VERSION: "8.0" + - func: "compile extension" + # TODO: Remove this variable once PHPC 1.17.0 is released + vars: + EXTENSION_BRANCH: "master" + - func: "upload extension" +# TODO: Re-enable these tasks once PHPC 1.17.0 is released +# - name: "build-php-8.0-lowest" +# tags: ["build", "php8.0", "lowest"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.0" +# - func: "compile extension" +# vars: +# EXTENSION_VERSION: "1.17.0" +# - func: "upload extension" +# - name: "build-php-8.0-next-stable" +# tags: ["build", "php8.0", "next-stable"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.0" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "v1.17" +# - func: "upload extension" +# - name: "build-php-8.0-next-minor" +# tags: ["build", "php8.0", "next-minor"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.0" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "master" +# - func: "upload extension" diff --git a/.evergreen/config/generated/build/build-php-8.1.yml b/.evergreen/config/generated/build/build-php-8.1.yml new file mode 100644 index 000000000..912f0b5c1 --- /dev/null +++ b/.evergreen/config/generated/build/build-php-8.1.yml @@ -0,0 +1,44 @@ +# This file is generated automatically - please edit the "templates/build/build-extension.yml" template file instead. +tasks: + - name: "build-php-8.1" + tags: ["build", "php8.1", "stable"] + commands: + - func: "locate PHP binaries" + vars: + PHP_VERSION: "8.1" + - func: "compile extension" + # TODO: Remove this variable once PHPC 1.17.0 is released + vars: + EXTENSION_BRANCH: "master" + - func: "upload extension" +# TODO: Re-enable these tasks once PHPC 1.17.0 is released +# - name: "build-php-8.1-lowest" +# tags: ["build", "php8.1", "lowest"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.1" +# - func: "compile extension" +# vars: +# EXTENSION_VERSION: "1.17.0" +# - func: "upload extension" +# - name: "build-php-8.1-next-stable" +# tags: ["build", "php8.1", "next-stable"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.1" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "v1.17" +# - func: "upload extension" +# - name: "build-php-8.1-next-minor" +# tags: ["build", "php8.1", "next-minor"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.1" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "master" +# - func: "upload extension" diff --git a/.evergreen/config/generated/build/build-php-8.2.yml b/.evergreen/config/generated/build/build-php-8.2.yml new file mode 100644 index 000000000..af7223515 --- /dev/null +++ b/.evergreen/config/generated/build/build-php-8.2.yml @@ -0,0 +1,44 @@ +# This file is generated automatically - please edit the "templates/build/build-extension.yml" template file instead. +tasks: + - name: "build-php-8.2" + tags: ["build", "php8.2", "stable"] + commands: + - func: "locate PHP binaries" + vars: + PHP_VERSION: "8.2" + - func: "compile extension" + # TODO: Remove this variable once PHPC 1.17.0 is released + vars: + EXTENSION_BRANCH: "master" + - func: "upload extension" +# TODO: Re-enable these tasks once PHPC 1.17.0 is released +# - name: "build-php-8.2-lowest" +# tags: ["build", "php8.2", "lowest"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.2" +# - func: "compile extension" +# vars: +# EXTENSION_VERSION: "1.17.0" +# - func: "upload extension" +# - name: "build-php-8.2-next-stable" +# tags: ["build", "php8.2", "next-stable"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.2" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "v1.17" +# - func: "upload extension" +# - name: "build-php-8.2-next-minor" +# tags: ["build", "php8.2", "next-minor"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "8.2" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "master" +# - func: "upload extension" diff --git a/.evergreen/config/generated/test/csfle-4.2.yml b/.evergreen/config/generated/test/csfle-4.2.yml new file mode 100644 index 000000000..dda70b6c1 --- /dev/null +++ b/.evergreen/config/generated/test/csfle-4.2.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-4.2-crypt-shared" + tags: ["replicaset", "local", "4.2", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.2" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-4.2-mongocryptd" + tags: ["replicaset", "local", "4.2", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "4.2" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-4.2-no-aws-creds" + tags: ["replicaset", "local", "4.2", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.2" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/csfle-4.4.yml b/.evergreen/config/generated/test/csfle-4.4.yml new file mode 100644 index 000000000..ba58c8f0b --- /dev/null +++ b/.evergreen/config/generated/test/csfle-4.4.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-4.4-crypt-shared" + tags: ["replicaset", "local", "4.4", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.4" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-4.4-mongocryptd" + tags: ["replicaset", "local", "4.4", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "4.4" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-4.4-no-aws-creds" + tags: ["replicaset", "local", "4.4", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.4" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/csfle-5.0.yml b/.evergreen/config/generated/test/csfle-5.0.yml new file mode 100644 index 000000000..8088b15ef --- /dev/null +++ b/.evergreen/config/generated/test/csfle-5.0.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-5.0-crypt-shared" + tags: ["replicaset", "local", "5.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-5.0-mongocryptd" + tags: ["replicaset", "local", "5.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-5.0-no-aws-creds" + tags: ["replicaset", "local", "5.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/csfle-6.0.yml b/.evergreen/config/generated/test/csfle-6.0.yml new file mode 100644 index 000000000..799aa62aa --- /dev/null +++ b/.evergreen/config/generated/test/csfle-6.0.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-6.0-crypt-shared" + tags: ["replicaset", "local", "6.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-6.0-mongocryptd" + tags: ["replicaset", "local", "6.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-6.0-no-aws-creds" + tags: ["replicaset", "local", "6.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/csfle-7.0.yml b/.evergreen/config/generated/test/csfle-7.0.yml new file mode 100644 index 000000000..c34d0f7e2 --- /dev/null +++ b/.evergreen/config/generated/test/csfle-7.0.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-7.0-crypt-shared" + tags: ["replicaset", "local", "7.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-7.0-mongocryptd" + tags: ["replicaset", "local", "7.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-7.0-no-aws-creds" + tags: ["replicaset", "local", "7.0", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/csfle-latest.yml b/.evergreen/config/generated/test/csfle-latest.yml new file mode 100644 index 000000000..aaea2ff51 --- /dev/null +++ b/.evergreen/config/generated/test/csfle-latest.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-latest-crypt-shared" + tags: ["replicaset", "local", "latest", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-latest-mongocryptd" + tags: ["replicaset", "local", "latest", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-latest-no-aws-creds" + tags: ["replicaset", "local", "latest", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/csfle-rapid.yml b/.evergreen/config/generated/test/csfle-rapid.yml new file mode 100644 index 000000000..9be8bb7c4 --- /dev/null +++ b/.evergreen/config/generated/test/csfle-rapid.yml @@ -0,0 +1,42 @@ +# This file is generated automatically - please edit the "templates/test/csfle.yml" template file instead. +tasks: + - name: "test-mongodb-rapid-crypt-shared" + tags: ["replicaset", "local", "rapid", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-rapid-mongocryptd" + tags: ["replicaset", "local", "rapid", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-rapid-no-aws-creds" + tags: ["replicaset", "local", "rapid", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/generated/test/load-balanced-5.0.yml b/.evergreen/config/generated/test/load-balanced-5.0.yml new file mode 100644 index 000000000..298fd1d9b --- /dev/null +++ b/.evergreen/config/generated/test/load-balanced-5.0.yml @@ -0,0 +1,18 @@ +# This file is generated automatically - please edit the "templates/test/load-balanced.yml" template file instead. +tasks: + - name: "test-mongodb-5.0-loadbalanced" + tags: ["loadbalanced", "local", "5.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "5.0" + LOAD_BALANCER: "true" + SSL: "yes" + - func: "start load balancer" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" + SSL: "yes" diff --git a/.evergreen/config/generated/test/load-balanced-6.0.yml b/.evergreen/config/generated/test/load-balanced-6.0.yml new file mode 100644 index 000000000..08dc85242 --- /dev/null +++ b/.evergreen/config/generated/test/load-balanced-6.0.yml @@ -0,0 +1,18 @@ +# This file is generated automatically - please edit the "templates/test/load-balanced.yml" template file instead. +tasks: + - name: "test-mongodb-6.0-loadbalanced" + tags: ["loadbalanced", "local", "6.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "6.0" + LOAD_BALANCER: "true" + SSL: "yes" + - func: "start load balancer" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" + SSL: "yes" diff --git a/.evergreen/config/generated/test/load-balanced-7.0.yml b/.evergreen/config/generated/test/load-balanced-7.0.yml new file mode 100644 index 000000000..cfe28ac40 --- /dev/null +++ b/.evergreen/config/generated/test/load-balanced-7.0.yml @@ -0,0 +1,18 @@ +# This file is generated automatically - please edit the "templates/test/load-balanced.yml" template file instead. +tasks: + - name: "test-mongodb-7.0-loadbalanced" + tags: ["loadbalanced", "local", "7.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "7.0" + LOAD_BALANCER: "true" + SSL: "yes" + - func: "start load balancer" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" + SSL: "yes" diff --git a/.evergreen/config/generated/test/load-balanced-latest.yml b/.evergreen/config/generated/test/load-balanced-latest.yml new file mode 100644 index 000000000..16b197357 --- /dev/null +++ b/.evergreen/config/generated/test/load-balanced-latest.yml @@ -0,0 +1,18 @@ +# This file is generated automatically - please edit the "templates/test/load-balanced.yml" template file instead. +tasks: + - name: "test-mongodb-latest-loadbalanced" + tags: ["loadbalanced", "local", "latest"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "latest" + LOAD_BALANCER: "true" + SSL: "yes" + - func: "start load balancer" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" + SSL: "yes" diff --git a/.evergreen/config/generated/test/load-balanced-rapid.yml b/.evergreen/config/generated/test/load-balanced-rapid.yml new file mode 100644 index 000000000..adc8dc096 --- /dev/null +++ b/.evergreen/config/generated/test/load-balanced-rapid.yml @@ -0,0 +1,18 @@ +# This file is generated automatically - please edit the "templates/test/load-balanced.yml" template file instead. +tasks: + - name: "test-mongodb-rapid-loadbalanced" + tags: ["loadbalanced", "local", "rapid"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "rapid" + LOAD_BALANCER: "true" + SSL: "yes" + - func: "start load balancer" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" + SSL: "yes" diff --git a/.evergreen/config/generated/test/local-3.6.yml b/.evergreen/config/generated/test/local-3.6.yml new file mode 100644 index 000000000..377240fc8 --- /dev/null +++ b/.evergreen/config/generated/test/local-3.6.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-3.6-standalone-noauth-nossl" + tags: ["standalone", "local", "3.6"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "3.6" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-3.6-replicaset-noauth-nossl" + tags: ["replicaset", "local", "3.6"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "3.6" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-3.6-sharded-noauth-nossl" + tags: ["sharded", "local", "3.6"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "3.6" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-4.0.yml b/.evergreen/config/generated/test/local-4.0.yml new file mode 100644 index 000000000..6a12e4d68 --- /dev/null +++ b/.evergreen/config/generated/test/local-4.0.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-4.0-standalone-noauth-nossl" + tags: ["standalone", "local", "4.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "4.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-4.0-replicaset-noauth-nossl" + tags: ["replicaset", "local", "4.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-4.0-sharded-noauth-nossl" + tags: ["sharded", "local", "4.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "4.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-4.2.yml b/.evergreen/config/generated/test/local-4.2.yml new file mode 100644 index 000000000..56e6fd013 --- /dev/null +++ b/.evergreen/config/generated/test/local-4.2.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-4.2-standalone-noauth-nossl" + tags: ["standalone", "local", "4.2"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "4.2" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-4.2-replicaset-noauth-nossl" + tags: ["replicaset", "local", "4.2"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.2" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-4.2-sharded-noauth-nossl" + tags: ["sharded", "local", "4.2"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "4.2" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-4.4.yml b/.evergreen/config/generated/test/local-4.4.yml new file mode 100644 index 000000000..5b5c31bdc --- /dev/null +++ b/.evergreen/config/generated/test/local-4.4.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-4.4-standalone-noauth-nossl" + tags: ["standalone", "local", "4.4"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "4.4" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-4.4-replicaset-noauth-nossl" + tags: ["replicaset", "local", "4.4"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "4.4" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-4.4-sharded-noauth-nossl" + tags: ["sharded", "local", "4.4"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "4.4" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-5.0.yml b/.evergreen/config/generated/test/local-5.0.yml new file mode 100644 index 000000000..4dfbe6ae2 --- /dev/null +++ b/.evergreen/config/generated/test/local-5.0.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-5.0-standalone-noauth-nossl" + tags: ["standalone", "local", "5.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-5.0-replicaset-noauth-nossl" + tags: ["replicaset", "local", "5.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-5.0-sharded-noauth-nossl" + tags: ["sharded", "local", "5.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-6.0.yml b/.evergreen/config/generated/test/local-6.0.yml new file mode 100644 index 000000000..f9ce1e68c --- /dev/null +++ b/.evergreen/config/generated/test/local-6.0.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-6.0-standalone-noauth-nossl" + tags: ["standalone", "local", "6.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-6.0-replicaset-noauth-nossl" + tags: ["replicaset", "local", "6.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-6.0-sharded-noauth-nossl" + tags: ["sharded", "local", "6.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-7.0.yml b/.evergreen/config/generated/test/local-7.0.yml new file mode 100644 index 000000000..77a5c9696 --- /dev/null +++ b/.evergreen/config/generated/test/local-7.0.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-7.0-standalone-noauth-nossl" + tags: ["standalone", "local", "7.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-7.0-replicaset-noauth-nossl" + tags: ["replicaset", "local", "7.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-7.0-sharded-noauth-nossl" + tags: ["sharded", "local", "7.0"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-latest.yml b/.evergreen/config/generated/test/local-latest.yml new file mode 100644 index 000000000..be2b0611e --- /dev/null +++ b/.evergreen/config/generated/test/local-latest.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-latest-standalone-noauth-nossl" + tags: ["standalone", "local", "latest"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-latest-replicaset-noauth-nossl" + tags: ["replicaset", "local", "latest"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-latest-sharded-noauth-nossl" + tags: ["sharded", "local", "latest"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/local-rapid.yml b/.evergreen/config/generated/test/local-rapid.yml new file mode 100644 index 000000000..339e1ff3a --- /dev/null +++ b/.evergreen/config/generated/test/local-rapid.yml @@ -0,0 +1,34 @@ +# This file is generated automatically - please edit the "templates/test/local.yml" template file instead. +tasks: + - name: "test-mongodb-rapid-standalone-noauth-nossl" + tags: ["standalone", "local", "rapid"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-rapid-replicaset-noauth-nossl" + tags: ["replicaset", "local", "rapid"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-rapid-sharded-noauth-nossl" + tags: ["sharded", "local", "rapid"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/generated/test/require-api-version-5.0.yml b/.evergreen/config/generated/test/require-api-version-5.0.yml new file mode 100644 index 000000000..0a2a20fd2 --- /dev/null +++ b/.evergreen/config/generated/test/require-api-version-5.0.yml @@ -0,0 +1,30 @@ +# This file is generated automatically - please edit the "templates/test/require-api-version.yml" template file instead. +tasks: + - name: "test-requireApiVersion-5.0" + tags: ["standalone", "local", "5.0", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + AUTH: "auth" + REQUIRE_API_VERSION: "yes" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + API_VERSION: "1" + + - name: "test-acceptApiVersion2-5.0" + tags: ["standalone", "local", "5.0", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + ORCHESTRATION_FILE: "versioned-api-testing.json" + MONGODB_VERSION: "5.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "versioned-api" diff --git a/.evergreen/config/generated/test/require-api-version-6.0.yml b/.evergreen/config/generated/test/require-api-version-6.0.yml new file mode 100644 index 000000000..8af0b8c1a --- /dev/null +++ b/.evergreen/config/generated/test/require-api-version-6.0.yml @@ -0,0 +1,30 @@ +# This file is generated automatically - please edit the "templates/test/require-api-version.yml" template file instead. +tasks: + - name: "test-requireApiVersion-6.0" + tags: ["standalone", "local", "6.0", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + AUTH: "auth" + REQUIRE_API_VERSION: "yes" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + API_VERSION: "1" + + - name: "test-acceptApiVersion2-6.0" + tags: ["standalone", "local", "6.0", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + ORCHESTRATION_FILE: "versioned-api-testing.json" + MONGODB_VERSION: "6.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "versioned-api" diff --git a/.evergreen/config/generated/test/require-api-version-7.0.yml b/.evergreen/config/generated/test/require-api-version-7.0.yml new file mode 100644 index 000000000..74e747b7a --- /dev/null +++ b/.evergreen/config/generated/test/require-api-version-7.0.yml @@ -0,0 +1,30 @@ +# This file is generated automatically - please edit the "templates/test/require-api-version.yml" template file instead. +tasks: + - name: "test-requireApiVersion-7.0" + tags: ["standalone", "local", "7.0", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + AUTH: "auth" + REQUIRE_API_VERSION: "yes" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + API_VERSION: "1" + + - name: "test-acceptApiVersion2-7.0" + tags: ["standalone", "local", "7.0", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + ORCHESTRATION_FILE: "versioned-api-testing.json" + MONGODB_VERSION: "7.0" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "versioned-api" diff --git a/.evergreen/config/generated/test/require-api-version-latest.yml b/.evergreen/config/generated/test/require-api-version-latest.yml new file mode 100644 index 000000000..d41d54db7 --- /dev/null +++ b/.evergreen/config/generated/test/require-api-version-latest.yml @@ -0,0 +1,30 @@ +# This file is generated automatically - please edit the "templates/test/require-api-version.yml" template file instead. +tasks: + - name: "test-requireApiVersion-latest" + tags: ["standalone", "local", "latest", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + AUTH: "auth" + REQUIRE_API_VERSION: "yes" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + API_VERSION: "1" + + - name: "test-acceptApiVersion2-latest" + tags: ["standalone", "local", "latest", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + ORCHESTRATION_FILE: "versioned-api-testing.json" + MONGODB_VERSION: "latest" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "versioned-api" diff --git a/.evergreen/config/generated/test/require-api-version-rapid.yml b/.evergreen/config/generated/test/require-api-version-rapid.yml new file mode 100644 index 000000000..bd9015299 --- /dev/null +++ b/.evergreen/config/generated/test/require-api-version-rapid.yml @@ -0,0 +1,30 @@ +# This file is generated automatically - please edit the "templates/test/require-api-version.yml" template file instead. +tasks: + - name: "test-requireApiVersion-rapid" + tags: ["standalone", "local", "rapid", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + AUTH: "auth" + REQUIRE_API_VERSION: "yes" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + API_VERSION: "1" + + - name: "test-acceptApiVersion2-rapid" + tags: ["standalone", "local", "rapid", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + ORCHESTRATION_FILE: "versioned-api-testing.json" + MONGODB_VERSION: "rapid" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "versioned-api" diff --git a/.evergreen/config/templates/build/build-extension.yml b/.evergreen/config/templates/build/build-extension.yml new file mode 100644 index 000000000..9785cfa57 --- /dev/null +++ b/.evergreen/config/templates/build/build-extension.yml @@ -0,0 +1,43 @@ +tasks: + - name: "build-php-%phpVersion%" + tags: ["build", "php%phpVersion%", "stable"] + commands: + - func: "locate PHP binaries" + vars: + PHP_VERSION: "%phpVersion%" + - func: "compile extension" + # TODO: Remove this variable once PHPC 1.17.0 is released + vars: + EXTENSION_BRANCH: "master" + - func: "upload extension" +# TODO: Re-enable these tasks once PHPC 1.17.0 is released +# - name: "build-php-%phpVersion%-lowest" +# tags: ["build", "php%phpVersion%", "lowest"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "%phpVersion%" +# - func: "compile extension" +# vars: +# EXTENSION_VERSION: "1.17.0" +# - func: "upload extension" +# - name: "build-php-%phpVersion%-next-stable" +# tags: ["build", "php%phpVersion%", "next-stable"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "%phpVersion%" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "v1.17" +# - func: "upload extension" +# - name: "build-php-%phpVersion%-next-minor" +# tags: ["build", "php%phpVersion%", "next-minor"] +# commands: +# - func: "locate PHP binaries" +# vars: +# PHP_VERSION: "%phpVersion%" +# - func: "compile extension" +# vars: +# EXTENSION_BRANCH: "master" +# - func: "upload extension" diff --git a/.evergreen/config/templates/test/csfle.yml b/.evergreen/config/templates/test/csfle.yml new file mode 100644 index 000000000..b256c4d24 --- /dev/null +++ b/.evergreen/config/templates/test/csfle.yml @@ -0,0 +1,41 @@ +tasks: + - name: "test-mongodb-%mongodbVersion%-crypt-shared" + tags: ["replicaset", "local", "%mongodbVersion%", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-%mongodbVersion%-mongocryptd" + tags: ["replicaset", "local", "%mongodbVersion%", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + SKIP_CRYPT_SHARED: "yes" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "csfle" + + - name: "test-mongodb-%mongodbVersion%-no-aws-creds" + tags: ["replicaset", "local", "%mongodbVersion%", "csfle"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "run tests" + vars: + client_side_encryption_aws_access_key_id: "" + client_side_encryption_aws_secret_access_key: "" + TESTS: "csfle-without-aws-creds" diff --git a/.evergreen/config/templates/test/load-balanced.yml b/.evergreen/config/templates/test/load-balanced.yml new file mode 100644 index 000000000..ca36912ad --- /dev/null +++ b/.evergreen/config/templates/test/load-balanced.yml @@ -0,0 +1,17 @@ +tasks: + - name: "test-mongodb-%mongodbVersion%-loadbalanced" + tags: ["loadbalanced", "local", "%mongodbVersion%"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "%mongodbVersion%" + LOAD_BALANCER: "true" + SSL: "yes" + - func: "start load balancer" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + MONGODB_URI: "${SINGLE_MONGOS_LB_URI}" + SSL: "yes" diff --git a/.evergreen/config/templates/test/local.yml b/.evergreen/config/templates/test/local.yml new file mode 100644 index 000000000..462c7d4f6 --- /dev/null +++ b/.evergreen/config/templates/test/local.yml @@ -0,0 +1,33 @@ +tasks: + - name: "test-mongodb-%mongodbVersion%-standalone-noauth-nossl" + tags: ["standalone", "local", "%mongodbVersion%"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-%mongodbVersion%-replicaset-noauth-nossl" + tags: ["replicaset", "local", "%mongodbVersion%"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "replica_set" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + + - name: "test-mongodb-%mongodbVersion%-sharded-noauth-nossl" + tags: ["sharded", "local", "%mongodbVersion%"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "sharded_cluster" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" diff --git a/.evergreen/config/templates/test/require-api-version.yml b/.evergreen/config/templates/test/require-api-version.yml new file mode 100644 index 000000000..c567eceee --- /dev/null +++ b/.evergreen/config/templates/test/require-api-version.yml @@ -0,0 +1,29 @@ +tasks: + - name: "test-requireApiVersion-%mongodbVersion%" + tags: ["standalone", "local", "%mongodbVersion%", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + AUTH: "auth" + REQUIRE_API_VERSION: "yes" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + API_VERSION: "1" + + - name: "test-acceptApiVersion2-%mongodbVersion%" + tags: ["standalone", "local", "%mongodbVersion%", "versioned_api"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + TOPOLOGY: "server" + ORCHESTRATION_FILE: "versioned-api-testing.json" + MONGODB_VERSION: "%mongodbVersion%" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run tests" + vars: + TESTS: "versioned-api" diff --git a/.evergreen/config/test-task-groups.yml b/.evergreen/config/test-task-groups.yml new file mode 100644 index 000000000..fd5d1af26 --- /dev/null +++ b/.evergreen/config/test-task-groups.yml @@ -0,0 +1,36 @@ +task_groups: + - name: test_atlas_task_group + setup_group: + - func: "fetch source" + - func: "prepare resources" + - func: "fix absolute paths" + - func: "install dependencies" + - func: "locate PHP binaries" + - func: "fetch extension" + - func: "install composer" + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + env: + MONGODB_VERSION: '7.0' + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + - func: "upload test results" + - func: "cleanup" + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - test-atlas diff --git a/.evergreen/config/test-tasks.yml b/.evergreen/config/test-tasks.yml new file mode 100644 index 000000000..b708ec9cf --- /dev/null +++ b/.evergreen/config/test-tasks.yml @@ -0,0 +1,20 @@ +tasks: + - name: "test-atlas" + commands: + - func: "start kms servers" + - func: "run tests" + vars: + TESTS: "atlas" + + - name: "test-serverless" + tags: ["serverless"] + commands: + - func: "create serverless instance" + - func: "start kms servers" + - func: "set aws temp creds" + - func: "run serverless tests" + + - name: "test-atlas-data-lake" + commands: + - func: "bootstrap mongohoused" + - func: "run atlas data lake test" diff --git a/.evergreen/config/test-variants.yml b/.evergreen/config/test-variants.yml new file mode 100644 index 000000000..bc7d8f1e5 --- /dev/null +++ b/.evergreen/config/test-variants.yml @@ -0,0 +1,116 @@ +# +# Test variants - only run for select platforms worth testing on +# +buildvariants: + # PHP 8.2: test all topologies on all versions of MongoDB + - name: test-debian11-php82-local + tags: ["test", "debian", "x64"] + display_name: "Test: Debian 11, PHP 8.2" + run_on: debian11-small + expansions: + FETCH_BUILD_VARIANT: "build-debian11" + FETCH_BUILD_TASK: "build-php-8.2" + PHP_VERSION: "8.2" + VARIANT: debian11 # Referenced by ADL build script for downloading MQLRun + depends_on: + - variant: "build-debian11" + name: "build-php-8.2" + tasks: + - ".standalone .local !.csfle !.3.6 !.4.0 !.4.2 !.4.4 !.5.0" + - ".replicaset .local !.csfle !.3.6 !.4.0 !.4.2 !.4.4 !.5.0" + - ".sharded .local !.csfle !.3.6 !.4.0 !.4.2 !.4.4 !.5.0" + - ".loadbalanced .local !.csfle !.3.6 !.4.0 !.4.2 !.4.4 !.5.0" + - ".serverless" + - "test-atlas-data-lake" + - name: test-debian92-php82-local + tags: ["test", "debian", "x64"] + display_name: "Test: Debian 9.2, PHP 8.2" + run_on: debian92-small + expansions: + FETCH_BUILD_VARIANT: "build-debian92" + FETCH_BUILD_TASK: "build-php-8.2" + PHP_VERSION: "8.2" + depends_on: + - variant: "build-debian92" + name: "build-php-8.2" + tasks: + # Remember to add new major versions here as they are released + - ".standalone .local !.csfle !.6.0 !.7.0 !.rapid !.latest" + - ".replicaset .local !.csfle !.6.0 !.7.0 !.rapid !.latest" + - ".sharded .local !.csfle !.6.0 !.7.0 !.rapid !.latest" + - ".loadbalanced .local !.csfle !.6.0 !.7.0 !.rapid !.latest" + + # Test remaining PHP versions with replica sets on Debian 11 with MongoDB 7.0 + - name: test-debian11-php81-local + tags: ["test", "debian", "x64"] + display_name: "Test: Debian 11, PHP 8.1" + run_on: debian11-small + expansions: + FETCH_BUILD_VARIANT: "build-debian11" + FETCH_BUILD_TASK: "build-php-8.1" + PHP_VERSION: "8.1" + depends_on: + - variant: "build-debian11" + name: "build-php-8.1" + tasks: + - ".replicaset .local .7.0 !.csfle" + - name: test-debian11-php80-local + tags: ["test", "debian", "x64"] + display_name: "Test: Debian 11, PHP 8.0" + run_on: debian11-small + expansions: + FETCH_BUILD_VARIANT: "build-debian11" + FETCH_BUILD_TASK: "build-php-8.0" + PHP_VERSION: "8.0" + depends_on: + - variant: "build-debian11" + name: "build-php-8.0" + tasks: + - ".replicaset .local .7.0 !.csfle" + - name: test-debian11-php74-local + tags: ["test", "debian", "x64"] + display_name: "Test: Debian 11, PHP 7.4" + run_on: debian11-small + expansions: + FETCH_BUILD_VARIANT: "build-debian11" + FETCH_BUILD_TASK: "build-php-7.4" + PHP_VERSION: "7.4" + depends_on: + - variant: "build-debian11" + name: "build-php-7.4" + tasks: + - ".replicaset .local .7.0 !.csfle" + + # Test with lowest PHP, PHPC, MongoDB, and dependency versions + - name: test-debian92-php74-local-lowest + tags: ["test", "debian", "x64"] + display_name: "Test: Debian 9.2, PHP 7.4, Lowest Dependencies" + run_on: debian92-small + expansions: + FETCH_BUILD_VARIANT: "build-debian92" + # TODO: Change to build-php-7.4-lowest once PHPC 1.17.0 is released + FETCH_BUILD_TASK: "build-php-7.4" + PHP_VERSION: "7.4" + DEPENDENCIES: "lowest" + depends_on: + - variant: "build-debian92" + # TODO: Change to build-php-7.4-lowest once PHPC 1.17.0 is released + name: "build-php-7.4" + tasks: + - ".replicaset .local .3.6 !.csfle" + + # Test Atlas and CSFLE on RHEL 8 + - name: test-rhel80-php82-atlas + tags: ["test", "debian", "x64"] + display_name: "Test: RHEL 8.0, PHP 8.2" + run_on: rhel80-small + expansions: + FETCH_BUILD_VARIANT: "build-rhel80" + FETCH_BUILD_TASK: "build-php-8.2" + PHP_VERSION: "8.2" + depends_on: + - variant: "build-rhel80" + name: "build-php-8.2" + tasks: + - "test_atlas_task_group" + - ".csfle" diff --git a/.evergreen/install-composer.sh b/.evergreen/install-composer.sh new file mode 100644 index 000000000..08f383e96 --- /dev/null +++ b/.evergreen/install-composer.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -o errexit # Exit the script with error if any of the commands fail + +# Supported environment variables +DEPENDENCIES=${DEPENDENCIES:-} # Specify "lowest" to prefer lowest dependencies +COMPOSER_FLAGS="${COMPOSER_FLAGS:-}" # Optional, additional Composer flags + +PATH="$PHP_PATH/bin:$PATH" + +install_composer () +{ + EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" + + if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; then + >&2 echo 'ERROR: Invalid installer checksum' + rm composer-setup.php + exit 1 + fi + + php composer-setup.php --quiet + rm composer-setup.php +} + +case "$DEPENDENCIES" in + lowest*) + COMPOSER_FLAGS="${COMPOSER_FLAGS} --prefer-lowest" + ;; + + *) + ;; +esac + +cp ${PROJECT_DIRECTORY}/.evergreen/config/php.ini ${PHP_PATH}/lib/php.ini + +php --ri mongodb + +install_composer + +php composer.phar update $COMPOSER_FLAGS diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 587558c7c..860df9d31 100644 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -1,84 +1,6 @@ #!/bin/sh set -o errexit # Exit the script with error if any of the commands fail -set_php_version () -{ - PHP_VERSION=$1 - - if [ ! -d "/opt/php" ]; then - echo "PHP is not available" - exit 1 - fi - - if [ -d "/opt/php/${PHP_VERSION}-64bit/bin" ]; then - export PHP_PATH="/opt/php/${PHP_VERSION}-64bit" - else - # Try to find the newest version matching our constant - export PHP_PATH=`find /opt/php/ -maxdepth 1 -type d -name "${PHP_VERSION}.*-64bit" -print | sort -V -r | head -1` - fi - - if [ ! -d "$PHP_PATH" ]; then - echo "Could not find PHP binaries for version ${PHP_VERSION}. Listing available versions..." - ls -1 /opt/php - exit 1 - fi - - echo 'PHP_PATH: "'$PHP_PATH'"' > php-expansion.yml - export PATH=$PHP_PATH/bin:$PATH -} - -install_extension () -{ - rm -f ${PHP_PATH}/lib/php.ini - - if [ "x${EXTENSION_BRANCH}" != "x" ] || [ "x${EXTENSION_REPO}" != "x" ]; then - CLONE_REPO=${EXTENSION_REPO:-https://github.com/mongodb/mongo-php-driver} - CHECKOUT_BRANCH=${EXTENSION_BRANCH:-master} - - echo "Compiling driver branch ${CHECKOUT_BRANCH} from repository ${CLONE_REPO}" - - mkdir -p /tmp/compile - rm -rf /tmp/compile/mongo-php-driver - git clone ${CLONE_REPO} /tmp/compile/mongo-php-driver - cd /tmp/compile/mongo-php-driver - - git checkout ${CHECKOUT_BRANCH} - git submodule update --init - phpize - ./configure --enable-mongodb-developer-flags - make all -j20 > /dev/null - make install - - cd ${PROJECT_DIRECTORY} - elif [ "x${EXTENSION_VERSION}" != "x" ]; then - echo "Installing driver version ${EXTENSION_VERSION} from PECL" - pecl install -f mongodb-${EXTENSION_VERSION} - else - echo "Installing latest driver version from PECL" - pecl install -f mongodb - fi - - sudo cp ${PROJECT_DIRECTORY}/.evergreen/config/php.ini ${PHP_PATH}/lib/php.ini - - php --ri mongodb -} - -install_composer () -{ - EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" - ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" - - if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; then - >&2 echo 'ERROR: Invalid installer checksum' - rm composer-setup.php - exit 1 - fi - - php composer-setup.php --quiet - rm composer-setup.php -} - # Functions to fetch MongoDB binaries . ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh OS=$(uname -s | tr '[:upper:]' '[:lower:]') @@ -112,18 +34,3 @@ case "$DISTRO" in echo "All other platforms..." ;; esac - -case "$DEPENDENCIES" in - lowest*) - COMPOSER_FLAGS="${COMPOSER_FLAGS} --prefer-lowest" - ;; - - *) - ;; -esac - -set_php_version $PHP_VERSION -install_extension -install_composer - -php composer.phar update $COMPOSER_FLAGS diff --git a/.gitattributes b/.gitattributes index 370c9e0d7..6a04e7492 100644 --- a/.gitattributes +++ b/.gitattributes @@ -14,3 +14,6 @@ phpunit.evergreen.xml export-ignore phpunit.xml.dist export-ignore psalm.xml.dist export-ignore psalm-baseline.xml export-ignore + +# Prevent generated build files from showing diffs in pull requests +.evergreen/config/generated/** linguist-generated=true diff --git a/tests/TestCase.php b/tests/TestCase.php index 4ecfe8cda..f349b47b5 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -70,7 +70,7 @@ public static function printConfiguration(): void $configuration = [ '%uri%' => $uri, '%database%' => static::getDatabaseName(), - '%apiVersion%' => getenv('MONGODB_API_VERSION') ?: 'Not configured', + '%apiVersion%' => getenv('API_VERSION') ?: 'Not configured', '%cryptSharedAvailable%' => FunctionalTestCase::isCryptSharedLibAvailable() ? 'Available' : 'Not available', '%mongocryptdAvailable%' => FunctionalTestCase::isMongocryptdAvailable() ? 'Available' : 'Not available', ];