From a0526f8c1e9701c457d0e59fe22333b9b514430e Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 14 Nov 2023 10:44:02 -0700 Subject: [PATCH] RUBY-3352 Backport RUBY-3268 atlas search index helpers to 2.19-stable (#2804) * RUBY-3268 search index management helpers (#2777) * RUBY-3268 Index View API for Search Indexes * documentation, and align method names with the spec * return the correct value here * suppress NamespaceNotFound errors for the drop operation * prose tests for search indexes * rubocop * prose tests pass * first stab at evergreen config for index management specs * rubocop * gah, executable permissions * don't use FLE for full atlas tests * make sure MONGODB_URI is set * set the timeout higher for the search index specs * pass all aggregation options through to the list indexes command * use the correct implementation for #empty? * remove unnecessary validation * bump drivers-evergreen-tools * RUBY-3324 bump drivers-evergreen-tools to get updated atlas setup/teardown (#2780) also, expose task_id expansion as environment variable to those scripts * RUBY-3328 add `execution` expansion to environment for atlas cluster name uniqueness (#2783) --- .evergreen/config.yml | 62 +++++ .evergreen/config/common.yml.erb | 54 ++++ .evergreen/config/standard.yml.erb | 8 + .evergreen/run-tests-atlas-full.sh | 24 ++ .gitignore | 5 + .mod/drivers-evergreen-tools | 2 +- .rubocop.yml | 3 + lib/mongo.rb | 1 + lib/mongo/collection.rb | 24 +- lib/mongo/operation.rb | 3 + lib/mongo/operation/create_search_indexes.rb | 15 ++ .../operation/create_search_indexes/op_msg.rb | 31 +++ lib/mongo/operation/drop_search_index.rb | 15 ++ .../operation/drop_search_index/op_msg.rb | 33 +++ lib/mongo/operation/shared/specifiable.rb | 7 + lib/mongo/operation/update_search_index.rb | 15 ++ .../operation/update_search_index/op_msg.rb | 34 +++ lib/mongo/search_index/view.rb | 232 ++++++++++++++++++ spec/atlas/atlas_connectivity_spec.rb | 6 +- spec/atlas/operations_spec.rb | 6 +- spec/integration/search_indexes_prose_spec.rb | 168 +++++++++++++ spec/lite_spec_helper.rb | 42 +++- .../unified/search_index_operations.rb | 63 +++++ spec/runners/unified/test.rb | 4 +- spec/spec_helper.rb | 2 +- .../index_management/createSearchIndex.yml | 62 +++++ .../index_management/createSearchIndexes.yml | 83 +++++++ .../data/index_management/dropSearchIndex.yml | 42 ++++ .../index_management/listSearchIndexes.yml | 85 +++++++ .../index_management/updateSearchIndex.yml | 45 ++++ .../index_management_unified_spec.rb | 13 + spec/support/spec_config.rb | 5 + 32 files changed, 1170 insertions(+), 24 deletions(-) create mode 100755 .evergreen/run-tests-atlas-full.sh create mode 100644 lib/mongo/operation/create_search_indexes.rb create mode 100644 lib/mongo/operation/create_search_indexes/op_msg.rb create mode 100644 lib/mongo/operation/drop_search_index.rb create mode 100644 lib/mongo/operation/drop_search_index/op_msg.rb create mode 100644 lib/mongo/operation/update_search_index.rb create mode 100644 lib/mongo/operation/update_search_index/op_msg.rb create mode 100644 lib/mongo/search_index/view.rb create mode 100644 spec/integration/search_indexes_prose_spec.rb create mode 100644 spec/runners/unified/search_index_operations.rb create mode 100644 spec/spec_tests/data/index_management/createSearchIndex.yml create mode 100644 spec/spec_tests/data/index_management/createSearchIndexes.yml create mode 100644 spec/spec_tests/data/index_management/dropSearchIndex.yml create mode 100644 spec/spec_tests/data/index_management/listSearchIndexes.yml create mode 100644 spec/spec_tests/data/index_management/updateSearchIndex.yml create mode 100644 spec/spec_tests/index_management_unified_spec.rb diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 27a1ab0ed2..bf5671c769 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -469,6 +469,50 @@ post: - func: "delete serverless instance" task_groups: + - name: testatlas_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: create expansions + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \ + DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + MONGODB_VERSION="7.0" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh + tasks: + - test-full-atlas-task + - name: testgcpkms_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes @@ -586,6 +630,16 @@ tasks: commands: - func: "export AWS auth credentials" - func: "run AWS auth tests" + - name: "test-full-atlas-task" + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh - name: "testgcpkms-task" commands: - command: shell.exec @@ -1612,6 +1666,14 @@ buildvariants: - name: testazurekms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - matrix_name: atlas-full + matrix_spec: + ruby: "ruby-3.2" + os: rhel8 + display_name: "Atlas (Full)" + tasks: + - name: testatlas_task_group + - matrix_name: "atlas" matrix_spec: ruby: ["ruby-3.2", "ruby-3.1", "ruby-3.0", "ruby-2.7", "ruby-2.6", "ruby-2.5", "jruby-9.3", "jruby-9.2"] diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 4febb453fa..3892f442dc 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -466,6 +466,50 @@ post: - func: "delete serverless instance" task_groups: + - name: testatlas_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: create expansions + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \ + DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + MONGODB_VERSION="7.0" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh + tasks: + - test-full-atlas-task + - name: testgcpkms_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes @@ -583,6 +627,16 @@ tasks: commands: - func: "export AWS auth credentials" - func: "run AWS auth tests" + - name: "test-full-atlas-task" + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh - name: "testgcpkms-task" commands: - command: shell.exec diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 67e2a92686..3bcd1b9acb 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -497,6 +497,14 @@ buildvariants: - name: testazurekms_task_group batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - matrix_name: atlas-full + matrix_spec: + ruby: <%= latest_ruby %> + os: rhel8 + display_name: "Atlas (Full)" + tasks: + - name: testatlas_task_group + - matrix_name: "atlas" matrix_spec: ruby: <%= supported_rubies %> diff --git a/.evergreen/run-tests-atlas-full.sh b/.evergreen/run-tests-atlas-full.sh new file mode 100755 index 0000000000..2fb15001c1 --- /dev/null +++ b/.evergreen/run-tests-atlas-full.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +. `dirname "$0"`/../spec/shared/shlib/distro.sh +. `dirname "$0"`/../spec/shared/shlib/set_env.sh +. `dirname "$0"`/functions.sh + +set_env_vars +set_env_python +set_env_ruby + +bundle_install + +ATLAS_URI=$MONGODB_URI \ + SERVERLESS=1 \ + EXAMPLE_TIMEOUT=600 \ + bundle exec rspec -fd spec/integration/search_indexes_prose_spec.rb + +test_status=$? + +kill_jruby + +exit ${test_status} diff --git a/.gitignore b/.gitignore index b127af9a1c..24a54bc470 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,8 @@ data/* gemfiles/*.gemfile.lock .env.private* .env +build +profile/benchmarking/data +secrets-export.sh +secrets-expansion.yml +atlas-expansion.yml diff --git a/.mod/drivers-evergreen-tools b/.mod/drivers-evergreen-tools index 6b328a119f..5da50374e8 160000 --- a/.mod/drivers-evergreen-tools +++ b/.mod/drivers-evergreen-tools @@ -1 +1 @@ -Subproject commit 6b328a119fa0ffdcd13e62b2c1c2259873a9076d +Subproject commit 5da50374e8feff236621be00ee60f0aff914c9e5 diff --git a/.rubocop.yml b/.rubocop.yml index 2b7a7f90f7..c76890187f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -56,6 +56,9 @@ Metrics/ModuleLength: Metrics/MethodLength: Max: 20 +Naming/MethodParameterName: + AllowedNames: [ id, op ] + RSpec/BeforeAfterAll: Enabled: false diff --git a/lib/mongo.rb b/lib/mongo.rb index 0f24833f06..b90cd4a011 100644 --- a/lib/mongo.rb +++ b/lib/mongo.rb @@ -64,6 +64,7 @@ require 'mongo/dbref' require 'mongo/grid' require 'mongo/index' +require 'mongo/search_index/view' require 'mongo/lint' require 'mongo/query_cache' require 'mongo/server' diff --git a/lib/mongo/collection.rb b/lib/mongo/collection.rb index 5b8d5af128..90514f203f 100644 --- a/lib/mongo/collection.rb +++ b/lib/mongo/collection.rb @@ -725,13 +725,35 @@ def distinct(field_name, filter = nil, options = {}) # # @option options [ Session ] :session The session to use. # - # @return [ View::Index ] The index view. + # @return [ Index::View ] The index view. # # @since 2.0.0 def indexes(options = {}) Index::View.new(self, options) end + # Get a view of all search indexes for this collection. Can be iterated or + # operated on directly. If id or name are given, the iterator will return + # only the indicated index. For all other operations, id and name are + # ignored. + # + # @note Only one of id or name may be given; it is an error to specify both, + # although both may be omitted safely. + # + # @param [ Hash ] options The options to use to configure the view. + # + # @option options [ String ] :id The id of the specific index to query (optional) + # @option options [ String ] :name The name of the specific index to query (optional) + # @option options [ Hash ] :aggregate The options hash to pass to the + # aggregate command (optional) + # + # @return [ SearchIndex::View ] The search index view. + # + # @since 2.0.0 + def search_indexes(options = {}) + SearchIndex::View.new(self, options) + end + # Get a pretty printed string inspection for the collection. # # @example Inspect the collection. diff --git a/lib/mongo/operation.rb b/lib/mongo/operation.rb index b9b24968ab..8def25dbe3 100644 --- a/lib/mongo/operation.rb +++ b/lib/mongo/operation.rb @@ -51,6 +51,9 @@ require 'mongo/operation/remove_user' require 'mongo/operation/create_index' require 'mongo/operation/drop_index' +require 'mongo/operation/create_search_indexes' +require 'mongo/operation/drop_search_index' +require 'mongo/operation/update_search_index' module Mongo diff --git a/lib/mongo/operation/create_search_indexes.rb b/lib/mongo/operation/create_search_indexes.rb new file mode 100644 index 0000000000..1b07ac4241 --- /dev/null +++ b/lib/mongo/operation/create_search_indexes.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'mongo/operation/create_search_indexes/op_msg' + +module Mongo + module Operation + # A MongoDB createSearchIndexes command operation. + # + # @api private + class CreateSearchIndexes + include Specifiable + include OpMsgExecutable + end + end +end diff --git a/lib/mongo/operation/create_search_indexes/op_msg.rb b/lib/mongo/operation/create_search_indexes/op_msg.rb new file mode 100644 index 0000000000..444d35721b --- /dev/null +++ b/lib/mongo/operation/create_search_indexes/op_msg.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Mongo + module Operation + class CreateSearchIndexes + # A MongoDB createSearchIndexes operation sent as an op message. + # + # @api private + class OpMsg < OpMsgBase + include ExecutableTransactionLabel + + private + + # Returns the command to send to the database, describing the + # desired createSearchIndexes operation. + # + # @param [ Mongo::Server ] _server the server that will receive the + # command + # + # @return [ Hash ] the selector + def selector(_server) + { + createSearchIndexes: coll_name, + :$db => db_name, + indexes: indexes, + } + end + end + end + end +end diff --git a/lib/mongo/operation/drop_search_index.rb b/lib/mongo/operation/drop_search_index.rb new file mode 100644 index 0000000000..8a12a06fc2 --- /dev/null +++ b/lib/mongo/operation/drop_search_index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'mongo/operation/drop_search_index/op_msg' + +module Mongo + module Operation + # A MongoDB dropSearchIndex command operation. + # + # @api private + class DropSearchIndex + include Specifiable + include OpMsgExecutable + end + end +end diff --git a/lib/mongo/operation/drop_search_index/op_msg.rb b/lib/mongo/operation/drop_search_index/op_msg.rb new file mode 100644 index 0000000000..8f4d323c55 --- /dev/null +++ b/lib/mongo/operation/drop_search_index/op_msg.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Mongo + module Operation + class DropSearchIndex + # A MongoDB createSearchIndexes operation sent as an op message. + # + # @api private + class OpMsg < OpMsgBase + include ExecutableTransactionLabel + + private + + # Returns the command to send to the database, describing the + # desired dropSearchIndex operation. + # + # @param [ Mongo::Server ] _server the server that will receive the + # command + # + # @return [ Hash ] the selector + def selector(_server) + { + dropSearchIndex: coll_name, + :$db => db_name, + }.tap do |sel| + sel[:id] = index_id if index_id + sel[:name] = index_name if index_name + end + end + end + end + end +end diff --git a/lib/mongo/operation/shared/specifiable.rb b/lib/mongo/operation/shared/specifiable.rb index 0a19dba1f6..afc799f46e 100644 --- a/lib/mongo/operation/shared/specifiable.rb +++ b/lib/mongo/operation/shared/specifiable.rb @@ -260,6 +260,13 @@ def index spec[INDEX] end + # Get the index id from the spec. + # + # @return [ String ] The index id. + def index_id + spec[:index_id] + end + # Get the index name from the spec. # # @example Get the index name. diff --git a/lib/mongo/operation/update_search_index.rb b/lib/mongo/operation/update_search_index.rb new file mode 100644 index 0000000000..05d8155bb1 --- /dev/null +++ b/lib/mongo/operation/update_search_index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'mongo/operation/update_search_index/op_msg' + +module Mongo + module Operation + # A MongoDB updateSearchIndex command operation. + # + # @api private + class UpdateSearchIndex + include Specifiable + include OpMsgExecutable + end + end +end diff --git a/lib/mongo/operation/update_search_index/op_msg.rb b/lib/mongo/operation/update_search_index/op_msg.rb new file mode 100644 index 0000000000..c6d21aaf0d --- /dev/null +++ b/lib/mongo/operation/update_search_index/op_msg.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Mongo + module Operation + class UpdateSearchIndex + # A MongoDB updateSearchIndex operation sent as an op message. + # + # @api private + class OpMsg < OpMsgBase + include ExecutableTransactionLabel + + private + + # Returns the command to send to the database, describing the + # desired updateSearchIndex operation. + # + # @param [ Mongo::Server ] _server the server that will receive the + # command + # + # @return [ Hash ] the selector + def selector(_server) + { + updateSearchIndex: coll_name, + :$db => db_name, + definition: index, + }.tap do |sel| + sel[:id] = index_id if index_id + sel[:name] = index_name if index_name + end + end + end + end + end +end diff --git a/lib/mongo/search_index/view.rb b/lib/mongo/search_index/view.rb new file mode 100644 index 0000000000..8c59771eab --- /dev/null +++ b/lib/mongo/search_index/view.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: true + +module Mongo + module SearchIndex + # A class representing a view of search indexes. + class View + include Enumerable + include Retryable + include Collection::Helpers + + # @return [ Mongo::Collection ] the collection this view belongs to + attr_reader :collection + + # @return [ nil | String ] the index id to query + attr_reader :requested_index_id + + # @return [ nil | String ] the index name to query + attr_reader :requested_index_name + + # @return [ Hash ] the options hash to use for the aggregate command + # when querying the available indexes. + attr_reader :aggregate_options + + # Create the new search index view. + # + # @param [ Collection ] collection The collection. + # @param [ Hash ] options The options that configure the behavior of the view. + # + # @option options [ String ] :id The specific index id to query (optional) + # @option options [ String ] :name The name of the specific index to query (optional) + # @option options [ Hash ] :aggregate The options hash to send to the + # aggregate command when querying the available indexes. + def initialize(collection, options = {}) + @collection = collection + @requested_index_id = options[:id] + @requested_index_name = options[:name] + @aggregate_options = options[:aggregate] || {} + + return if @aggregate_options.is_a?(Hash) + + raise ArgumentError, "The :aggregate option must be a Hash (got a #{@aggregate_options.class})" + end + + # Create a single search index with the given definition. If the name is + # provided, the new index will be given that name. + # + # @param [ Hash ] definition The definition of the search index. + # @param [ nil | String ] name The name to give the new search index. + # + # @return [ String ] the name of the new search index. + def create_one(definition, name: nil) + create_many([ { name: name, definition: definition } ]).first + end + + # Create multiple search indexes with a single command. + # + # @param [ Array ] indexes The description of the indexes to + # create. Each element of the list must be a hash with a definition + # key, and an optional name key. + # + # @return [ Array ] the names of the new search indexes. + def create_many(indexes) + spec = spec_with(indexes: indexes.map { |v| validate_search_index!(v) }) + result = Operation::CreateSearchIndexes.new(spec).execute(next_primary, context: execution_context) + result.first['indexesCreated'].map { |idx| idx['name'] } + end + + # Drop the search index with the given id, or name. One or the other must + # be specified, but not both. + # + # @param [ String ] id the id of the index to drop + # @param [ String ] name the name of the index to drop + # + # @return [ Mongo::Operation::Result | false ] the result of the + # operation, or false if the given index does not exist. + def drop_one(id: nil, name: nil) + validate_id_or_name!(id, name) + + spec = spec_with(index_id: id, index_name: name) + op = Operation::DropSearchIndex.new(spec) + + # per the spec: + # Drivers MUST suppress NamespaceNotFound errors for the + # ``dropSearchIndex`` helper. Drop operations should be idempotent. + do_drop(op, nil, execution_context) + end + + # Iterate over the search indexes. + # + # @param [ Proc ] block if given, each search index will be yieleded to + # the block. + # + # @return [ self | Enumerator ] if a block is given, self is returned. + # Otherwise, an enumerator will be returned. + def each(&block) + @result ||= begin + spec = {}.tap do |s| + s[:id] = requested_index_id if requested_index_id + s[:name] = requested_index_name if requested_index_name + end + + collection.aggregate( + [ { '$listSearchIndexes' => spec } ], + aggregate_options + ) + end + + return @result.to_enum unless block + + @result.each(&block) + self + end + + # Update the search index with the given id or name. One or the other + # must be provided, but not both. + # + # @param [ Hash ] definition the definition to replace the given search + # index with. + # @param [ nil | String ] id the id of the search index to update + # @param [ nil | String ] name the name of the search index to update + # + # @return [ Mongo::Operation::Result ] the result of the operation + def update_one(definition, id: nil, name: nil) + validate_id_or_name!(id, name) + + spec = spec_with(index_id: id, index_name: name, index: definition) + Operation::UpdateSearchIndex.new(spec).execute(next_primary, context: execution_context) + end + + # The following methods are to make the view act more like an array, + # without having to explicitly make it an array... + + # Queries whether the search index enumerable is empty. + # + # @return [ true | false ] whether the enumerable is empty or not. + def empty? + count.zero? + end + + private + + # A helper method for building the specification document with certain + # values pre-populated. + # + # @param [ Hash ] extras the values to put into the specification + # + # @return [ Hash ] the specification document + def spec_with(extras) + { + coll_name: collection.name, + db_name: collection.database.name, + }.merge(extras) + end + + # A helper method for retrieving the primary server from the cluster. + # + # @return [ Mongo::Server ] the server to use + def next_primary(ping = nil, session = nil) + collection.cluster.next_primary(ping, session) + end + + # A helper method for constructing a new operation context for executing + # an operation. + # + # @return [ Mongo::Operation::Context ] the operation context + def execution_context + Operation::Context.new(client: collection.client) + end + + # Validates the given id and name, ensuring that exactly one of them + # is non-nil. + # + # @param [ nil | String ] id the id to validate + # @param [ nil | String ] name the name to validate + # + # @raise [ ArgumentError ] if neither or both arguments are nil + def validate_id_or_name!(id, name) + return unless (id.nil? && name.nil?) || (!id.nil? && !name.nil?) + + raise ArgumentError, 'exactly one of id or name must be specified' + end + + # Validates the given search index document, ensuring that it has no + # extra keys, and that the name and definition are valid. + # + # @param [ Hash ] doc the document to validate + # + # @raise [ ArgumentError ] if the document is invalid. + def validate_search_index!(doc) + validate_search_index_keys!(doc.keys) + validate_search_index_name!(doc[:name] || doc['name']) + validate_search_index_definition!(doc[:definition] || doc['definition']) + doc + end + + # Validates the keys of a search index document, ensuring that + # they are all valid. + # + # @param [ Array ] keys the keys of a search index document + # + # @raise [ ArgumentError ] if the list contains any invalid keys + def validate_search_index_keys!(keys) + extras = keys - [ 'name', 'definition', :name, :definition ] + + raise ArgumentError, "invalid keys in search index creation: #{extras.inspect}" if extras.any? + end + + # Validates the name of a search index, ensuring that it is either a + # String or nil. + # + # @param [ nil | String ] name the name of a search index + # + # @raise [ ArgumentError ] if the name is not valid + def validate_search_index_name!(name) + return if name.nil? || name.is_a?(String) + + raise ArgumentError, "search index name must be nil or a string (got #{name.inspect})" + end + + # Validates the definition of a search index. + # + # @param [ Hash ] definition the definition of a search index + # + # @raise [ ArgumentError ] if the definition is not valid + def validate_search_index_definition!(definition) + return if definition.is_a?(Hash) + + raise ArgumentError, "search index definition must be a Hash (got #{definition.inspect})" + end + end + end +end diff --git a/spec/atlas/atlas_connectivity_spec.rb b/spec/atlas/atlas_connectivity_spec.rb index 9c519dacda..0412d1bd84 100644 --- a/spec/atlas/atlas_connectivity_spec.rb +++ b/spec/atlas/atlas_connectivity_spec.rb @@ -7,11 +7,7 @@ let(:uri) { ENV['ATLAS_URI'] } let(:client) { Mongo::Client.new(uri) } - before do - if uri.nil? - skip "ATLAS_URI not set in environment" - end - end + require_atlas describe 'connection to Atlas' do it 'runs ismaster successfully' do diff --git a/spec/atlas/operations_spec.rb b/spec/atlas/operations_spec.rb index 8ac9495e67..8a46ab3702 100644 --- a/spec/atlas/operations_spec.rb +++ b/spec/atlas/operations_spec.rb @@ -7,11 +7,7 @@ let(:uri) { ENV['ATLAS_URI'] } let(:client) { Mongo::Client.new(uri) } - before do - if uri.nil? - skip "ATLAS_URI not set in environment" - end - end + require_atlas describe 'ping' do it 'works' do diff --git a/spec/integration/search_indexes_prose_spec.rb b/spec/integration/search_indexes_prose_spec.rb new file mode 100644 index 0000000000..0a17accaf2 --- /dev/null +++ b/spec/integration/search_indexes_prose_spec.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +require 'spec_helper' + +class SearchIndexHelper + attr_reader :client, :collection_name + + def initialize(client) + @client = client + + # https://github.com/mongodb/specifications/blob/master/source/index-management/tests/README.rst#id4 + # "...each test uses a randomly generated collection name. Drivers may + # generate this collection name however they like, but a suggested + # implementation is a hex representation of an ObjectId..." + @collection_name = BSON::ObjectId.new.to_s + end + + # `soft_create` means to create the collection object without forcing it to + # be created in the database. + def collection(soft_create: false) + @collection ||= client.database[collection_name].tap do |collection| + collection.create unless soft_create + end + end + + # Wait for all of the indexes with the given names to be ready; then return + # the list of index definitions corresponding to those names. + def wait_for(*names, &condition) + timeboxed_wait do + result = collection.search_indexes + return filter_results(result, names) if names.all? { |name| ready?(result, name, &condition) } + end + end + + # Wait until all of the indexes with the given names are absent from the + # search index list. + def wait_for_absense_of(*names) + names.each do |name| + timeboxed_wait do + break if collection.search_indexes(name: name).empty? + end + end + end + + private + + def timeboxed_wait(step: 5, max: 300) + start = Mongo::Utils.monotonic_time + + loop do + yield + + sleep step + raise Timeout::Error, 'wait took too long' if Mongo::Utils.monotonic_time - start > max + end + end + + # Returns true if the list of search indexes includes one with the given name, + # which is ready to be queried. + def ready?(list, name, &condition) + condition ||= ->(index) { index['queryable'] } + list.any? { |index| index['name'] == name && condition[index] } + end + + def filter_results(result, names) + result.select { |index| names.include?(index['name']) } + end +end + +describe 'Mongo::Collection#search_indexes prose tests' do + # https://github.com/mongodb/specifications/blob/master/source/index-management/tests/README.rst#id5 + # "These tests must run against an Atlas cluster with a 7.0+ server." + require_atlas + + let(:client) do + Mongo::Client.new( + ENV['ATLAS_URI'], + database: SpecConfig.instance.test_db, + ssl: true, + ssl_verify: true + ) + end + + let(:helper) { SearchIndexHelper.new(client) } + + let(:name) { 'test-search-index' } + let(:definition) { { 'mappings' => { 'dynamic' => false } } } + let(:create_index) { helper.collection.search_indexes.create_one(definition, name: name) } + + # Case 1: Driver can successfully create and list search indexes + context 'when creating and listing search indexes' do + let(:index) { helper.wait_for(name).first } + + it 'succeeds' do + expect(create_index).to be == name + expect(index['latestDefinition']).to be == definition + end + end + + # Case 2: Driver can successfully create multiple indexes in batch + context 'when creating multiple indexes in batch' do + let(:specs) do + [ + { 'name' => 'test-search-index-1', 'definition' => definition }, + { 'name' => 'test-search-index-2', 'definition' => definition } + ] + end + + let(:names) { specs.map { |spec| spec['name'] } } + let(:create_indexes) { helper.collection.search_indexes.create_many(specs) } + + let(:indexes) { helper.wait_for(*names) } + + let(:index1) { indexes[0] } + let(:index2) { indexes[1] } + + it 'succeeds' do + expect(create_indexes).to be == names + expect(index1['latestDefinition']).to be == specs[0]['definition'] + expect(index2['latestDefinition']).to be == specs[1]['definition'] + end + end + + # Case 3: Driver can successfully drop search indexes + context 'when dropping search indexes' do + it 'succeeds' do + expect(create_index).to be == name + helper.wait_for(name) + + helper.collection.search_indexes.drop_one(name: name) + + expect { helper.wait_for_absense_of(name) }.not_to raise_error + end + end + + # Case 4: Driver can update a search index + context 'when updating search indexes' do + let(:new_definition) { { 'mappings' => { 'dynamic' => true } } } + + let(:index) do + helper + .wait_for(name) { |idx| idx['queryable'] && idx['status'] == 'READY' } + .first + end + + # rubocop:disable RSpec/ExampleLength + it 'succeeds' do + expect(create_index).to be == name + helper.wait_for(name) + + expect do + helper.collection.search_indexes.update_one(new_definition, name: name) + end.not_to raise_error + + expect(index['latestDefinition']).to be == new_definition + end + # rubocop:enable RSpec/ExampleLength + end + + # Case 5: dropSearchIndex suppresses namespace not found errors + context 'when dropping a non-existent search index' do + it 'ignores `namespace not found` errors' do + collection = helper.collection(soft_create: true) + expect { collection.search_indexes.drop_one(name: name) } + .not_to raise_error + end + end +end diff --git a/spec/lite_spec_helper.rb b/spec/lite_spec_helper.rb index 7bd6675a70..f7db224690 100644 --- a/spec/lite_spec_helper.rb +++ b/spec/lite_spec_helper.rb @@ -106,6 +106,31 @@ module Mrss class ExampleTimeout < StandardError; end +STANDARD_TIMEOUTS = { + stress: 210, + jruby: 90, + default: 45, +}.freeze + +def timeout_type + if ENV['EXAMPLE_TIMEOUT'].to_i > 0 + :custom + elsif %w(1 true yes).include?(ENV['STRESS']&.downcase) + :stress + elsif BSON::Environment.jruby? + :jruby + else + :default + end +end + +def example_timeout_seconds + STANDARD_TIMEOUTS.fetch( + timeout_type, + (ENV['EXAMPLE_TIMEOUT'] || STANDARD_TIMEOUTS[:default]).to_i + ) +end + RSpec.configure do |config| config.extend(CommonShortcuts::ClassMethods) config.include(CommonShortcuts::InstanceMethods) @@ -123,6 +148,12 @@ def require_solo end end + def require_atlas + before do + skip 'Set ATLAS_URI in environment to run atlas tests' if ENV['ATLAS_URI'].nil? + end + end + if SpecConfig.instance.ci? SdamFormatterIntegration.subscribe config.add_formatter(JsonExtFormatter, File.join(File.dirname(__FILE__), '../tmp/rspec.json')) @@ -141,16 +172,7 @@ def require_solo # Tests should take under 10 seconds ideally but it seems # we have some that run for more than 10 seconds in CI. config.around(:each) do |example| - timeout = if %w(1 true yes).include?(ENV['STRESS']&.downcase) - 210 - else - if BSON::Environment.jruby? - 90 - else - 45 - end - end - TimeoutInterrupt.timeout(timeout, ExampleTimeout) do + TimeoutInterrupt.timeout(example_timeout_seconds, ExampleTimeout) do example.run end end diff --git a/spec/runners/unified/search_index_operations.rb b/spec/runners/unified/search_index_operations.rb new file mode 100644 index 0000000000..d74ab57776 --- /dev/null +++ b/spec/runners/unified/search_index_operations.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Unified + # The definitions of available search index operations, as used by the + # unified tests. + module SearchIndexOperations + def create_search_index(op) + collection = entities.get(:collection, op.use!('object')) + + use_arguments(op) do |args| + model = args.use('model') + name = model.use('name') + definition = model.use('definition') + collection.search_indexes.create_one(definition, name: name) + end + end + + def create_search_indexes(op) + collection = entities.get(:collection, op.use!('object')) + + use_arguments(op) do |args| + models = args.use('models') + collection.search_indexes.create_many(models) + end + end + + def drop_search_index(op) + collection = entities.get(:collection, op.use!('object')) + + use_arguments(op) do |args| + collection.search_indexes.drop_one( + id: args.use('id'), + name: args.use('name') + ) + end + end + + def list_search_indexes(op) + collection = entities.get(:collection, op.use!('object')) + + use_arguments(op) do |args| + agg_opts = args.use('aggregationOptions') || {} + collection.search_indexes( + id: args.use('id'), + name: args.use('name'), + aggregate: ::Utils.underscore_hash(agg_opts) + ).to_a + end + end + + def update_search_index(op) + collection = entities.get(:collection, op.use!('object')) + + use_arguments(op) do |args| + collection.search_indexes.update_one( + args.use('definition'), + id: args.use('id'), + name: args.use('name') + ) + end + end + end +end diff --git a/spec/runners/unified/test.rb b/spec/runners/unified/test.rb index bc48a01d65..fb9083e6e9 100644 --- a/spec/runners/unified/test.rb +++ b/spec/runners/unified/test.rb @@ -9,6 +9,7 @@ require 'runners/unified/change_stream_operations' require 'runners/unified/support_operations' require 'runners/unified/thread_operations' +require 'runners/unified/search_index_operations' require 'runners/unified/assertions' require 'support/utils' require 'support/crypt' @@ -23,6 +24,7 @@ class Test include ChangeStreamOperations include SupportOperations include ThreadOperations + include SearchIndexOperations include Assertions include RSpec::Core::Pending @@ -120,7 +122,7 @@ def generate_entities(es) # the other set members, in standalone deployments because # there is only one server, but changes behavior in # sharded clusters compared to how the test suite is configured. - opts[:single_address] = true + options[:single_address] = true end if store_events = spec.use('storeEventsAsEntities') diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3684894c54..f3ffc785e2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -20,7 +20,7 @@ config.extend(Constraints) config.before(:all) do - if ClusterConfig.instance.fcv_ish >= '3.6' && !SpecConfig.instance.serverless? # Serverless instances do not support killAllSessions command. + if SpecConfig.instance.kill_all_server_sessions? kill_all_server_sessions end end diff --git a/spec/spec_tests/data/index_management/createSearchIndex.yml b/spec/spec_tests/data/index_management/createSearchIndex.yml new file mode 100644 index 0000000000..6aa56f3bc4 --- /dev/null +++ b/spec/spec_tests/data/index_management/createSearchIndex.yml @@ -0,0 +1,62 @@ +description: "createSearchIndex" +schemaVersion: "1.4" +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: *database0 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: *collection0 + +runOnRequirements: + - minServerVersion: "7.0.0" + topologies: [ replicaset, load-balanced, sharded ] + serverless: forbid + +tests: + - description: "no name provided for an index definition" + operations: + - name: createSearchIndex + object: *collection0 + arguments: + model: { definition: &definition { mappings: { dynamic: true } } } + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + createSearchIndexes: *collection0 + indexes: [ { definition: *definition } ] + $db: *database0 + + - description: "name provided for an index definition" + operations: + - name: createSearchIndex + object: *collection0 + arguments: + model: { definition: &definition { mappings: { dynamic: true } } , name: 'test index' } + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + createSearchIndexes: *collection0 + indexes: [ { definition: *definition, name: 'test index' } ] + $db: *database0 \ No newline at end of file diff --git a/spec/spec_tests/data/index_management/createSearchIndexes.yml b/spec/spec_tests/data/index_management/createSearchIndexes.yml new file mode 100644 index 0000000000..54a6e84ccb --- /dev/null +++ b/spec/spec_tests/data/index_management/createSearchIndexes.yml @@ -0,0 +1,83 @@ +description: "createSearchIndexes" +schemaVersion: "1.4" +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: *database0 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: *collection0 + +runOnRequirements: + - minServerVersion: "7.0.0" + topologies: [ replicaset, load-balanced, sharded ] + serverless: forbid + +tests: + - description: "empty index definition array" + operations: + - name: createSearchIndexes + object: *collection0 + arguments: + models: [] + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + createSearchIndexes: *collection0 + indexes: [] + $db: *database0 + + + - description: "no name provided for an index definition" + operations: + - name: createSearchIndexes + object: *collection0 + arguments: + models: [ { definition: &definition { mappings: { dynamic: true } } } ] + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + createSearchIndexes: *collection0 + indexes: [ { definition: *definition } ] + $db: *database0 + + - description: "name provided for an index definition" + operations: + - name: createSearchIndexes + object: *collection0 + arguments: + models: [ { definition: &definition { mappings: { dynamic: true } } , name: 'test index' } ] + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + createSearchIndexes: *collection0 + indexes: [ { definition: *definition, name: 'test index' } ] + $db: *database0 \ No newline at end of file diff --git a/spec/spec_tests/data/index_management/dropSearchIndex.yml b/spec/spec_tests/data/index_management/dropSearchIndex.yml new file mode 100644 index 0000000000..e384cf26c5 --- /dev/null +++ b/spec/spec_tests/data/index_management/dropSearchIndex.yml @@ -0,0 +1,42 @@ +description: "dropSearchIndex" +schemaVersion: "1.4" +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: *database0 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: *collection0 + +runOnRequirements: + - minServerVersion: "7.0.0" + topologies: [ replicaset, load-balanced, sharded ] + serverless: forbid + +tests: + - description: "sends the correct command" + operations: + - name: dropSearchIndex + object: *collection0 + arguments: + name: &indexName 'test index' + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + dropSearchIndex: *collection0 + name: *indexName + $db: *database0 diff --git a/spec/spec_tests/data/index_management/listSearchIndexes.yml b/spec/spec_tests/data/index_management/listSearchIndexes.yml new file mode 100644 index 0000000000..a50becdf1d --- /dev/null +++ b/spec/spec_tests/data/index_management/listSearchIndexes.yml @@ -0,0 +1,85 @@ +description: "listSearchIndexes" +schemaVersion: "1.4" +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: *database0 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: *collection0 + +runOnRequirements: + - minServerVersion: "7.0.0" + topologies: [ replicaset, load-balanced, sharded ] + serverless: forbid + +tests: + - description: "when no name is provided, it does not populate the filter" + operations: + - name: listSearchIndexes + object: *collection0 + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection0 + pipeline: + - $listSearchIndexes: {} + + - description: "when a name is provided, it is present in the filter" + operations: + - name: listSearchIndexes + object: *collection0 + arguments: + name: &indexName "test index" + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection0 + pipeline: + - $listSearchIndexes: { name: *indexName } + $db: *database0 + + - description: aggregation cursor options are supported + operations: + - name: listSearchIndexes + object: *collection0 + arguments: + name: &indexName "test index" + aggregationOptions: + batchSize: 10 + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection0 + cursor: { batchSize: 10 } + pipeline: + - $listSearchIndexes: { name: *indexName } + $db: *database0 \ No newline at end of file diff --git a/spec/spec_tests/data/index_management/updateSearchIndex.yml b/spec/spec_tests/data/index_management/updateSearchIndex.yml new file mode 100644 index 0000000000..bb18ab512e --- /dev/null +++ b/spec/spec_tests/data/index_management/updateSearchIndex.yml @@ -0,0 +1,45 @@ +description: "updateSearchIndex" +schemaVersion: "1.4" +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: *database0 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: *collection0 + +runOnRequirements: + - minServerVersion: "7.0.0" + topologies: [ replicaset, load-balanced, sharded ] + serverless: forbid + +tests: + - description: "sends the correct command" + operations: + - name: updateSearchIndex + object: *collection0 + arguments: + name: &indexName 'test index' + definition: &definition {} + expectError: + # This test always errors in a non-Atlas environment. The test functions as a unit test by asserting + # that the driver constructs and sends the correct command. + isError: true + errorContains: Search index commands are only supported with Atlas + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + updateSearchIndex: *collection0 + name: *indexName + definition: *definition + $db: *database0 + diff --git a/spec/spec_tests/index_management_unified_spec.rb b/spec/spec_tests/index_management_unified_spec.rb new file mode 100644 index 0000000000..e93d30cfb5 --- /dev/null +++ b/spec/spec_tests/index_management_unified_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'runners/unified' + +base = "#{CURRENT_PATH}/spec_tests/data/index_management" +INDEX_MANAGEMENT_UNIFIED_TESTS = Dir.glob("#{base}/**/*.yml").sort + +# rubocop:disable RSpec/EmptyExampleGroup +describe 'index management unified spec tests' do + define_unified_spec_tests(base, INDEX_MANAGEMENT_UNIFIED_TESTS) +end +# rubocop:enable RSpec/EmptyExampleGroup diff --git a/spec/support/spec_config.rb b/spec/support/spec_config.rb index eb090db2ac..34d6c6d478 100644 --- a/spec/support/spec_config.rb +++ b/spec/support/spec_config.rb @@ -172,6 +172,11 @@ def serverless? !!ENV['SERVERLESS'] end + def kill_all_server_sessions? + !serverless? && # Serverless instances do not support killAllSessions command. + ClusterConfig.instance.fcv_ish >= '3.6' + end + # Test suite configuration def client_debug?