diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb1e0ac..7703c3ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `remote store` client APIs ([#552](https://github.com/opensearch-project/opensearch-py/pull/552)) - Added `nox -rs generate` ([#554](https://github.com/opensearch-project/opensearch-py/pull/554)) - Added a utf-8 header to all .py files ([#557](https://github.com/opensearch-project/opensearch-py/pull/557)) +- Added `samples`, `benchmarks` and `docs` to `nox -rs format` ([#556](https://github.com/opensearch-project/opensearch-py/pull/556)) ### Changed - Generate `tasks` client from API specs ([#508](https://github.com/opensearch-project/opensearch-py/pull/508)) - Generate `ingest` client from API specs ([#513](https://github.com/opensearch-project/opensearch-py/pull/513)) diff --git a/benchmarks/bench_async.py b/benchmarks/bench_async.py index 015801f4..c7eb5714 100644 --- a/benchmarks/bench_async.py +++ b/benchmarks/bench_async.py @@ -6,6 +6,9 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. import asyncio import uuid @@ -25,7 +28,7 @@ async def index_records(client, item_count): client.index( index=index_name, body={ - "title": f"Moneyball", + "title": "Moneyball", "director": "Bennett Miller", "year": "2011", }, diff --git a/benchmarks/bench_info_sync.py b/benchmarks/bench_info_sync.py index 618954d7..229a2e4d 100644 --- a/benchmarks/bench_info_sync.py +++ b/benchmarks/bench_info_sync.py @@ -6,6 +6,10 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import logging import sys @@ -36,7 +40,7 @@ def get_info(client, request_count): tt = 0 for n in range(request_count): start = time.time() * 1000 - rc = client.info() + client.info() total_time = time.time() * 1000 - start tt += total_time return tt diff --git a/benchmarks/bench_sync.py b/benchmarks/bench_sync.py index 146974f1..e201eaba 100644 --- a/benchmarks/bench_sync.py +++ b/benchmarks/bench_sync.py @@ -6,8 +6,13 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. import json +import logging +import sys import time import uuid @@ -21,9 +26,6 @@ index_name = "test-index-sync" item_count = 1000 -import logging -import sys - root = logging.getLogger() # root.setLevel(logging.DEBUG) # logging.getLogger("urllib3.connectionpool").setLevel(logging.DEBUG) diff --git a/benchmarks/bench_sync_async.py b/benchmarks/bench_sync_async.py index 8c43e278..7950dc64 100644 --- a/benchmarks/bench_sync_async.py +++ b/benchmarks/bench_sync_async.py @@ -6,6 +6,10 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import bench_async import bench_sync diff --git a/benchmarks/thread_with_return_value.py b/benchmarks/thread_with_return_value.py index 46fefe1f..b6bc9c09 100644 --- a/benchmarks/thread_with_return_value.py +++ b/benchmarks/thread_with_return_value.py @@ -1,11 +1,13 @@ -#!/usr/bin/env python - # -*- coding: utf-8 -*- # SPDX-License-Identifier: Apache-2.0 # # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + from threading import Thread diff --git a/docs/source/conf.py b/docs/source/conf.py index ea677630..133a2564 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,3 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full diff --git a/noxfile.py b/noxfile.py index 04374cd4..a9cd9068 100644 --- a/noxfile.py +++ b/noxfile.py @@ -34,6 +34,9 @@ "opensearchpy/", "test_opensearchpy/", "utils/", + "samples/", + "benchmarks/", + "docs/", ) diff --git a/samples/advanced_index_actions/advanced_index_actions_sample.py b/samples/advanced_index_actions/advanced_index_actions_sample.py index 391d36b9..96d7d742 100644 --- a/samples/advanced_index_actions/advanced_index_actions_sample.py +++ b/samples/advanced_index_actions/advanced_index_actions_sample.py @@ -1,6 +1,17 @@ -from opensearchpy import OpenSearch +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. import time +from opensearchpy import OpenSearch # For cleaner output, comment in the two lines below to disable warnings and informational messages # import urllib3 @@ -10,73 +21,84 @@ def test_opensearch_examples(): # Set up client = OpenSearch( - hosts=['https://localhost:9200'], + hosts=["https://localhost:9200"], use_ssl=True, verify_certs=False, - http_auth=('admin', 'admin') + http_auth=("admin", "admin"), ) - client.indices.create(index='movies') + client.indices.create(index="movies") print("'movies' index created!") - + # Test Clear Index Cache - client.indices.clear_cache(index='movies') + client.indices.clear_cache(index="movies") print("Cache for 'movies' index cleared!") - client.indices.clear_cache(index='movies', query=True) + client.indices.clear_cache(index="movies", query=True) print("Query cache for 'movies' index cleared!") - client.indices.clear_cache(index='movies', fielddata=True, request=True) + client.indices.clear_cache(index="movies", fielddata=True, request=True) print("Field data and request cache for 'movies' index cleared!") - + # Test Flush Index - client.indices.flush(index='movies') + client.indices.flush(index="movies") print("'movies' index flushed!") - + # Test Refresh Index - client.indices.refresh(index='movies') + client.indices.refresh(index="movies") print("'movies' index refreshed!") - + # Test Close or Open Index - client.indices.close(index='movies') + client.indices.close(index="movies") print("'movies' index closed!") time.sleep(2) # add sleep to ensure the index has time to close - client.indices.open(index='movies') + client.indices.open(index="movies") print("'movies' index opened!") - + # Test Force Merge Index - client.indices.forcemerge(index='movies') + client.indices.forcemerge(index="movies") print("'movies' index force merged!") - + # Test Clone - client.indices.put_settings(index='movies', body={'index': {'blocks': {'write': True}}}) + client.indices.put_settings( + index="movies", body={"index": {"blocks": {"write": True}}} + ) print("Write operations blocked for 'movies' index!") time.sleep(2) - client.indices.clone(index='movies', target='movies_clone') + client.indices.clone(index="movies", target="movies_clone") print("'movies' index cloned to 'movies_clone'!") - client.indices.put_settings(index='movies', body={'index': {'blocks': {'write': False}}}) + client.indices.put_settings( + index="movies", body={"index": {"blocks": {"write": False}}} + ) print("Write operations enabled for 'movies' index!") - - # Test Split + + # Test Split client.indices.create( - index='books', - body={'settings': { - 'index': {'number_of_shards': 5, 'number_of_routing_shards': 30, 'blocks': {'write': True}}}} + index="books", + body={ + "settings": { + "index": { + "number_of_shards": 5, + "number_of_routing_shards": 30, + "blocks": {"write": True}, + } + } + }, ) print("'books' index created!") time.sleep(2) # add sleep to ensure the index has time to become read-only client.indices.split( - index='books', - target='bigger_books', - body={'settings': {'index': {'number_of_shards': 10 }}} + index="books", + target="bigger_books", + body={"settings": {"index": {"number_of_shards": 10}}}, ) print("'books' index split into 'bigger_books'!") - client.indices.put_settings(index='books', body={'index': {'blocks': {'write': False}}}) + client.indices.put_settings( + index="books", body={"index": {"blocks": {"write": False}}} + ) print("Write operations enabled for 'books' index!") - + # Cleanup - client.indices.delete(index=['movies', 'books', 'movies_clone', 'bigger_books']) + client.indices.delete(index=["movies", "books", "movies_clone", "bigger_books"]) print("All indices deleted!") - - if __name__ == "__main__": - test_opensearch_examples() \ No newline at end of file + test_opensearch_examples() diff --git a/samples/aws/search-requests.py b/samples/aws/search-requests.py index 885c4693..0af366f0 100644 --- a/samples/aws/search-requests.py +++ b/samples/aws/search-requests.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # -*- coding: utf-8 -*- # SPDX-License-Identifier: Apache-2.0 # @@ -9,62 +11,59 @@ # GitHub history for details. import logging - from os import environ from time import sleep from urllib.parse import urlparse from boto3 import Session -from opensearchpy import RequestsAWSV4SignerAuth, OpenSearch, RequestsHttpConnection + +from opensearchpy import OpenSearch, RequestsAWSV4SignerAuth, RequestsHttpConnection # verbose logging -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) # cluster endpoint, for example: my-test-domain.us-east-1.es.amazonaws.com -url = urlparse(environ['ENDPOINT']) -region = environ.get('AWS_REGION', 'us-east-1') -service = environ.get('SERVICE', 'es') +url = urlparse(environ["ENDPOINT"]) +region = environ.get("AWS_REGION", "us-east-1") +service = environ.get("SERVICE", "es") credentials = Session().get_credentials() auth = RequestsAWSV4SignerAuth(credentials, region, service) client = OpenSearch( - hosts=[{ - 'host': url.netloc, - 'port': url.port or 443 - }], - http_auth=auth, - use_ssl=True, - verify_certs=True, - connection_class=RequestsHttpConnection, - timeout=30 + hosts=[{"host": url.netloc, "port": url.port or 443}], + http_auth=auth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection, + timeout=30, ) # TODO: remove when OpenSearch Serverless adds support for / -if service == 'es': - info = client.info() - print(f"{info['version']['distribution']}: {info['version']['number']}") +if service == "es": + info = client.info() + print(f"{info['version']['distribution']}: {info['version']['number']}") # create an index -index = 'movies' +index = "movies" client.indices.create(index=index) try: - # index data - document = {'director': 'Bennett Miller', 'title': 'Moneyball', 'year': 2011} - client.index(index=index, body=document, id='1') + # index data + document = {"director": "Bennett Miller", "title": "Moneyball", "year": 2011} + client.index(index=index, body=document, id="1") - # wait for the document to index - sleep(1) + # wait for the document to index + sleep(1) - # search for the document - results = client.search(body={'query': {'match': {'director': 'miller'}}}) - for hit in results['hits']['hits']: - print(hit['_source']) + # search for the document + results = client.search(body={"query": {"match": {"director": "miller"}}}) + for hit in results["hits"]["hits"]: + print(hit["_source"]) - # delete the document - client.delete(index=index, id='1') + # delete the document + client.delete(index=index, id="1") finally: - # delete the index - client.indices.delete(index=index) \ No newline at end of file + # delete the index + client.indices.delete(index=index) diff --git a/samples/aws/search-urllib3.py b/samples/aws/search-urllib3.py index 8fccfe9f..534caf40 100644 --- a/samples/aws/search-urllib3.py +++ b/samples/aws/search-urllib3.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # -*- coding: utf-8 -*- # SPDX-License-Identifier: Apache-2.0 # @@ -9,62 +11,59 @@ # GitHub history for details. import logging - from os import environ from time import sleep from urllib.parse import urlparse from boto3 import Session -from opensearchpy import Urllib3AWSV4SignerAuth, OpenSearch, Urllib3HttpConnection + +from opensearchpy import OpenSearch, Urllib3AWSV4SignerAuth, Urllib3HttpConnection # verbose logging -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) # cluster endpoint, for example: my-test-domain.us-east-1.es.amazonaws.com -url = urlparse(environ['ENDPOINT']) -region = environ.get('AWS_REGION', 'us-east-1') -service = environ.get('SERVICE', 'es') +url = urlparse(environ["ENDPOINT"]) +region = environ.get("AWS_REGION", "us-east-1") +service = environ.get("SERVICE", "es") credentials = Session().get_credentials() auth = Urllib3AWSV4SignerAuth(credentials, region, service) client = OpenSearch( - hosts=[{ - 'host': url.netloc, - 'port': url.port or 443 - }], - http_auth=auth, - use_ssl=True, - verify_certs=True, - connection_class=Urllib3HttpConnection, - timeout=30 + hosts=[{"host": url.netloc, "port": url.port or 443}], + http_auth=auth, + use_ssl=True, + verify_certs=True, + connection_class=Urllib3HttpConnection, + timeout=30, ) # TODO: remove when OpenSearch Serverless adds support for / -if service == 'es': - info = client.info() - print(f"{info['version']['distribution']}: {info['version']['number']}") +if service == "es": + info = client.info() + print(f"{info['version']['distribution']}: {info['version']['number']}") # create an index -index = 'movies' +index = "movies" client.indices.create(index=index) try: - # index data - document = {'director': 'Bennett Miller', 'title': 'Moneyball', 'year': 2011} - client.index(index=index, body=document, id='1') + # index data + document = {"director": "Bennett Miller", "title": "Moneyball", "year": 2011} + client.index(index=index, body=document, id="1") - # wait for the document to index - sleep(1) + # wait for the document to index + sleep(1) - # search for the document - results = client.search(body={'query': {'match': {'director': 'miller'}}}) - for hit in results['hits']['hits']: - print(hit['_source']) + # search for the document + results = client.search(body={"query": {"match": {"director": "miller"}}}) + for hit in results["hits"]["hits"]: + print(hit["_source"]) - # delete the document - client.delete(index=index, id='1') + # delete the document + client.delete(index=index, id="1") finally: - # delete the index - client.indices.delete(index=index) \ No newline at end of file + # delete the index + client.indices.delete(index=index) diff --git a/samples/bulk/bulk-array.py b/samples/bulk/bulk-array.py index dea7fae1..1859d541 100755 --- a/samples/bulk/bulk-array.py +++ b/samples/bulk/bulk-array.py @@ -6,55 +6,53 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import os -import json from opensearchpy import OpenSearch # connect to an instance of OpenSearch -host = os.getenv('HOST', default='localhost') -port = int(os.getenv('PORT', 9200)) -auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') -) +host = os.getenv("HOST", default="localhost") +port = int(os.getenv("PORT", 9200)) +auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # check whether an index exists index_name = "my-index" if not client.indices.exists(index_name): - - client.indices.create(index_name, + client.indices.create( + index_name, body={ - "mappings":{ + "mappings": { "properties": { - "value": { - "type": "float" - }, + "value": {"type": "float"}, } } - } + }, ) # index data data = [] for i in range(100): - data.append({ "index": {"_index": index_name, "_id": i }}) - data.append({ "value": i }) + data.append({"index": {"_index": index_name, "_id": i}}) + data.append({"value": i}) rc = client.bulk(data) if rc["errors"]: - print(f"There were errors:") + print("There were errors:") for item in rc["items"]: print(f"{item['index']['status']}: {item['index']['error']['type']}") else: @@ -62,4 +60,3 @@ # delete index client.indices.delete(index=index_name) - diff --git a/samples/bulk/bulk-helpers.py b/samples/bulk/bulk-helpers.py index b6aff98f..3dc165c8 100755 --- a/samples/bulk/bulk-helpers.py +++ b/samples/bulk/bulk-helpers.py @@ -6,54 +6,51 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import os -import json from opensearchpy import OpenSearch, helpers # connect to an instance of OpenSearch -host = os.getenv('HOST', default='localhost') -port = int(os.getenv('PORT', 9200)) -auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') -) +host = os.getenv("HOST", default="localhost") +port = int(os.getenv("PORT", 9200)) +auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # check whether an index exists index_name = "my-index" if not client.indices.exists(index_name): - - client.indices.create(index_name, + client.indices.create( + index_name, body={ - "mappings":{ + "mappings": { "properties": { - "value": { - "type": "float" - }, + "value": {"type": "float"}, } } - } + }, ) # index data data = [] for i in range(100): - data.append({ "_index": index_name, "_id": i, "value": i }) + data.append({"_index": index_name, "_id": i, "value": i}) rc = helpers.bulk(client, data) print(f"Bulk-inserted {rc[0]} items.") # delete index client.indices.delete(index=index_name) - diff --git a/samples/bulk/bulk-ld.py b/samples/bulk/bulk-ld.py index 299e16d3..fff0ae98 100755 --- a/samples/bulk/bulk-ld.py +++ b/samples/bulk/bulk-ld.py @@ -6,55 +6,54 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + -import os import json +import os from opensearchpy import OpenSearch # connect to an instance of OpenSearch -host = os.getenv('HOST', default='localhost') -port = int(os.getenv('PORT', 9200)) -auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') -) +host = os.getenv("HOST", default="localhost") +port = int(os.getenv("PORT", 9200)) +auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # check whether an index exists index_name = "my-index" if not client.indices.exists(index_name): - - client.indices.create(index_name, + client.indices.create( + index_name, body={ - "mappings":{ + "mappings": { "properties": { - "value": { - "type": "float" - }, + "value": {"type": "float"}, } } - } + }, ) # index data -data = '' +data = "" for i in range(100): - data += json.dumps({ "index": {"_index": index_name, "_id": i }}) + "\n" - data += json.dumps({ "value": i }) + "\n" + data += json.dumps({"index": {"_index": index_name, "_id": i}}) + "\n" + data += json.dumps({"value": i}) + "\n" rc = client.bulk(data) if rc["errors"]: - print(f"There were errors:") + print("There were errors:") for item in rc["items"]: print(f"{item['index']['status']}: {item['index']['error']['type']}") else: @@ -62,4 +61,3 @@ # delete index client.indices.delete(index=index_name) - diff --git a/samples/hello/hello-async.py b/samples/hello/hello-async.py index cebe703d..9975f575 100755 --- a/samples/hello/hello-async.py +++ b/samples/hello/hello-async.py @@ -6,103 +6,91 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import asyncio from opensearchpy import AsyncOpenSearch + async def main(): # connect to OpenSearch - host = 'localhost' + host = "localhost" port = 9200 - auth = ('admin', 'admin') # For testing only. Don't store credentials in code. + auth = ("admin", "admin") # For testing only. Don't store credentials in code. client = AsyncOpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) try: - info = await client.info() - print(f"Welcome to {info['version']['distribution']} {info['version']['number']}!") + info = await client.info() + print( + f"Welcome to {info['version']['distribution']} {info['version']['number']}!" + ) + + # create an index + + index_name = "test-index" + + index_body = {"settings": {"index": {"number_of_shards": 4}}} + + if not await client.indices.exists(index=index_name): + await client.indices.create(index_name, body=index_body) + + # add some documents to the index, asynchronously + await asyncio.gather( + *[ + client.index( + index=index_name, + body={ + "title": f"Moneyball {i}", + "director": "Bennett Miller", + "year": "2011", + }, + id=i, + ) + for i in range(10) + ] + ) - # create an index + # refresh the index + await client.indices.refresh(index=index_name) - index_name = 'test-index' + # search for a document + q = "miller" - index_body = { - 'settings': { - 'index': { - 'number_of_shards': 4 - } + query = { + "size": 5, + "query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}}, } - } - if not await client.indices.exists(index=index_name): - await client.indices.create( - index_name, - body=index_body + results = await client.search(body=query, index=index_name) + + for hit in results["hits"]["hits"]: + print(hit) + + # delete the documents + await asyncio.gather( + *[client.delete(index=index_name, id=i) for i in range(10)] ) - # add some documents to the index, asynchronously - await asyncio.gather(*[ - client.index( - index = index_name, - body = { - 'title': f"Moneyball {i}", - 'director': 'Bennett Miller', - 'year': '2011' - }, - id = i - ) for i in range(10) - ]) - - # refresh the index - await client.indices.refresh(index=index_name) - - # search for a document - q = 'miller' - - query = { - 'size': 5, - 'query': { - 'multi_match': { - 'query': q, - 'fields': ['title^2', 'director'] - } - } - } - - results = await client.search( - body = query, - index = index_name - ) - - for hit in results["hits"]["hits"]: - print(hit) - - # delete the documents - await asyncio.gather(*[ - client.delete( - index = index_name, - id = i - ) for i in range(10) - ]) - - # delete the index - await client.indices.delete( - index = index_name - ) - - finally: - await client.close() + # delete the index + await client.indices.delete(index=index_name) + + finally: + await client.close() + if __name__ == "__main__": loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete(main()) loop.close() - diff --git a/samples/hello/hello.py b/samples/hello/hello.py index 20363d14..0b589c9d 100755 --- a/samples/hello/hello.py +++ b/samples/hello/hello.py @@ -6,21 +6,25 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + from opensearchpy import OpenSearch # connect to OpenSearch -host = 'localhost' +host = "localhost" port = 9200 -auth = ('admin', 'admin') # For testing only. Don't store credentials in code. +auth = ("admin", "admin") # For testing only. Don't store credentials in code. client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) info = client.info() @@ -28,76 +32,45 @@ # create an index -index_name = 'test-index' +index_name = "test-index" -index_body = { - 'settings': { - 'index': { - 'number_of_shards': 4 - } - } -} +index_body = {"settings": {"index": {"number_of_shards": 4}}} -response = client.indices.create( - index_name, - body=index_body -) +response = client.indices.create(index_name, body=index_body) print(response) # add a document to the index -document = { - 'title': 'Moneyball', - 'director': 'Bennett Miller', - 'year': '2011' -} +document = {"title": "Moneyball", "director": "Bennett Miller", "year": "2011"} -id = '1' +id = "1" -response = client.index( - index = index_name, - body = document, - id = id, - refresh = True -) +response = client.index(index=index_name, body=document, id=id, refresh=True) print(response) # search for a document -q = 'miller' +q = "miller" query = { - 'size': 5, - 'query': { - 'multi_match': { - 'query': q, - 'fields': ['title^2', 'director'] - } - } + "size": 5, + "query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}}, } -response = client.search( - body = query, - index = index_name -) +response = client.search(body=query, index=index_name) print(response) # delete the document -response = client.delete( - index = index_name, - id = id -) +response = client.delete(index=index_name, id=id) print(response) # delete the index -response = client.indices.delete( - index = index_name -) +response = client.indices.delete(index=index_name) print(response) diff --git a/samples/index_template/index_template_sample.py b/samples/index_template/index_template_sample.py index dab504be..4fe580ac 100644 --- a/samples/index_template/index_template_sample.py +++ b/samples/index_template/index_template_sample.py @@ -1,143 +1,129 @@ +#!/usr/bin/env python + +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. from opensearchpy import OpenSearch # Create a client instance client = OpenSearch( - hosts=['https://localhost:9200'], - use_ssl=True, - verify_certs=False, - http_auth=('admin', 'admin') + hosts=["https://localhost:9200"], + use_ssl=True, + verify_certs=False, + http_auth=("admin", "admin"), ) # You can create an index template to define default settings and mappings for indices of certain patterns. The following example creates an index template named `books` with default settings and mappings for indices of the `books-*` pattern: client.indices.put_index_template( -name='books', -body={ - 'index_patterns': ['books-*'], - 'priority': 1, - 'template': { - 'settings': { - 'index': { - 'number_of_shards': 3, - 'number_of_replicas': 0 - } + name="books", + body={ + "index_patterns": ["books-*"], + "priority": 1, + "template": { + "settings": {"index": {"number_of_shards": 3, "number_of_replicas": 0}}, + "mappings": { + "properties": { + "title": {"type": "text"}, + "author": {"type": "text"}, + "published_on": {"type": "date"}, + "pages": {"type": "integer"}, + } + }, + }, }, - 'mappings': { - 'properties': { - 'title': { 'type': 'text' }, - 'author': { 'type': 'text' }, - 'published_on': { 'type': 'date' }, - 'pages': { 'type': 'integer' } - } - } - } -} ) # Now, when you create an index that matches the `books-*` pattern, OpenSearch will automatically apply the template's settings and mappings to the index. Let's create an index named books-nonfiction and verify that its settings and mappings match those of the template: -client.indices.create(index='books-nonfiction') -print(client.indices.get(index='books-nonfiction')) +client.indices.create(index="books-nonfiction") +print(client.indices.get(index="books-nonfiction")) # If multiple index templates match the index's name, OpenSearch will apply the template with the highest `priority`. The following example creates two index templates named `books-*` and `books-fiction-*` with different settings: client.indices.put_index_template( -name='books', -body={ - 'index_patterns': ['books-*'], - 'priority': 1, - 'template': { - 'settings': { - 'index': { - 'number_of_shards': 3, - 'number_of_replicas': 0 - } - } - } -} + name="books", + body={ + "index_patterns": ["books-*"], + "priority": 1, + "template": { + "settings": {"index": {"number_of_shards": 3, "number_of_replicas": 0}} + }, + }, ) client.indices.put_index_template( -name='books-fiction', -body={ - 'index_patterns': ['books-fiction-*'], - 'priority': 2, - 'template': { - 'settings': { - 'index': { - 'number_of_shards': 1, - 'number_of_replicas': 1 - } - } - } -} + name="books-fiction", + body={ + "index_patterns": ["books-fiction-*"], + "priority": 2, + "template": { + "settings": {"index": {"number_of_shards": 1, "number_of_replicas": 1}} + }, + }, ) # # Test multiple index templates -client.indices.create(index='books-fiction-romance') -print(client.indices.get(index='books-fiction-romance')) +client.indices.create(index="books-fiction-romance") +print(client.indices.get(index="books-fiction-romance")) # Composable index templates are a new type of index template that allow you to define multiple component templates and compose them into a final template. The following example creates a component template named `books_mappings` with default mappings for indices of the `books-*` and `books-fiction-*` patterns: client.cluster.put_component_template( -name='books_mappings', -body={ - 'template': { - 'mappings': { - 'properties': { - 'title': { 'type': 'text' }, - 'author': { 'type': 'text' }, - 'published_on': { 'type': 'date' }, - 'pages': { 'type': 'integer' } - } - } - } -} + name="books_mappings", + body={ + "template": { + "mappings": { + "properties": { + "title": {"type": "text"}, + "author": {"type": "text"}, + "published_on": {"type": "date"}, + "pages": {"type": "integer"}, + } + } + } + }, ) client.indices.put_index_template( -name='books', -body={ - 'index_patterns': ['books-*'], - 'composed_of': ['books_mappings'], - 'priority': 4, - 'template': { - 'settings': { - 'index': { - 'number_of_shards': 3, - 'number_of_replicas': 0 - } - } - } -} + name="books", + body={ + "index_patterns": ["books-*"], + "composed_of": ["books_mappings"], + "priority": 4, + "template": { + "settings": {"index": {"number_of_shards": 3, "number_of_replicas": 0}} + }, + }, ) client.indices.put_index_template( -name='books-fiction', -body={ - 'index_patterns': ['books-fiction-*'], - 'composed_of': ['books_mappings'], - 'priority': 5, - 'template': { - 'settings': { - 'index': { - 'number_of_shards': 1, - 'number_of_replicas': 1 - } - } - } -} + name="books-fiction", + body={ + "index_patterns": ["books-fiction-*"], + "composed_of": ["books_mappings"], + "priority": 5, + "template": { + "settings": {"index": {"number_of_shards": 1, "number_of_replicas": 1}} + }, + }, ) # Test composable index templates -client.indices.create(index='books-fiction-horror') -print(client.indices.get(index='books-fiction-horror')) +client.indices.create(index="books-fiction-horror") +print(client.indices.get(index="books-fiction-horror")) # Get an index template -print(client.indices.get_index_template(name='books')) +print(client.indices.get_index_template(name="books")) # Delete an index template -client.indices.delete_index_template(name='books') +client.indices.delete_index_template(name="books") # Cleanup -client.indices.delete(index='books-*') -client.indices.delete_index_template(name='books-fiction') -client.cluster.delete_component_template(name='books_mappings') \ No newline at end of file +client.indices.delete(index="books-*") +client.indices.delete_index_template(name="books-fiction") +client.cluster.delete_component_template(name="books_mappings") diff --git a/samples/json/hello-async.py b/samples/json/hello-async.py index 4ce93dbe..b9105d35 100755 --- a/samples/json/hello-async.py +++ b/samples/json/hello-async.py @@ -6,86 +6,91 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import asyncio from opensearchpy import AsyncOpenSearch + async def main(): # connect to OpenSearch - host = 'localhost' + host = "localhost" port = 9200 - auth = ('admin', 'admin') # For testing only. Don't store credentials in code. + auth = ("admin", "admin") # For testing only. Don't store credentials in code. client = AsyncOpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) try: - info = await client.transport.perform_request('GET', '/') - print(f"Welcome to {info['version']['distribution']} {info['version']['number']}!") + info = await client.transport.perform_request("GET", "/") + print( + f"Welcome to {info['version']['distribution']} {info['version']['number']}!" + ) - # create an index + # create an index - index_name = 'movies' + index_name = "movies" - index_body = { - 'settings': { - 'index': { - 'number_of_shards': 4 - } - } - } + index_body = {"settings": {"index": {"number_of_shards": 4}}} - print(await client.transport.perform_request("PUT", f"/{index_name}", body=index_body)) + print( + await client.transport.perform_request( + "PUT", f"/{index_name}", body=index_body + ) + ) - # add a document to the index + # add a document to the index - document = { - 'title': 'Moneyball', - 'director': 'Bennett Miller', - 'year': '2011' - } + document = {"title": "Moneyball", "director": "Bennett Miller", "year": "2011"} - id = '1' + id = "1" - print(await client.transport.perform_request("PUT", f"/{index_name}/_doc/{id}?refresh=true", body = document)) + print( + await client.transport.perform_request( + "PUT", f"/{index_name}/_doc/{id}?refresh=true", body=document + ) + ) - # search for a document + # search for a document - q = 'miller' + q = "miller" - query = { - 'size': 5, - 'query': { - 'multi_match': { - 'query': q, - 'fields': ['title^2', 'director'] - } + query = { + "size": 5, + "query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}}, } - } - print(await client.transport.perform_request("POST", f"/{index_name}/_search", body = query)) + print( + await client.transport.perform_request( + "POST", f"/{index_name}/_search", body=query + ) + ) - # delete the document + # delete the document - print(await client.transport.perform_request("DELETE", f"/{index_name}/_doc/{id}")) + print( + await client.transport.perform_request("DELETE", f"/{index_name}/_doc/{id}") + ) - # delete the index + # delete the index - print(await client.transport.perform_request("DELETE", f"/{index_name}")) + print(await client.transport.perform_request("DELETE", f"/{index_name}")) + finally: + await client.close() - finally: - await client.close() if __name__ == "__main__": loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete(main()) loop.close() - diff --git a/samples/json/hello.py b/samples/json/hello.py index f0c81640..5df36f5f 100755 --- a/samples/json/hello.py +++ b/samples/json/hello.py @@ -6,67 +6,60 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + from opensearchpy import OpenSearch # connect to OpenSearch -host = 'localhost' +host = "localhost" port = 9200 -auth = ('admin', 'admin') # For testing only. Don't store credentials in code. +auth = ("admin", "admin") # For testing only. Don't store credentials in code. client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) -info = client.transport.perform_request('GET', '/') +info = client.transport.perform_request("GET", "/") print(f"Welcome to {info['version']['distribution']} {info['version']['number']}!") # create an index -index_name = 'movies' +index_name = "movies" -index_body = { - 'settings': { - 'index': { - 'number_of_shards': 4 - } - } -} +index_body = {"settings": {"index": {"number_of_shards": 4}}} print(client.transport.perform_request("PUT", f"/{index_name}", body=index_body)) # add a document to the index -document = { - 'title': 'Moneyball', - 'director': 'Bennett Miller', - 'year': '2011' -} +document = {"title": "Moneyball", "director": "Bennett Miller", "year": "2011"} -id = '1' +id = "1" -print(client.transport.perform_request("PUT", f"/{index_name}/_doc/{id}?refresh=true", body = document)) +print( + client.transport.perform_request( + "PUT", f"/{index_name}/_doc/{id}?refresh=true", body=document + ) +) # search for a document -q = 'miller' +q = "miller" query = { - 'size': 5, - 'query': { - 'multi_match': { - 'query': q, - 'fields': ['title^2', 'director'] - } - } + "size": 5, + "query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}}, } -print(client.transport.perform_request("POST", f"/{index_name}/_search", body = query)) +print(client.transport.perform_request("POST", f"/{index_name}/_search", body=query)) # delete the document diff --git a/samples/knn/knn-async-basics.py b/samples/knn/knn-async-basics.py index f92acfe7..a7bb9d2f 100755 --- a/samples/knn/knn-async-basics.py +++ b/samples/knn/knn-async-basics.py @@ -6,29 +6,31 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import asyncio import os import random -from opensearchpy import AsyncOpenSearch, AsyncHttpConnection, helpers +from opensearchpy import AsyncHttpConnection, AsyncOpenSearch, helpers + async def main(): # connect to an instance of OpenSearch - host = os.getenv('HOST', default='localhost') - port = int(os.getenv('PORT', 9200)) - auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') - ) + host = os.getenv("HOST", default="localhost") + port = int(os.getenv("PORT", 9200)) + auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = AsyncOpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, connection_class=AsyncHttpConnection, - ssl_show_warn = False + ssl_show_warn=False, ) # check whether an index exists @@ -36,34 +38,32 @@ async def main(): dimensions = 5 if not await client.indices.exists(index_name): - await client.indices.create(index_name, + await client.indices.create( + index_name, body={ - "settings":{ - "index.knn": True - }, - "mappings":{ + "settings": {"index.knn": True}, + "mappings": { "properties": { - "values": { - "type": "knn_vector", - "dimension": dimensions - }, + "values": {"type": "knn_vector", "dimension": dimensions}, } - } - } + }, + }, ) # index data vectors = [] for i in range(10): vec = [] - for j in range(dimensions): - vec.append(round(random.uniform(0, 1), 2)) - - vectors.append({ - "_index": index_name, - "_id": i, - "values": vec, - }) + for j in range(dimensions): + vec.append(round(random.uniform(0, 1), 2)) + + vectors.append( + { + "_index": index_name, + "_id": i, + "values": vec, + } + ) # bulk index await helpers.async_bulk(client, vectors) @@ -72,8 +72,8 @@ async def main(): # search vec = [] - for j in range(dimensions): - vec.append(round(random.uniform(0, 1), 2)) + for j in range(dimensions): + vec.append(round(random.uniform(0, 1), 2)) print(f"Searching for {vec} ...") search_query = {"query": {"knn": {"values": {"vector": vec, "k": 3}}}} @@ -86,9 +86,9 @@ async def main(): await client.close() + if __name__ == "__main__": loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete(main()) loop.close() - diff --git a/samples/knn/knn-basics.py b/samples/knn/knn-basics.py index 942118a8..96efb028 100755 --- a/samples/knn/knn-basics.py +++ b/samples/knn/knn-basics.py @@ -6,6 +6,10 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import os import random @@ -14,19 +18,16 @@ # connect to an instance of OpenSearch -host = os.getenv('HOST', default='localhost') -port = int(os.getenv('PORT', 9200)) -auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') -) +host = os.getenv("HOST", default="localhost") +port = int(os.getenv("PORT", 9200)) +auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # check whether an index exists @@ -34,34 +35,32 @@ dimensions = 5 if not client.indices.exists(index_name): - client.indices.create(index_name, + client.indices.create( + index_name, body={ - "settings":{ - "index.knn": True - }, - "mappings":{ + "settings": {"index.knn": True}, + "mappings": { "properties": { - "values": { - "type": "knn_vector", - "dimension": dimensions - }, + "values": {"type": "knn_vector", "dimension": dimensions}, } - } - } + }, + }, ) # index data vectors = [] for i in range(10): vec = [] - for j in range(dimensions): - vec.append(round(random.uniform(0, 1), 2)) - - vectors.append({ - "_index": index_name, - "_id": i, - "values": vec, - }) + for j in range(dimensions): + vec.append(round(random.uniform(0, 1), 2)) + + vectors.append( + { + "_index": index_name, + "_id": i, + "values": vec, + } + ) # bulk index helpers.bulk(client, vectors) @@ -70,8 +69,8 @@ # search vec = [] -for j in range(dimensions): - vec.append(round(random.uniform(0, 1), 2)) +for j in range(dimensions): + vec.append(round(random.uniform(0, 1), 2)) print(f"Searching for {vec} ...") search_query = {"query": {"knn": {"values": {"vector": vec, "k": 3}}}} @@ -81,4 +80,3 @@ # delete index client.indices.delete(index=index_name) - diff --git a/samples/knn/knn-boolean-filter.py b/samples/knn/knn-boolean-filter.py index 6a69b5f8..5ae7704c 100755 --- a/samples/knn/knn-boolean-filter.py +++ b/samples/knn/knn-boolean-filter.py @@ -6,6 +6,10 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import os import random @@ -14,19 +18,16 @@ # connect to an instance of OpenSearch -host = os.getenv('HOST', default='localhost') -port = int(os.getenv('PORT', 9200)) -auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') -) +host = os.getenv("HOST", default="localhost") +port = int(os.getenv("PORT", 9200)) +auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # check whether an index exists @@ -34,38 +35,34 @@ dimensions = 5 if not client.indices.exists(index_name): - client.indices.create(index_name, + client.indices.create( + index_name, body={ - "settings":{ - "index.knn": True - }, - "mappings":{ + "settings": {"index.knn": True}, + "mappings": { "properties": { - "values": { - "type": "knn_vector", - "dimension": dimensions - }, + "values": {"type": "knn_vector", "dimension": dimensions}, } - } - } + }, + }, ) # index data vectors = [] -genres = ['fiction', 'drama', 'romance'] +genres = ["fiction", "drama", "romance"] for i in range(3000): vec = [] - for j in range(dimensions): - vec.append(round(random.uniform(0, 1), 2)) - - vectors.append({ - "_index": index_name, - "_id": i, - "values": vec, - "metadata": { - "genre": random.choice(genres) + for j in range(dimensions): + vec.append(round(random.uniform(0, 1), 2)) + + vectors.append( + { + "_index": index_name, + "_id": i, + "values": vec, + "metadata": {"genre": random.choice(genres)}, } - }) + ) # bulk index helpers.bulk(client, vectors) @@ -75,30 +72,15 @@ # search genre = random.choice(genres) vec = [] -for j in range(dimensions): - vec.append(round(random.uniform(0, 1), 2)) +for j in range(dimensions): + vec.append(round(random.uniform(0, 1), 2)) print(f"Searching for {vec} with the '{genre}' genre ...") search_query = { "query": { "bool": { - "filter": { - "bool": { - "must": [{ - "term": { - "metadata.genre": genre - } - }] - } - }, - "must": { - "knn": { - "values": { - "vector": vec, - "k": 5 - } - } - } + "filter": {"bool": {"must": [{"term": {"metadata.genre": genre}}]}}, + "must": {"knn": {"values": {"vector": vec, "k": 5}}}, } } } diff --git a/samples/knn/knn-efficient-filter.py b/samples/knn/knn-efficient-filter.py index 569e4685..cbfd41ad 100755 --- a/samples/knn/knn-efficient-filter.py +++ b/samples/knn/knn-efficient-filter.py @@ -6,75 +6,145 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + import os -import random from opensearchpy import OpenSearch, helpers # connect to an instance of OpenSearch -host = os.getenv('HOST', default='localhost') -port = int(os.getenv('PORT', 9200)) -auth = ( - os.getenv('USERNAME', 'admin'), - os.getenv('PASSWORD', 'admin') -) +host = os.getenv("HOST", default="localhost") +port = int(os.getenv("PORT", 9200)) +auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin")) client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # check whether an index exists index_name = "hotels-index" if not client.indices.exists(index_name): - client.indices.create(index_name, + client.indices.create( + index_name, body={ - "settings":{ + "settings": { "index.knn": True, "knn.algo_param.ef_search": 100, "number_of_shards": 1, - "number_of_replicas": 0 + "number_of_replicas": 0, }, - "mappings":{ + "mappings": { "properties": { "location": { - "type": "knn_vector", + "type": "knn_vector", "dimension": 2, "method": { "name": "hnsw", "space_type": "l2", "engine": "lucene", - "parameters": { - "ef_construction": 100, - "m": 16 - } - } + "parameters": {"ef_construction": 100, "m": 16}, + }, }, } - } - } + }, + }, ) # index data vectors = [ - { "_index": "hotels-index", "_id": "1", "location": [5.2, 4.4], "parking" : "true", "rating" : 5 }, - { "_index": "hotels-index", "_id": "2", "location": [5.2, 3.9], "parking" : "false", "rating" : 4 }, - { "_index": "hotels-index", "_id": "3", "location": [4.9, 3.4], "parking" : "true", "rating" : 9 }, - { "_index": "hotels-index", "_id": "4", "location": [4.2, 4.6], "parking" : "false", "rating" : 6}, - { "_index": "hotels-index", "_id": "5", "location": [3.3, 4.5], "parking" : "true", "rating" : 8 }, - { "_index": "hotels-index", "_id": "6", "location": [6.4, 3.4], "parking" : "true", "rating" : 9 }, - { "_index": "hotels-index", "_id": "7", "location": [4.2, 6.2], "parking" : "true", "rating" : 5 }, - { "_index": "hotels-index", "_id": "8", "location": [2.4, 4.0], "parking" : "true", "rating" : 8 }, - { "_index": "hotels-index", "_id": "9", "location": [1.4, 3.2], "parking" : "false", "rating" : 5 }, - { "_index": "hotels-index", "_id": "10", "location": [7.0, 9.9], "parking" : "true", "rating" : 9 }, - { "_index": "hotels-index", "_id": "11", "location": [3.0, 2.3], "parking" : "false", "rating" : 6 }, - { "_index": "hotels-index", "_id": "12", "location": [5.0, 1.0], "parking" : "true", "rating" : 3 }, + { + "_index": "hotels-index", + "_id": "1", + "location": [5.2, 4.4], + "parking": "true", + "rating": 5, + }, + { + "_index": "hotels-index", + "_id": "2", + "location": [5.2, 3.9], + "parking": "false", + "rating": 4, + }, + { + "_index": "hotels-index", + "_id": "3", + "location": [4.9, 3.4], + "parking": "true", + "rating": 9, + }, + { + "_index": "hotels-index", + "_id": "4", + "location": [4.2, 4.6], + "parking": "false", + "rating": 6, + }, + { + "_index": "hotels-index", + "_id": "5", + "location": [3.3, 4.5], + "parking": "true", + "rating": 8, + }, + { + "_index": "hotels-index", + "_id": "6", + "location": [6.4, 3.4], + "parking": "true", + "rating": 9, + }, + { + "_index": "hotels-index", + "_id": "7", + "location": [4.2, 6.2], + "parking": "true", + "rating": 5, + }, + { + "_index": "hotels-index", + "_id": "8", + "location": [2.4, 4.0], + "parking": "true", + "rating": 8, + }, + { + "_index": "hotels-index", + "_id": "9", + "location": [1.4, 3.2], + "parking": "false", + "rating": 5, + }, + { + "_index": "hotels-index", + "_id": "10", + "location": [7.0, 9.9], + "parking": "true", + "rating": 9, + }, + { + "_index": "hotels-index", + "_id": "11", + "location": [3.0, 2.3], + "parking": "false", + "rating": 6, + }, + { + "_index": "hotels-index", + "_id": "12", + "location": [5.0, 1.0], + "parking": "true", + "rating": 3, + }, ] helpers.bulk(client, vectors) @@ -87,30 +157,19 @@ "query": { "knn": { "location": { - "vector": [5, 4], - "k": 3, - "filter": { - "bool": { - "must": [ - { - "range": { - "rating": { - "gte": 8, - "lte": 10 - } - } - }, - { - "term": { - "parking": "true" - } - } + "vector": [5, 4], + "k": 3, + "filter": { + "bool": { + "must": [ + {"range": {"rating": {"gte": 8, "lte": 10}}}, + {"term": {"parking": "true"}}, ] } - } + }, } } - } + }, } results = client.search(index=index_name, body=search_query) diff --git a/samples/security/roles.py b/samples/security/roles.py index 0f0f2f61..8a2d1ef5 100644 --- a/samples/security/roles.py +++ b/samples/security/roles.py @@ -6,6 +6,9 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. # A basic OpenSearch sample that create and manage roles. @@ -14,16 +17,16 @@ # connect to OpenSearch -host = 'localhost' +host = "localhost" port = 9200 -auth = ('admin', 'admin') # For testing only. Don't store credentials in code. +auth = ("admin", "admin") # For testing only. Don't store credentials in code. client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # Create a Role @@ -31,16 +34,16 @@ role_name = "test-role" role_content = { - "cluster_permissions": ["cluster_monitor"], - "index_permissions": [ - { - "index_patterns": ["index", "test-*"], - "allowed_actions": [ - "data_access", - "indices_monitor", - ], - } - ], + "cluster_permissions": ["cluster_monitor"], + "index_permissions": [ + { + "index_patterns": ["index", "test-*"], + "allowed_actions": [ + "data_access", + "indices_monitor", + ], + } + ], } response = client.security.create_role(role_name, body=role_content) diff --git a/samples/security/users.py b/samples/security/users.py index d33bd058..0a778b8d 100644 --- a/samples/security/users.py +++ b/samples/security/users.py @@ -6,6 +6,9 @@ # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. # A basic OpenSearch sample that create and manage users. @@ -14,16 +17,16 @@ # connect to OpenSearch -host = 'localhost' +host = "localhost" port = 9200 -auth = ('admin', 'admin') # For testing only. Don't store credentials in code. +auth = ("admin", "admin") # For testing only. Don't store credentials in code. client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_auth = auth, - use_ssl = True, - verify_certs = False, - ssl_show_warn = False + hosts=[{"host": host, "port": port}], + http_auth=auth, + use_ssl=True, + verify_certs=False, + ssl_show_warn=False, ) # Create a User diff --git a/test_opensearchpy/run_tests.py b/test_opensearchpy/run_tests.py index 7c8eb9ca..55f1e586 100755 --- a/test_opensearchpy/run_tests.py +++ b/test_opensearchpy/run_tests.py @@ -1,10 +1,14 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: Apache-2.0 # # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + # # Modifications Copyright OpenSearch Contributors. See # GitHub history for details. diff --git a/utils/generate-api.py b/utils/generate-api.py index 049038a4..7e241236 100644 --- a/utils/generate-api.py +++ b/utils/generate-api.py @@ -1,10 +1,14 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: Apache-2.0 # # The OpenSearch Contributors require contributions made to # this file be licensed under the Apache-2.0 license or a # compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + # # Modifications Copyright OpenSearch Contributors. See # GitHub history for details. diff --git a/utils/license-headers.py b/utils/license-headers.py index 255097d8..67b0ef4a 100644 --- a/utils/license-headers.py +++ b/utils/license-headers.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # SPDX-License-Identifier: Apache-2.0 # # The OpenSearch Contributors require contributions made to @@ -6,24 +7,6 @@ # # Modifications Copyright OpenSearch Contributors. See # GitHub history for details. -# -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - """Script which verifies that all source files have a license header. Has two modes: 'fix' and 'check'. 'fix' fixes problems, 'check' will @@ -33,20 +16,20 @@ import os import re import sys -from itertools import chain from typing import Iterator, List -lines_to_keep = ["# -*- coding: utf-8 -*-\n", "#!/usr/bin/env python\n"] -license_header_lines = [ - "# SPDX-License-Identifier: Apache-2.0\n", - "#\n", - "# The OpenSearch Contributors require contributions made to\n", - "# this file be licensed under the Apache-2.0 license or a\n", - "# compatible open source license.\n", - "#\n", - "# Modifications Copyright OpenSearch Contributors. See\n", - "# GitHub history for details.\n", -] +lines_to_keep = ["# -*- coding: utf-8 -*-", "#!/usr/bin/env python"] + +license_header = """ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +""".strip() def find_files_to_fix(sources: List[str]) -> Iterator[str]: @@ -67,20 +50,18 @@ def find_files_to_fix(sources: List[str]) -> Iterator[str]: def does_file_need_fix(filepath: str) -> bool: if not re.search(r"\.pyi?$", filepath): return False + existing_header = "" with open(filepath, mode="r") as f: - first_license_line = None for line in f: - if line == license_header_lines[0]: - first_license_line = line + line = line.strip() + if len(line) == 0 or line in lines_to_keep: + pass + elif line[0] == "#": + existing_header += line + existing_header += "\n" + else: break - elif line not in lines_to_keep: - return True - for header_line, line in zip( - license_header_lines, chain((first_license_line,), f) - ): - if line != header_line: - return True - return False + return not existing_header.startswith(license_header) def add_header_to_file(filepath: str) -> None: @@ -88,9 +69,9 @@ def add_header_to_file(filepath: str) -> None: lines = list(f) i = 0 for i, line in enumerate(lines): - if line not in lines_to_keep: + if len(line) > 0 and line not in lines_to_keep: break - lines = lines[:i] + license_header_lines + lines[i:] + lines = lines[:i] + [license_header] + lines[i:] with open(filepath, mode="w") as f: f.truncate() f.write("".join(lines))