From d86622439d5b2a0b81f28385a6bbec4a3de588f4 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Fri, 8 Nov 2024 18:25:59 -0800 Subject: [PATCH 1/7] feat: Add ssl credentials workaround with optional OpenSSL import --- python/lib/sift_py/grpc/transport.py | 27 +++++++++++++++++++++++++-- python/pyproject.toml | 4 ++++ python/scripts/dev | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/python/lib/sift_py/grpc/transport.py b/python/lib/sift_py/grpc/transport.py index cc14bbe1..4637b315 100644 --- a/python/lib/sift_py/grpc/transport.py +++ b/python/lib/sift_py/grpc/transport.py @@ -20,10 +20,33 @@ from sift_py.grpc._retry import RetryPolicy from sift_py.grpc.keepalive import DEFAULT_KEEPALIVE_CONFIG, KeepaliveConfig + SiftChannel: TypeAlias = grpc.Channel SiftAsyncChannel: TypeAlias = grpc_aio.Channel +def get_ssl_credentials() -> grpc.ChannelCredentials: + """ + Returns SSL credentials for use with gRPC. + Workaround for this issue: https://github.com/grpc/grpc/issues/29682 + """ + try: + import ssl + + from OpenSSL import crypto + + ssl_context = ssl.create_default_context() + certs_der = ssl_context.get_ca_certs(binary_form=True) + certs_x509 = [crypto.load_certificate(crypto.FILETYPE_ASN1, x) for x in certs_der] + certs_pem = [crypto.dump_certificate(crypto.FILETYPE_PEM, x) for x in certs_x509] + certs_bytes = b"".join(certs_pem) + + return grpc.ssl_channel_credentials(certs_bytes) + except ImportError: + return grpc.ssl_channel_credentials() + + + def use_sift_channel( config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None ) -> SiftChannel: @@ -41,7 +64,7 @@ def use_sift_channel( if not use_ssl: return _use_insecure_sift_channel(config, metadata) - credentials = grpc.ssl_channel_credentials() + credentials = get_ssl_credentials() options = _compute_channel_options(config) api_uri = _clean_uri(config["uri"], use_ssl) channel = grpc.secure_channel(api_uri, credentials, options) @@ -63,7 +86,7 @@ def use_sift_async_channel( return grpc_aio.secure_channel( target=_clean_uri(config["uri"], use_ssl), - credentials=grpc.ssl_channel_credentials(), + credentials=get_ssl_credentials(), options=_compute_channel_options(config), interceptors=_compute_sift_async_interceptors(config, metadata), ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 28b7d4bc..a33a3157 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -60,6 +60,10 @@ build = [ "pdoc==14.5.0", "build==1.2.1", ] +other = [ + "pyOpenSSL<24.0.0", + "cffi~=1.14", +] [build-system] requires = ["setuptools"] diff --git a/python/scripts/dev b/python/scripts/dev index 4a89d084..7590afa5 100755 --- a/python/scripts/dev +++ b/python/scripts/dev @@ -31,6 +31,7 @@ pip_install() { source venv/bin/activate pip install '.[development]' pip install '.[build]' + pip install '.[other]' pip install -e . } From b7bd77edf19e21a027ff4e0ca766f414a91eafbf Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 11 Nov 2024 08:08:18 -0800 Subject: [PATCH 2/7] chore: Lint fix --- python/lib/sift_py/grpc/transport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/lib/sift_py/grpc/transport.py b/python/lib/sift_py/grpc/transport.py index 4637b315..de4ff6ff 100644 --- a/python/lib/sift_py/grpc/transport.py +++ b/python/lib/sift_py/grpc/transport.py @@ -20,7 +20,6 @@ from sift_py.grpc._retry import RetryPolicy from sift_py.grpc.keepalive import DEFAULT_KEEPALIVE_CONFIG, KeepaliveConfig - SiftChannel: TypeAlias = grpc.Channel SiftAsyncChannel: TypeAlias = grpc_aio.Channel From 4f263228c69cd9d2dff6fcbe2af76da7d4b4675f Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 11 Nov 2024 08:12:50 -0800 Subject: [PATCH 3/7] chore: fmt --- python/lib/sift_py/grpc/transport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/lib/sift_py/grpc/transport.py b/python/lib/sift_py/grpc/transport.py index de4ff6ff..393a2339 100644 --- a/python/lib/sift_py/grpc/transport.py +++ b/python/lib/sift_py/grpc/transport.py @@ -45,7 +45,6 @@ def get_ssl_credentials() -> grpc.ChannelCredentials: return grpc.ssl_channel_credentials() - def use_sift_channel( config: SiftChannelConfig, metadata: Optional[Dict[str, Any]] = None ) -> SiftChannel: From c759ba938cd937503c83ec3d271734d8055ce03d Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 11 Nov 2024 08:22:53 -0800 Subject: [PATCH 4/7] chore: Add types-pyOpenSSL --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index a33a3157..04fa0417 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -62,6 +62,7 @@ build = [ ] other = [ "pyOpenSSL<24.0.0", + "types-pyOpenSSL<24.0.0", "cffi~=1.14", ] From d8e45d8d5b74b5994de7185e9ce58c211ee20622 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 11 Nov 2024 08:27:56 -0800 Subject: [PATCH 5/7] build: Install other --- .github/workflows/python_ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index 2d7922d3..67c49c0c 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -28,6 +28,7 @@ jobs: run: | python -m pip install --upgrade pip pip install '.[development]' + pip install '.[other]' pip install . - name: Lint From e411c06cc22fc9a4d11bc2bc5f88186318ce64d4 Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 11 Nov 2024 12:57:43 -0800 Subject: [PATCH 6/7] chore: PR feedback; configure via channel config --- .github/workflows/python_ci.yaml | 2 +- python/lib/sift_py/grpc/transport.py | 20 +++++++++++++++----- python/lib/sift_py/grpc/transport_test.py | 2 +- python/pyproject.toml | 2 +- python/scripts/dev | 2 +- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index 67c49c0c..f017d63f 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -28,7 +28,7 @@ jobs: run: | python -m pip install --upgrade pip pip install '.[development]' - pip install '.[other]' + pip install '.[openssl]' pip install . - name: Lint diff --git a/python/lib/sift_py/grpc/transport.py b/python/lib/sift_py/grpc/transport.py index 393a2339..f1d591a8 100644 --- a/python/lib/sift_py/grpc/transport.py +++ b/python/lib/sift_py/grpc/transport.py @@ -24,11 +24,14 @@ SiftAsyncChannel: TypeAlias = grpc_aio.Channel -def get_ssl_credentials() -> grpc.ChannelCredentials: +def get_ssl_credentials(cert_via_openssl: bool) -> grpc.ChannelCredentials: """ Returns SSL credentials for use with gRPC. Workaround for this issue: https://github.com/grpc/grpc/issues/29682 """ + if not cert_via_openssl: + return grpc.ssl_channel_credentials() + try: import ssl @@ -41,8 +44,8 @@ def get_ssl_credentials() -> grpc.ChannelCredentials: certs_bytes = b"".join(certs_pem) return grpc.ssl_channel_credentials(certs_bytes) - except ImportError: - return grpc.ssl_channel_credentials() + except ImportError as e: + raise Exception("Missing required dependencies for cert_via_openssl. Run `pip install sift-stack-py[openssl]` to install the required dependencies.") from e def use_sift_channel( @@ -58,11 +61,12 @@ def use_sift_channel( are exceeded, after which the underlying exception will be raised. """ use_ssl = config.get("use_ssl", True) + cert_via_openssl = config.get("cert_via_openssl", False) if not use_ssl: return _use_insecure_sift_channel(config, metadata) - credentials = get_ssl_credentials() + credentials = get_ssl_credentials(cert_via_openssl) options = _compute_channel_options(config) api_uri = _clean_uri(config["uri"], use_ssl) channel = grpc.secure_channel(api_uri, credentials, options) @@ -78,13 +82,14 @@ def use_sift_async_channel( of an async runtime when asynchonous I/O is required. """ use_ssl = config.get("use_ssl", True) + cert_via_openssl = config.get("cert_via_openssl", False) if not use_ssl: return _use_insecure_sift_async_channel(config, metadata) return grpc_aio.secure_channel( target=_clean_uri(config["uri"], use_ssl), - credentials=get_ssl_credentials(), + credentials=get_ssl_credentials(cert_via_openssl), options=_compute_channel_options(config), interceptors=_compute_sift_async_interceptors(config, metadata), ) @@ -215,9 +220,14 @@ class SiftChannelConfig(TypedDict): set to `True`, it will use the default values configured in `sift_py.grpc.keepalive` to configure keepalive. A custom `sift_py.grpc.keepalive.KeepaliveConfig` may also be provided. Default disabled. - `use_ssl`: INTERNAL USE. Meant to be used for local development. + - `cert_via_openssl`: Enable this if you want to use OpenSSL to load the certificates. + Run `pip install sift-stack-py[openssl]` to install the dependencies required to use this option. + This works around this issue with grpc loading SSL certificates: https://github.com/grpc/grpc/issues/29682. + Default is False. """ uri: str apikey: str enable_keepalive: NotRequired[Union[bool, KeepaliveConfig]] use_ssl: NotRequired[bool] + cert_via_openssl: NotRequired[bool] diff --git a/python/lib/sift_py/grpc/transport_test.py b/python/lib/sift_py/grpc/transport_test.py index a9e96fe4..ddf67ca2 100644 --- a/python/lib/sift_py/grpc/transport_test.py +++ b/python/lib/sift_py/grpc/transport_test.py @@ -16,7 +16,7 @@ ) from sift_py._internal.test_util.server_interceptor import ServerInterceptor -from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel +from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel, get_ssl_credentials class DataService(DataServiceServicer): diff --git a/python/pyproject.toml b/python/pyproject.toml index 04fa0417..12fa0321 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -60,7 +60,7 @@ build = [ "pdoc==14.5.0", "build==1.2.1", ] -other = [ +openssl = [ "pyOpenSSL<24.0.0", "types-pyOpenSSL<24.0.0", "cffi~=1.14", diff --git a/python/scripts/dev b/python/scripts/dev index 7590afa5..6cb00436 100755 --- a/python/scripts/dev +++ b/python/scripts/dev @@ -31,7 +31,7 @@ pip_install() { source venv/bin/activate pip install '.[development]' pip install '.[build]' - pip install '.[other]' + pip install '.[openssl]' pip install -e . } From 78a461434d4b0f25f81f585ca3310355b1e7ee7a Mon Sep 17 00:00:00 2001 From: Ailin Yu Date: Mon, 11 Nov 2024 15:05:47 -0800 Subject: [PATCH 7/7] chore: lint and format --- python/lib/sift_py/grpc/transport.py | 4 +++- python/lib/sift_py/grpc/transport_test.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/lib/sift_py/grpc/transport.py b/python/lib/sift_py/grpc/transport.py index f1d591a8..da9204f7 100644 --- a/python/lib/sift_py/grpc/transport.py +++ b/python/lib/sift_py/grpc/transport.py @@ -45,7 +45,9 @@ def get_ssl_credentials(cert_via_openssl: bool) -> grpc.ChannelCredentials: return grpc.ssl_channel_credentials(certs_bytes) except ImportError as e: - raise Exception("Missing required dependencies for cert_via_openssl. Run `pip install sift-stack-py[openssl]` to install the required dependencies.") from e + raise Exception( + "Missing required dependencies for cert_via_openssl. Run `pip install sift-stack-py[openssl]` to install the required dependencies." + ) from e def use_sift_channel( diff --git a/python/lib/sift_py/grpc/transport_test.py b/python/lib/sift_py/grpc/transport_test.py index ddf67ca2..a9e96fe4 100644 --- a/python/lib/sift_py/grpc/transport_test.py +++ b/python/lib/sift_py/grpc/transport_test.py @@ -16,7 +16,7 @@ ) from sift_py._internal.test_util.server_interceptor import ServerInterceptor -from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel, get_ssl_credentials +from sift_py.grpc.transport import SiftChannelConfig, use_sift_channel class DataService(DataServiceServicer):