Skip to content

Commit

Permalink
Use TLS CA chain for backups (#484)
Browse files Browse the repository at this point in the history 
Signed-off-by: Marcelo Henrique Neppel <[email protected]>
  • Loading branch information
@marceloneppel
marceloneppel authored Jun 13, 2024
1 parent f135759 commit 1f8ff44
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 13 deletions.
26 changes: 24 additions & 2 deletions src/backups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,14 @@ def stanza_name(self) -> str:
"""Stanza name, composed by model and cluster name."""
return f"{self.model.name}.{self.charm.cluster_name}"

@property
def _tls_ca_chain_filename(self) -> str:
"""Returns the path to the TLS CA chain file."""
s3_parameters, _ = self._retrieve_s3_parameters()
if s3_parameters.get("tls-ca-chain") is not None:
return f"{self.charm._storage_path}/pgbackrest-tls-ca-chain.crt"
return ""

def _are_backup_settings_ok(self) -> Tuple[bool, Optional[str]]:
"""Validates whether backup settings are OK."""
if self.model.get_relation(self.relation_name) is None:
Expand Down Expand Up @@ -234,7 +242,11 @@ def _create_bucket_if_not_exists(self) -> None:
)

try:
s3 = session.resource("s3", endpoint_url=self._construct_endpoint(s3_parameters))
s3 = session.resource(
"s3",
endpoint_url=self._construct_endpoint(s3_parameters),
verify=(self._tls_ca_chain_filename or None),
)
except ValueError as e:
logger.exception("Failed to create a session '%s' in region=%s.", bucket_name, region)
raise e
Expand Down Expand Up @@ -896,6 +908,11 @@ def _render_pgbackrest_conf_file(self) -> bool:
)
return False

if self._tls_ca_chain_filename != "":
self.charm._patroni.render_file(
self._tls_ca_chain_filename, "\n".join(s3_parameters["tls-ca-chain"]), 0o644
)

with open("templates/pgbackrest.conf.j2", "r") as file:
template = Template(file.read())
# Render the template file with the correct values.
Expand All @@ -909,6 +926,7 @@ def _render_pgbackrest_conf_file(self) -> bool:
endpoint=s3_parameters["endpoint"],
bucket=s3_parameters["bucket"],
s3_uri_style=s3_parameters["s3-uri-style"],
tls_ca_chain=self._tls_ca_chain_filename,
access_key=s3_parameters["access-key"],
secret_key=s3_parameters["secret-key"],
stanza=self.stanza_name,
Expand Down Expand Up @@ -1029,7 +1047,11 @@ def _upload_content_to_s3(
region_name=s3_parameters["region"],
)

s3 = session.resource("s3", endpoint_url=self._construct_endpoint(s3_parameters))
s3 = session.resource(
"s3",
endpoint_url=self._construct_endpoint(s3_parameters),
verify=(self._tls_ca_chain_filename or None),
)
bucket = s3.Bucket(bucket_name)

with tempfile.NamedTemporaryFile() as temp_file:
Expand Down
3 changes: 3 additions & 0 deletions templates/pgbackrest.conf.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ repo1-s3-region={{ region }}
repo1-s3-endpoint={{ endpoint }}
repo1-s3-bucket={{ bucket }}
repo1-s3-uri-style={{ s3_uri_style }}
{%- if tls_ca_chain != '' %}
repo1-s3-ca-file={{ tls_ca_chain }}
{%- endif %}
repo1-s3-key={{ access_key }}
repo1-s3-key-secret={{ secret_key }}
repo1-block=y
Expand Down
96 changes: 85 additions & 11 deletions tests/unit/test_backups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,33 @@ def test_stanza_name(harness):
)


def test_tls_ca_chain_filename(harness):
# Test when the TLS CA chain is not available.
tc.assertEqual(
harness.charm.backup._tls_ca_chain_filename,
"",
)

# Test when the TLS CA chain is available.
with harness.hooks_disabled():
remote_application = "s3-integrator"
s3_rel_id = harness.add_relation(S3_PARAMETERS_RELATION, remote_application)
harness.update_relation_data(
s3_rel_id,
remote_application,
{
"bucket": "fake-bucket",
"access-key": "fake-access-key",
"secret-key": "fake-secret-key",
"tls-ca-chain": '["fake-tls-ca-chain"]',
},
)
tc.assertEqual(
harness.charm.backup._tls_ca_chain_filename,
"/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt",
)


def test_are_backup_settings_ok(harness):
# Test without S3 relation.
tc.assertEqual(
Expand Down Expand Up @@ -401,9 +428,17 @@ def test_construct_endpoint(harness):
)


def test_create_bucket_if_not_exists(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename",
["", "/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt"],
)
def test_create_bucket_if_not_exists(harness, tls_ca_chain_filename):
with (
patch("boto3.session.Session.resource") as _resource,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
):
# Test when there are missing S3 parameters.
Expand All @@ -427,11 +462,15 @@ def test_create_bucket_if_not_exists(harness):
harness.charm.backup._create_bucket_if_not_exists()

# Test when the bucket already exists.
_resource.reset_mock()
_resource.side_effect = None
head_bucket = _resource.return_value.Bucket.return_value.meta.client.head_bucket
create = _resource.return_value.Bucket.return_value.create
wait_until_exists = _resource.return_value.Bucket.return_value.wait_until_exists
harness.charm.backup._create_bucket_if_not_exists()
_resource.assert_called_once_with(
"s3", endpoint_url="test-endpoint", verify=(tls_ca_chain_filename or None)
)
head_bucket.assert_called_once()
create.assert_not_called()
wait_until_exists.assert_not_called()
Expand Down Expand Up @@ -1482,9 +1521,17 @@ def test_pre_restore_checks(harness):


@patch_network_get(private_address="1.1.1.1")
def test_render_pgbackrest_conf_file(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename",
["", "/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt"],
)
def test_render_pgbackrest_conf_file(harness, tls_ca_chain_filename):
with (
patch("charm.Patroni.render_file") as _render_file,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
):
# Set up a mock for the `open` method, set returned data to postgresql.conf template.
Expand Down Expand Up @@ -1513,6 +1560,7 @@ def test_render_pgbackrest_conf_file(harness):
"region": "us-east-1",
"s3-uri-style": "path",
"delete-older-than-days": "30",
"tls-ca-chain": (["fake-tls-ca-chain"] if tls_ca_chain_filename != "" else ""),
},
[],
)
Expand All @@ -1531,6 +1579,7 @@ def test_render_pgbackrest_conf_file(harness):
endpoint="https://storage.googleapis.com",
bucket="test-bucket",
s3_uri_style="path",
tls_ca_chain=(tls_ca_chain_filename or ""),
access_key="test-access-key",
secret_key="test-secret-key",
stanza=harness.charm.backup.stanza_name,
Expand All @@ -1548,11 +1597,16 @@ def test_render_pgbackrest_conf_file(harness):
tc.assertEqual(mock.call_args_list[0][0], ("templates/pgbackrest.conf.j2", "r"))

# Ensure the correct rendered template is sent to _render_file method.
_render_file.assert_called_once_with(
"/var/snap/charmed-postgresql/current/etc/pgbackrest/pgbackrest.conf",
expected_content,
0o644,
)
calls = [
call(
"/var/snap/charmed-postgresql/current/etc/pgbackrest/pgbackrest.conf",
expected_content,
0o644,
)
]
if tls_ca_chain_filename != "":
calls.insert(0, call(tls_ca_chain_filename, "fake-tls-ca-chain", 0o644))
_render_file.assert_has_calls(calls)


@patch_network_get(private_address="1.1.1.1")
Expand Down Expand Up @@ -1737,11 +1791,19 @@ def test_start_stop_pgbackrest_service(harness):
restart.assert_called_once()


def test_upload_content_to_s3(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename",
["", "/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt"],
)
def test_upload_content_to_s3(harness, tls_ca_chain_filename):
with (
patch("tempfile.NamedTemporaryFile") as _named_temporary_file,
patch("charm.PostgreSQLBackups._construct_endpoint") as _construct_endpoint,
patch("boto3.session.Session.resource") as _resource,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
):
# Set some parameters.
content = "test-content"
Expand All @@ -1764,7 +1826,11 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
False,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_not_called()
upload_file.assert_not_called()

Expand All @@ -1775,7 +1841,11 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
False,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_called_once()
upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")

Expand All @@ -1788,6 +1858,10 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
True,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_called_once()
upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")

0 comments on commit 1f8ff44

Please sign in to comment.