Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DPE-4412] Use TLS CA chain for backups #484

Merged
merged 5 commits into from
Jun 13, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions src/backups.py
Original file line number Diff line number Diff line change
@@ -83,6 +83,14 @@ def stanza_name(self) -> str:
"""Stanza name, composed by model and cluster name."""
return f"{self.model.name}.{self.charm.cluster_name}"

@property
def _tls_ca_chain_filename(self) -> str:
"""Returns the path to the TLS CA chain file."""
s3_parameters, _ = self._retrieve_s3_parameters()
if s3_parameters.get("tls-ca-chain") is not None:
return f"{self.charm._storage_path}/pgbackrest-tls-ca-chain.crt"
return ""

def _are_backup_settings_ok(self) -> Tuple[bool, Optional[str]]:
"""Validates whether backup settings are OK."""
if self.model.get_relation(self.relation_name) is None:
@@ -234,7 +242,11 @@ def _create_bucket_if_not_exists(self) -> None:
)

try:
s3 = session.resource("s3", endpoint_url=self._construct_endpoint(s3_parameters))
s3 = session.resource(
"s3",
endpoint_url=self._construct_endpoint(s3_parameters),
verify=(self._tls_ca_chain_filename or None),
)
except ValueError as e:
logger.exception("Failed to create a session '%s' in region=%s.", bucket_name, region)
raise e
@@ -896,6 +908,11 @@ def _render_pgbackrest_conf_file(self) -> bool:
)
return False

if self._tls_ca_chain_filename != "":
self.charm._patroni.render_file(
self._tls_ca_chain_filename, "\n".join(s3_parameters["tls-ca-chain"]), 0o644
)

with open("templates/pgbackrest.conf.j2", "r") as file:
template = Template(file.read())
# Render the template file with the correct values.
@@ -909,6 +926,7 @@ def _render_pgbackrest_conf_file(self) -> bool:
endpoint=s3_parameters["endpoint"],
bucket=s3_parameters["bucket"],
s3_uri_style=s3_parameters["s3-uri-style"],
tls_ca_chain=self._tls_ca_chain_filename,
access_key=s3_parameters["access-key"],
secret_key=s3_parameters["secret-key"],
stanza=self.stanza_name,
@@ -1029,7 +1047,11 @@ def _upload_content_to_s3(
region_name=s3_parameters["region"],
)

s3 = session.resource("s3", endpoint_url=self._construct_endpoint(s3_parameters))
s3 = session.resource(
"s3",
endpoint_url=self._construct_endpoint(s3_parameters),
verify=(self._tls_ca_chain_filename or None),
)
bucket = s3.Bucket(bucket_name)

with tempfile.NamedTemporaryFile() as temp_file:
3 changes: 3 additions & 0 deletions templates/pgbackrest.conf.j2
Original file line number Diff line number Diff line change
@@ -11,6 +11,9 @@ repo1-s3-region={{ region }}
repo1-s3-endpoint={{ endpoint }}
repo1-s3-bucket={{ bucket }}
repo1-s3-uri-style={{ s3_uri_style }}
{%- if tls_ca_chain != '' %}
repo1-s3-ca-file={{ tls_ca_chain }}
{%- endif %}
repo1-s3-key={{ access_key }}
repo1-s3-key-secret={{ secret_key }}
repo1-block=y
96 changes: 85 additions & 11 deletions tests/unit/test_backups.py
Original file line number Diff line number Diff line change
@@ -50,6 +50,33 @@ def test_stanza_name(harness):
)


def test_tls_ca_chain_filename(harness):
# Test when the TLS CA chain is not available.
tc.assertEqual(
harness.charm.backup._tls_ca_chain_filename,
"",
)

# Test when the TLS CA chain is available.
with harness.hooks_disabled():
remote_application = "s3-integrator"
s3_rel_id = harness.add_relation(S3_PARAMETERS_RELATION, remote_application)
harness.update_relation_data(
s3_rel_id,
remote_application,
{
"bucket": "fake-bucket",
"access-key": "fake-access-key",
"secret-key": "fake-secret-key",
"tls-ca-chain": '["fake-tls-ca-chain"]',
},
)
tc.assertEqual(
harness.charm.backup._tls_ca_chain_filename,
"/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt",
)


def test_are_backup_settings_ok(harness):
# Test without S3 relation.
tc.assertEqual(
@@ -401,9 +428,17 @@ def test_construct_endpoint(harness):
)


def test_create_bucket_if_not_exists(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename",
["", "/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt"],
)
def test_create_bucket_if_not_exists(harness, tls_ca_chain_filename):
with (
patch("boto3.session.Session.resource") as _resource,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
):
# Test when there are missing S3 parameters.
@@ -427,11 +462,15 @@ def test_create_bucket_if_not_exists(harness):
harness.charm.backup._create_bucket_if_not_exists()

# Test when the bucket already exists.
_resource.reset_mock()
_resource.side_effect = None
head_bucket = _resource.return_value.Bucket.return_value.meta.client.head_bucket
create = _resource.return_value.Bucket.return_value.create
wait_until_exists = _resource.return_value.Bucket.return_value.wait_until_exists
harness.charm.backup._create_bucket_if_not_exists()
_resource.assert_called_once_with(
"s3", endpoint_url="test-endpoint", verify=(tls_ca_chain_filename or None)
)
head_bucket.assert_called_once()
create.assert_not_called()
wait_until_exists.assert_not_called()
@@ -1482,9 +1521,17 @@ def test_pre_restore_checks(harness):


@patch_network_get(private_address="1.1.1.1")
def test_render_pgbackrest_conf_file(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename",
["", "/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt"],
)
def test_render_pgbackrest_conf_file(harness, tls_ca_chain_filename):
with (
patch("charm.Patroni.render_file") as _render_file,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
):
# Set up a mock for the `open` method, set returned data to postgresql.conf template.
@@ -1513,6 +1560,7 @@ def test_render_pgbackrest_conf_file(harness):
"region": "us-east-1",
"s3-uri-style": "path",
"delete-older-than-days": "30",
"tls-ca-chain": (["fake-tls-ca-chain"] if tls_ca_chain_filename != "" else ""),
},
[],
)
@@ -1531,6 +1579,7 @@ def test_render_pgbackrest_conf_file(harness):
endpoint="https://storage.googleapis.com",
bucket="test-bucket",
s3_uri_style="path",
tls_ca_chain=(tls_ca_chain_filename or ""),
access_key="test-access-key",
secret_key="test-secret-key",
stanza=harness.charm.backup.stanza_name,
@@ -1548,11 +1597,16 @@ def test_render_pgbackrest_conf_file(harness):
tc.assertEqual(mock.call_args_list[0][0], ("templates/pgbackrest.conf.j2", "r"))

# Ensure the correct rendered template is sent to _render_file method.
_render_file.assert_called_once_with(
"/var/snap/charmed-postgresql/current/etc/pgbackrest/pgbackrest.conf",
expected_content,
0o644,
)
calls = [
call(
"/var/snap/charmed-postgresql/current/etc/pgbackrest/pgbackrest.conf",
expected_content,
0o644,
)
]
if tls_ca_chain_filename != "":
calls.insert(0, call(tls_ca_chain_filename, "fake-tls-ca-chain", 0o644))
_render_file.assert_has_calls(calls)


@patch_network_get(private_address="1.1.1.1")
@@ -1737,11 +1791,19 @@ def test_start_stop_pgbackrest_service(harness):
restart.assert_called_once()


def test_upload_content_to_s3(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename",
["", "/var/snap/charmed-postgresql/common/pgbackrest-tls-ca-chain.crt"],
)
def test_upload_content_to_s3(harness, tls_ca_chain_filename):
with (
patch("tempfile.NamedTemporaryFile") as _named_temporary_file,
patch("charm.PostgreSQLBackups._construct_endpoint") as _construct_endpoint,
patch("boto3.session.Session.resource") as _resource,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
):
# Set some parameters.
content = "test-content"
@@ -1764,7 +1826,11 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
False,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_not_called()
upload_file.assert_not_called()

@@ -1775,7 +1841,11 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
False,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_called_once()
upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")

@@ -1788,6 +1858,10 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
True,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_called_once()
upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")