diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 460fcb5..8f4e670 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -83,10 +83,6 @@ jobs: # skipping matrix items that aren't meaningful matrix: include: - - mysql-version: "8.0.32" - percona-version: "8.0.32-26-1.focal" - python-version: "3.9" - ubuntu-version: "20.04" - mysql-version: "8.0.32" percona-version: "8.0.32-26-1.focal" python-version: "3.10" @@ -99,6 +95,10 @@ jobs: percona-version: "8.0.35-30-1.focal" python-version: "3.11" ubuntu-version: "20.04" + - mysql-version: "8.0.35" + percona-version: "8.0.35-30-1.focal" + python-version: "3.12" + ubuntu-version: "20.04" steps: - id: checkout-code diff --git a/README.md b/README.md index be71228..40a9529 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ backed up. # Requirements -MyHoard requires Python 3.9 or later and some additional components to operate: +MyHoard requires Python 3.10 or later and some additional components to operate: - [percona-xtrabackup](https://github.com/percona/percona-xtrabackup) - [python3-PyMySQL](https://github.com/PyMySQL/PyMySQL) diff --git a/myhoard/backup_stream.py b/myhoard/backup_stream.py index e9ff3ca..81ca7cc 100644 --- a/myhoard/backup_stream.py +++ b/myhoard/backup_stream.py @@ -1,6 +1,4 @@ # Copyright (c) 2019 Aiven, Helsinki, Finland. https://aiven.io/ -from rohmu.errors import FileNotFoundFromStorageError - from .append_only_state_manager import AppendOnlyStateManager from .basebackup_operation import BasebackupOperation from .binary_io_slice import BinaryIOSlice @@ -33,11 +31,13 @@ from rohmu import errors as rohmu_errors from rohmu.compressor import CompressionStream from rohmu.encryptor import EncryptorStream +from rohmu.errors import FileNotFoundFromStorageError from rohmu.object_storage.s3 import S3Transfer from socket import gaierror from socks import GeneralProxyError, ProxyConnectionError from ssl import SSLEOFError from typing import Any, Callable, cast, Dict, Iterable, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, TypedDict + import contextlib import enum import json @@ -256,6 +256,8 @@ def __init__( "stream_id": stream_id, "updated_at": time.time(), "valid_local_binlog_found": False, + "number_of_splits": None, + "split_size": None, } self.state_manager = StateManager(lock=self.lock, state=self.state, state_file=state_file) self.stats = stats @@ -672,8 +674,7 @@ def upload_progress(bytes_sent): # a separate measurement because uploads are not ongoing all the time and calculating # rate based on raw byte counter requires knowing when the operation started and ended self.stats.gauge_int( - "myhoard.backup_stream.basebackup_bytes_uploaded", - self.basebackup_bytes_uploaded + bytes_sent + "myhoard.backup_stream.basebackup_bytes_uploaded", self.basebackup_bytes_uploaded + bytes_sent ) last_value[0], last_time[0] = track_rate( current=bytes_sent, @@ -698,10 +699,7 @@ def upload_progress(bytes_sent): self.log.info("Uploading basebackup to %s", storage_file_name) file_storage.store_file_object( - storage_file_name, - stream_to_use, - metadata=metadata, - upload_progress_fn=upload_progress + storage_file_name, stream_to_use, metadata=metadata, upload_progress_fn=upload_progress ) # Unfortunately, at least for GCP, the upload_progress_fn doesn't get called for the last chunk, @@ -714,12 +712,14 @@ def upload_progress(bytes_sent): if not self.split_size or self.split_size < 1: break - metadata = make_fs_metadata({ - **metadata_template, - "number_of_splits": split_nr, - "split_size": self.split_size, - "basebackup_compressed_size": self.basebackup_bytes_uploaded, - }) + metadata = make_fs_metadata( + { + **metadata_template, + "number_of_splits": split_nr, + "split_size": self.split_size, + "basebackup_compressed_size": self.basebackup_bytes_uploaded, + } + ) self.state_manager.update_state(basebackup_file_metadata=metadata) self.log.info("Done uploading basebackup files, split into %d files", split_nr) @@ -1265,9 +1265,9 @@ def _upload_binlog(self, binlog: BinlogInfo) -> bool: index_name, encrypt_stream, metadata=metadata, - upload_progress_fn=self.binlog_progress_tracker.increment - if self.binlog_progress_tracker - else None, + upload_progress_fn=( + self.binlog_progress_tracker.increment if self.binlog_progress_tracker else None + ), ) self.stats.increase("myhoard.binlog.upload") if self.file_uploaded_callback: diff --git a/myhoard/basebackup_operation.py b/myhoard/basebackup_operation.py index efb3183..e2095bf 100644 --- a/myhoard/basebackup_operation.py +++ b/myhoard/basebackup_operation.py @@ -1,8 +1,8 @@ # Copyright (c) 2019 Aiven, Helsinki, Finland. https://aiven.io/ from contextlib import suppress -from distutils.version import LooseVersion # pylint:disable=deprecated-module from myhoard.errors import BlockMismatchError, XtraBackupError from myhoard.util import get_mysql_version, mysql_cursor +from packaging.version import Version from rohmu.util import increase_pipe_capacity, set_stream_nonblocking from typing import Optional @@ -156,7 +156,7 @@ def _optimize_tables(self) -> None: params["timeout"] = CURSOR_TIMEOUT_DURING_OPTIMIZE with mysql_cursor(**params) as cursor: version = get_mysql_version(cursor) - if LooseVersion(version) < LooseVersion("8.0.29"): + if Version(version) < Version("8.0.29"): return # allow OPTIMIZE TABLE to run on tables without primary keys diff --git a/myhoard/binary_io_slice.py b/myhoard/binary_io_slice.py index 81fb85a..67f4eb8 100644 --- a/myhoard/binary_io_slice.py +++ b/myhoard/binary_io_slice.py @@ -1,6 +1,6 @@ # Copyright (c) 2024 Aiven, Helsinki, Finland. https://aiven.io/ from types import TracebackType -from typing import BinaryIO, Type, Iterable, Iterator +from typing import BinaryIO, Iterable, Iterator, Type class MethodNotSupportedError(Exception): @@ -14,7 +14,7 @@ def __init__(self, max_file_size: int, stream: BinaryIO): self._size_remaining = max_file_size self.stream = stream - def read(self, __n: int = ...) -> bytes: + def read(self, __n: int = -1) -> bytes: if __n < 0: to_read = self._size_remaining else: @@ -53,13 +53,13 @@ def isatty(self) -> bool: def readable(self) -> bool: return self.stream.readable() - def readline(self, __limit: int = ...) -> bytes: + def readline(self, __limit: int = -1) -> bytes: raise MethodNotSupportedError() - def readlines(self, __hint: int = ...) -> list[bytes]: + def readlines(self, __hint: int = -1) -> list[bytes]: raise MethodNotSupportedError() - def seek(self, __offset: int, __whence: int = ...) -> int: + def seek(self, __offset: int, __whence: int = 0) -> int: return self.stream.seek(__offset, __whence) def seekable(self) -> bool: @@ -68,7 +68,7 @@ def seekable(self) -> bool: def tell(self) -> int: return self._max_file_size - self._size_remaining - def truncate(self, __size: int | None = ...) -> int: + def truncate(self, __size: int | None = None) -> int: raise MethodNotSupportedError() def writable(self) -> bool: @@ -90,6 +90,6 @@ def __enter__(self) -> BinaryIO: return self.stream.__enter__() def __exit__( - self, __t: Type[BaseException] | None, __value: BaseException | None, __traceback: TracebackType | None + self, __t: Type[BaseException] | None, __value: BaseException | None, __traceback: TracebackType | None ) -> None: return self.stream.__exit__(__t, __value, __traceback) diff --git a/myhoard/restore_coordinator.py b/myhoard/restore_coordinator.py index d433f56..69b574a 100644 --- a/myhoard/restore_coordinator.py +++ b/myhoard/restore_coordinator.py @@ -855,7 +855,7 @@ def _load_file_data(self, name, missing_ok=False): def _basebackup_data_provider(self, target_stream) -> None: compressed_size = self.state["basebackup_info"].get("compressed_size") - with (self.file_storage_pool.with_transfer(self.file_storage_config) as file_storage): + with self.file_storage_pool.with_transfer(self.file_storage_config) as file_storage: last_time = [time.monotonic()] last_value = [0] self.basebackup_bytes_downloaded = 0 @@ -865,8 +865,9 @@ def _basebackup_data_provider(self, target_stream) -> None: def download_progress(progress, max_progress): if progress and max_progress and compressed_size: # progress may be the actual number of bytes or it may be percentages - self.basebackup_bytes_downloaded = total_at_last_split_download[0] \ - + int(current_file_size[0] * progress / max_progress) + self.basebackup_bytes_downloaded = total_at_last_split_download[0] + int( + current_file_size[0] * progress / max_progress + ) # Track both absolute number and explicitly calculated rate. The rate can be useful as # a separate measurement because downloads are not ongoing all the time and calculating diff --git a/pyproject.toml b/pyproject.toml index 3623b17..0fdfacd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,16 +10,16 @@ authors = [ ] description = "MyHoard is a daemon for creating, managing and restoring MySQL backups." readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ] @@ -37,6 +37,7 @@ dependencies = [ "python-snappy == 0.6.1", "rohmu >= 1.1.2", "sentry-sdk==1.14.0", + "packaging", ] [project.optional-dependencies] diff --git a/test/test_backup_stream.py b/test/test_backup_stream.py index 199f0f0..ebb79f0 100644 --- a/test/test_backup_stream.py +++ b/test/test_backup_stream.py @@ -1,6 +1,4 @@ # Copyright (c) 2019 Aiven, Helsinki, Finland. https://aiven.io/ -import rohmu - from . import build_statsd_client, generate_rsa_key_pair, MySQLConfig, wait_for_condition from myhoard.backup_stream import BackupStream from myhoard.binlog_scanner import BinlogScanner @@ -13,6 +11,7 @@ import myhoard.util as myhoard_util import os import pytest +import rohmu pytestmark = [pytest.mark.unittest, pytest.mark.all] @@ -150,6 +149,7 @@ def _run_backup_stream_test(session_tmpdir, mysql_master: MySQLConfig, backup_st "directory": backup_target_location, "storage_type": "local", }, + "split_size": None, } } backups = Controller.get_backup_list(backup_sites) diff --git a/test/test_basebackup_operation.py b/test/test_basebackup_operation.py index 94c9522..51c4258 100644 --- a/test/test_basebackup_operation.py +++ b/test/test_basebackup_operation.py @@ -1,7 +1,7 @@ # Copyright (c) 2019 Aiven, Helsinki, Finland. https://aiven.io/ from . import build_statsd_client, MySQLConfig, restart_mysql -from distutils.version import LooseVersion # pylint:disable=deprecated-module from myhoard.basebackup_operation import BasebackupOperation +from packaging.version import Version from typing import IO from unittest import SkipTest from unittest.mock import mock_open, patch @@ -139,7 +139,7 @@ def stream_handler(_stream): def test_backup_with_non_optimized_tables(mysql_master: MySQLConfig) -> None: with myhoard_util.mysql_cursor(**mysql_master.connect_options) as cursor: version = myhoard_util.get_mysql_version(cursor) - if LooseVersion(version) < LooseVersion("8.0.29"): + if Version(version) < Version("8.0.29"): raise SkipTest("DB version doesn't need OPTIMIZE TABLE") def create_test_db(*, db_name: str, table_name: str, add_pk: bool) -> None: diff --git a/test/test_restore_coordinator.py b/test/test_restore_coordinator.py index 57bd2b0..eb07644 100644 --- a/test/test_restore_coordinator.py +++ b/test/test_restore_coordinator.py @@ -24,7 +24,12 @@ def test_restore_coordinator(session_tmpdir, mysql_master, mysql_empty): def test_restore_coordinator_with_split_basebackup(session_tmpdir, mysql_master, mysql_empty): _restore_coordinator_sequence( - session_tmpdir, mysql_master, mysql_empty, pitr=False, rebuild_tables=False, fail_and_resume=False, + session_tmpdir, + mysql_master, + mysql_empty, + pitr=False, + rebuild_tables=False, + fail_and_resume=False, split_size=10_000, ) @@ -48,7 +53,14 @@ def test_restore_coordinator_resume_rebuild_tables(session_tmpdir, mysql_master, def _restore_coordinator_sequence( - session_tmpdir, mysql_master, mysql_empty, *, pitr: bool, rebuild_tables: bool, fail_and_resume: bool, split_size: int=0, + session_tmpdir, + mysql_master, + mysql_empty, + *, + pitr: bool, + rebuild_tables: bool, + fail_and_resume: bool, + split_size: int = 0, ): with myhoard_util.mysql_cursor(**mysql_master.connect_options) as cursor: cursor.execute("CREATE DATABASE db1") @@ -428,6 +440,7 @@ def test_empty_last_relay(running_state, session_tmpdir, mysql_master, mysql_emp def test_restore_coordinator_check_parameter_before_restart(session_tmpdir): + # pylint: disable=W0212,W0108 restarts = [] state_file = os.path.join(session_tmpdir().strpath, "the_state_file.json") @@ -436,10 +449,8 @@ def _register_restart(**kwargs): restarts.append({**kwargs}) rc = RestoreCoordinator( - binlog_streams=[ - ], - file_storage_config={ - }, + binlog_streams=[], + file_storage_config={}, free_memory_percentage=80, mysql_client_params={}, mysql_config_file_name="",