From b20313cd0a5224a9b97996c3b0f3e4fd70940f65 Mon Sep 17 00:00:00 2001 From: petersilva Date: Thu, 8 Feb 2024 12:15:07 -0500 Subject: [PATCH 1/8] tag version stable --- debian/changelog | 2 +- sarracenia/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b178c3c2b..14ffe646e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -metpx-sr3 (3.00.51rc7) unstable; urgency=medium +metpx-sr3 (3.00.51) unstable; urgency=medium * PR #910 detect v02 messages without content_type header. * PR #900 major revamp of AM protocol reception. diff --git a/sarracenia/_version.py b/sarracenia/_version.py index 20a58355e..068fe814b 100755 --- a/sarracenia/_version.py +++ b/sarracenia/_version.py @@ -1 +1 @@ -__version__ = "3.00.51rc7" +__version__ = "3.00.51" From 75ce6d01451667fb581cca06623e4a928b9d61d9 Mon Sep 17 00:00:00 2001 From: petersilva Date: Thu, 8 Feb 2024 12:27:15 -0500 Subject: [PATCH 2/8] adjust for older os and python versions --- debian/control | 2 +- pyproject.toml | 206 ------------------------------------------------- 2 files changed, 1 insertion(+), 207 deletions(-) delete mode 100644 pyproject.toml diff --git a/debian/control b/debian/control index 13c64f987..e29198d3a 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: metpx-sr3 Section: python Priority: optional Maintainer: Shared Services Canada Supercomputing -Build-Depends: debhelper (>= 9), dh-python, python3, python3-appdirs, python3-all, python3-setuptools, python3-docutils, python3-jsonpickle, python3-humanfriendly, python3-humanize, python3-paramiko, python3-psutil, python3-watchdog, pybuild-plugin-pyproject +Build-Depends: debhelper (>= 9), dh-python, python3, python3-appdirs, python3-all, python3-setuptools, python3-docutils, python3-jsonpickle, python3-humanfriendly, python3-humanize, python3-paramiko, python3-psutil, python3-watchdog Standards-Version: 3.9.5 X-Python3-Version: >= 3.6 Vcs-Git: https://github.com/MetPX/sarracenia diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 358ffc62d..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,206 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "metpx-sr3" -dynamic = ["version"] -description = 'Subscribe, Acquire, and Re-Advertise (managed multiple hop file transfers)' -readme = "README.rst" -requires-python = ">=3.7" -license = "GPL-2.0-only" -keywords = ["file transfer"] -authors = [ - { name = "Peter Silva", email = "peter.silva@ssc-spc.gc.ca" }, - { name = "Reid Sunderland", email = "" }, - { name = "André Leblanc", email = "" }, - { name = "Michel Grenier", email ="" }, - { name = "Greg Linton", email ="" }, - { name = "Cléa Aumont", email ="" }, - { name = "Cody Au", email ="" }, - { name = "Benoit Lapointe", email ="" }, -] - -classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] - -dependencies =[ - "appdirs", "humanfriendly", "humanize", "jsonpickle", "paramiko", - "psutil >= 5.3.0", "watchdog", -] - -# NOTE: -# on Redhat, when building an rpm, need to replace the -# python-magic (last two lines of dependencies above ) -# by either nothing (older releases) or *file-magic* (fc39 -# 'file-magic' - -[project.scripts] -sr3 = "sarracenia.sr:main" -sr3_post = "sarracenia.sr_post:main" -sr3_rotateLogsManually = "sarracenia.sr_rotateLogsManually:main" -sr3_tailf = "sarracenia.sr_tailf:main" - -# future entry points: -#sr3_poll="sarracenia.sr_flow:main" -#sr3_report="sarracenia.sr_flow:main" -#sr3_watch="sarracenia.sr_flow:main" -#sr3_winnow="sarracenia.sr_flow:main" -#sr3_sarra="sarracenia.sr_flow:main" -#sr3_shovel="sarracenia.sr_flow:main" -#sr3_sender="sarracenia.sr_flow:main" -#sr3_subscribe="sarracenia.sr_flow:main" - -[project.optional-dependencies] - -amqp = [ "amqp" ] -ftppoll = [ 'dateparser' ] -mqtt = [ 'paho.mqtt>=1.5.1' ] -vip = [ 'netifaces' ] -redis = [ 'redis' ] -filetypes-redhat = [ 'file-magic' ] -filetypes = [ 'python-magic' ] - - -[project.urls] -Documentation = "https://metpx.github.io/sarracenia" -Issues = "https://github.com/MetPX/sarracenia/issues" -Source = "https://github.com/MetPX/sarracenia.git" - -[tool.hatch.version] -path = "sarracenia/_version.py" - -[tool.hatch.build] -include = [ "sarracenia" ] -exclude = [ "tests", "debian" ] - -[tool.hatch.envs.default] -dependencies = [ - "coverage[toml]>=6.5", - "pytest", -] -[tool.hatch.envs.default.scripts] -test = "pytest {args:tests}" -test-cov = "coverage run -m pytest {args:tests}" -cov-report = [ - "- coverage combine", - "coverage report", -] -cov = [ - "test-cov", - "cov-report", -] - -[[tool.hatch.envs.all.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11"] - -[tool.hatch.envs.lint] -detached = true -dependencies = [ - "black>=23.1.0", - "mypy>=1.0.0", - "ruff>=0.0.243", -] -[tool.hatch.envs.lint.scripts] -typing = "mypy --install-types --non-interactive {args: tests}" -style = [ - "ruff {args:.}", - "black --check --diff {args:.}", -] -fmt = [ - "black {args:.}", - "ruff --fix {args:.}", - "style", -] -all = [ - "style", - "typing", -] - -[tool.black] -target-version = ["py37"] -line-length = 120 -skip-string-normalization = true - -[tool.ruff] -target-version = "py37" -line-length = 120 -select = [ - "A", - "ARG", - "B", - "C", - "DTZ", - "E", - "EM", - "F", - "FBT", - "I", - "ICN", - "ISC", - "N", - "PLC", - "PLE", - "PLR", - "PLW", - "Q", - "RUF", - "S", - "T", - "TID", - "UP", - "W", - "YTT", -] -ignore = [ - # Allow non-abstract empty methods in abstract base classes - "B027", - # Allow boolean positional values in function calls, like `dict.get(... True)` - "FBT003", - # Ignore checks for possible passwords - "S105", "S106", "S107", - # Ignore complexity - "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", -] -unfixable = [ - # Don't touch unused imports - "F401", -] - -[tool.ruff.isort] -known-first-party = ["temporary"] - -[tool.ruff.flake8-tidy-imports] -ban-relative-imports = "all" - -[tool.ruff.per-file-ignores] -# Tests can use magic values, assertions, and relative imports -"tests/**/*" = ["PLR2004", "S101", "TID252"] - -[tool.coverage.run] -source_pkgs = ["temporary", "tests"] -branch = true -parallel = true -omit = [ - "src/temporary/__about__.py", -] - -[tool.coverage.paths] -temporary = ["src/temporary", "*/temporary/src/temporary"] -tests = ["tests", "*/temporary/tests"] - -[tool.coverage.report] -exclude_lines = [ - "no cov", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", -] From bc04a8119fb5f1ef4e8ba9028a2e17a0fe02b832 Mon Sep 17 00:00:00 2001 From: petersilva Date: Thu, 14 Mar 2024 16:04:09 -0400 Subject: [PATCH 3/8] tagging release version --- debian/changelog | 17 +---------------- sarracenia/_version.py | 2 +- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/debian/changelog b/debian/changelog index 45730f780..e3654188c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,36 +1,21 @@ -metpx-sr3 (3.00.52rc4) unstable; urgency=medium +metpx-sr3 (3.00.52) unstable; urgency=medium * copernicus marine data store polling support ( #959 ) * override baseUrl with message field ( for #951 ) * os.kill for older python version instead of raise_signal #948 * fix #955 error on cleanup when cache dir missing. * gather/am fix to handle improperly parsed station names from getStation - - -- Peter Silva Thu, 07 Mar 2024 16:28:26 -0500 - -metpx-sr3 (3.00.52rc3) unstable; urgency=medium - * fix #953 PRECONDITION FAILED error on busy winnows * above fix breaks/reopens #649 (cannot see queue size anymore.) * AM renamer moved to gather for simplicity, also fixes there. * fix: sr3 declare would fail when no admin.conf configured. * misc. fixes with AMQP message acknowledgement. - - -- peter Wed, 06 Mar 2024 09:42:57 -0500 - -metpx-sr3 (3.00.52rc2) unstable; urgency=medium - * fix #934 found references to v2 documentation. Modernized. * fix #942, #943 processing of rename and retrievePath headers. * fix #940 display crash in sr3 overview * fix #920 add _isRetry for duplicate suppression support in new retry logic. * more #918 bug fixes for AM renaming now matches Sundew perfectly. - - -- peter Wed, 14 Feb 2024 21:15:59 -0500 - -metpx-sr3 (3.00.52rc1) unstable; urgency=medium - * nodupe_fileAgeX (X=Min or Max) name change to fileAgeX replacing inflight. * fix #907 declare exchange in admin.conf wasn't working. * fix #912 enable v2 style retry processing with new --retry_refilter flag. diff --git a/sarracenia/_version.py b/sarracenia/_version.py index c48cc8fce..d5f3ca067 100755 --- a/sarracenia/_version.py +++ b/sarracenia/_version.py @@ -1 +1 @@ -__version__ = "3.00.52rc4" +__version__ = "3.00.52" From 5ec2563dc87a10bf01e41214e426e5a5f963530f Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Mon, 26 Feb 2024 15:45:16 +0000 Subject: [PATCH 4/8] modifying for unsupported old python 3.6 --- debian/control | 2 +- pyproject.toml | 206 ------------------------------------------------- 2 files changed, 1 insertion(+), 207 deletions(-) delete mode 100644 pyproject.toml diff --git a/debian/control b/debian/control index a4d6926f0..e29198d3a 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: metpx-sr3 Section: python Priority: optional Maintainer: Shared Services Canada Supercomputing -Build-Depends: debhelper (>= 9), dh-python, python3, python3-appdirs, python3-all, python3-setuptools, python3-docutils, python3-jsonpickle, python3-humanfriendly, python3-humanize, python3-paramiko, python3-psutil, python3-watchdog, pybuild-plugin-pyproject, python3-hatchling +Build-Depends: debhelper (>= 9), dh-python, python3, python3-appdirs, python3-all, python3-setuptools, python3-docutils, python3-jsonpickle, python3-humanfriendly, python3-humanize, python3-paramiko, python3-psutil, python3-watchdog Standards-Version: 3.9.5 X-Python3-Version: >= 3.6 Vcs-Git: https://github.com/MetPX/sarracenia diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 358ffc62d..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,206 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "metpx-sr3" -dynamic = ["version"] -description = 'Subscribe, Acquire, and Re-Advertise (managed multiple hop file transfers)' -readme = "README.rst" -requires-python = ">=3.7" -license = "GPL-2.0-only" -keywords = ["file transfer"] -authors = [ - { name = "Peter Silva", email = "peter.silva@ssc-spc.gc.ca" }, - { name = "Reid Sunderland", email = "" }, - { name = "André Leblanc", email = "" }, - { name = "Michel Grenier", email ="" }, - { name = "Greg Linton", email ="" }, - { name = "Cléa Aumont", email ="" }, - { name = "Cody Au", email ="" }, - { name = "Benoit Lapointe", email ="" }, -] - -classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] - -dependencies =[ - "appdirs", "humanfriendly", "humanize", "jsonpickle", "paramiko", - "psutil >= 5.3.0", "watchdog", -] - -# NOTE: -# on Redhat, when building an rpm, need to replace the -# python-magic (last two lines of dependencies above ) -# by either nothing (older releases) or *file-magic* (fc39 -# 'file-magic' - -[project.scripts] -sr3 = "sarracenia.sr:main" -sr3_post = "sarracenia.sr_post:main" -sr3_rotateLogsManually = "sarracenia.sr_rotateLogsManually:main" -sr3_tailf = "sarracenia.sr_tailf:main" - -# future entry points: -#sr3_poll="sarracenia.sr_flow:main" -#sr3_report="sarracenia.sr_flow:main" -#sr3_watch="sarracenia.sr_flow:main" -#sr3_winnow="sarracenia.sr_flow:main" -#sr3_sarra="sarracenia.sr_flow:main" -#sr3_shovel="sarracenia.sr_flow:main" -#sr3_sender="sarracenia.sr_flow:main" -#sr3_subscribe="sarracenia.sr_flow:main" - -[project.optional-dependencies] - -amqp = [ "amqp" ] -ftppoll = [ 'dateparser' ] -mqtt = [ 'paho.mqtt>=1.5.1' ] -vip = [ 'netifaces' ] -redis = [ 'redis' ] -filetypes-redhat = [ 'file-magic' ] -filetypes = [ 'python-magic' ] - - -[project.urls] -Documentation = "https://metpx.github.io/sarracenia" -Issues = "https://github.com/MetPX/sarracenia/issues" -Source = "https://github.com/MetPX/sarracenia.git" - -[tool.hatch.version] -path = "sarracenia/_version.py" - -[tool.hatch.build] -include = [ "sarracenia" ] -exclude = [ "tests", "debian" ] - -[tool.hatch.envs.default] -dependencies = [ - "coverage[toml]>=6.5", - "pytest", -] -[tool.hatch.envs.default.scripts] -test = "pytest {args:tests}" -test-cov = "coverage run -m pytest {args:tests}" -cov-report = [ - "- coverage combine", - "coverage report", -] -cov = [ - "test-cov", - "cov-report", -] - -[[tool.hatch.envs.all.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11"] - -[tool.hatch.envs.lint] -detached = true -dependencies = [ - "black>=23.1.0", - "mypy>=1.0.0", - "ruff>=0.0.243", -] -[tool.hatch.envs.lint.scripts] -typing = "mypy --install-types --non-interactive {args: tests}" -style = [ - "ruff {args:.}", - "black --check --diff {args:.}", -] -fmt = [ - "black {args:.}", - "ruff --fix {args:.}", - "style", -] -all = [ - "style", - "typing", -] - -[tool.black] -target-version = ["py37"] -line-length = 120 -skip-string-normalization = true - -[tool.ruff] -target-version = "py37" -line-length = 120 -select = [ - "A", - "ARG", - "B", - "C", - "DTZ", - "E", - "EM", - "F", - "FBT", - "I", - "ICN", - "ISC", - "N", - "PLC", - "PLE", - "PLR", - "PLW", - "Q", - "RUF", - "S", - "T", - "TID", - "UP", - "W", - "YTT", -] -ignore = [ - # Allow non-abstract empty methods in abstract base classes - "B027", - # Allow boolean positional values in function calls, like `dict.get(... True)` - "FBT003", - # Ignore checks for possible passwords - "S105", "S106", "S107", - # Ignore complexity - "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", -] -unfixable = [ - # Don't touch unused imports - "F401", -] - -[tool.ruff.isort] -known-first-party = ["temporary"] - -[tool.ruff.flake8-tidy-imports] -ban-relative-imports = "all" - -[tool.ruff.per-file-ignores] -# Tests can use magic values, assertions, and relative imports -"tests/**/*" = ["PLR2004", "S101", "TID252"] - -[tool.coverage.run] -source_pkgs = ["temporary", "tests"] -branch = true -parallel = true -omit = [ - "src/temporary/__about__.py", -] - -[tool.coverage.paths] -temporary = ["src/temporary", "*/temporary/src/temporary"] -tests = ["tests", "*/temporary/tests"] - -[tool.coverage.report] -exclude_lines = [ - "no cov", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", -] From 1e311611adc10417f3c99355cdc16faf6c7f38b1 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Tue, 18 Jun 2024 15:45:18 +0000 Subject: [PATCH 5/8] version mangling --- sarracenia/_version.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sarracenia/_version.py b/sarracenia/_version.py index 2654e0925..3249b2081 100755 --- a/sarracenia/_version.py +++ b/sarracenia/_version.py @@ -1,5 +1 @@ -<<<<<<< HEAD -__version__ = "3.00.53" -======= __version__ = "3.00.54rc1" ->>>>>>> development From d620a2fb9faae2202216267ac90151fe62ef8a0d Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Wed, 17 Jul 2024 18:41:04 +0000 Subject: [PATCH 6/8] weirdness in merge for py36 --- sarracenia/config.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sarracenia/config.py b/sarracenia/config.py index e1e663bf5..815209561 100755 --- a/sarracenia/config.py +++ b/sarracenia/config.py @@ -1806,12 +1806,8 @@ def _resolveQueueName(self,component,cfg): #if the queuefile is corrupt, then will need to guess anyways. if ( self.queueName is None ) or ( self.queueName == '' ): - queueName = 'q_' + self.broker.url.username + '_' + component + '.' + cfg - if hasattr(self, 'queue_suffix'): - queueName += '.' + self.queue_suffix - queueName += '.' + str(randint(0, 100000000)).zfill(8) - queueName += '.' + str(randint(0, 100000000)).zfill(8) - self.queueName = queueName + queueShare = self._varsub(self.queueShare) + self.queueName = f"q_{self.broker.url.username}." + '.'.join([component,cfg,queueShare]) logger.debug( f'default guessed queueName {self.queueName} ' ) if self.action not in [ 'start', 'foreground', 'declare' ]: From bee2f21763d035bc70d4f4dff1bb06a4a95d04f5 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Wed, 11 Sep 2024 23:38:22 +0000 Subject: [PATCH 7/8] mark for release --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index cd0018e28..37d0a33df 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -metpx-sr3 (3.00.55rc1) UNRELEASED; urgency=medium +metpx-sr3 (3.00.55rc1) unstable; urgency=medium * fix #953 sr3 status queue size updates again. * fix #1157 transfer/stat() call improves behaviour with overly dynamic data. @@ -19,7 +19,7 @@ metpx-sr3 (3.00.55rc1) UNRELEASED; urgency=medium * fix #1218 permissions of directories created by senders. - -- SSC-5CD2310S60 Tue, 08 Aug 2024 11:24:36 -0400 + -- SSC-5CD2310S60 Tue, 11 Sep 2024 11:24:36 -0400 metpx-sr3 (3.00.54) unstable; urgency=medium From f169b69dcbc754629bd0932447a001f99756475d Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Fri, 24 Jan 2025 13:52:28 -0500 Subject: [PATCH 8/8] correct path for file accellerator --- sarracenia/transfer/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sarracenia/transfer/file.py b/sarracenia/transfer/file.py index 245efea07..1dcc4498a 100755 --- a/sarracenia/transfer/file.py +++ b/sarracenia/transfer/file.py @@ -60,7 +60,7 @@ class File(Transfer): def __init__(self, proto, options): super().__init__(proto, options) - self.o.add_option("accelCpCommand", "str", "/usr/bin/cp %s %d") + self.o.add_option("accelCpCommand", "str", "/bin/cp %s %d") logger.debug("sr_file __init__") self.cwd = None