From b5de043ed2c357967c4ca52ca978a2f04d7e113a Mon Sep 17 00:00:00 2001 From: Ramakrishna Date: Thu, 2 Feb 2023 16:25:40 +0530 Subject: [PATCH 001/155] Upload the journals as individual packages --- portality/models/preservation.py | 4 ++ portality/tasks/preservation.py | 62 +++++++++++++------ .../templates/publisher/preservation.html | 4 ++ 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/portality/models/preservation.py b/portality/models/preservation.py index c8042e1465..60c19e6e04 100644 --- a/portality/models/preservation.py +++ b/portality/models/preservation.py @@ -98,6 +98,10 @@ def no_files_articles(self, articles_list): if articles_list is not None and len(articles_list) > 0: self.data["articles_info"]["no_files_articles"] = ", ".join(articles_list) + def uploaded_journals(self, uploaded_journals): + if uploaded_journals is not None and len(uploaded_journals) > 0: + self.data["articles_info"]["uploaded_journals"] = ", ".join(uploaded_journals) + @classmethod def by_owner(cls, owner, size=10): q = OwnerFileQuery(owner) diff --git a/portality/tasks/preservation.py b/portality/tasks/preservation.py index 727adc5ac7..00f9c644a6 100644 --- a/portality/tasks/preservation.py +++ b/portality/tasks/preservation.py @@ -126,11 +126,15 @@ def __init__(self): self.__unbagged_articles = [] self.__not_found_articles = [] self.__no_files_articles = [] + self.__uploaded_journals = [] self.has_errors = False def add_successful_article(self, article: ArticlePackage): self.__successful_articles.append(os.path.basename(article.article_dir)) + def add_uploaded_journal(self, journal_package): + self.__uploaded_journals.append(journal_package) + def add_unowned_articles(self, article: ArticlePackage): self.has_errors = True self.__unowned_articles.append(os.path.basename(article.article_dir)) @@ -168,6 +172,9 @@ def not_found_articles(self): def no_files_articles(self): return self.__no_files_articles + def uploaded_journals(self): + return self.__uploaded_journals + def get_count(self): return len(self.__successful_articles) + \ len(self.__unowned_articles) + \ @@ -243,24 +250,39 @@ def run(self): job.add_audit_message("Create Package structure") articles_list = preserv.create_package_structure() - self.save_articles_list(articles_list, preserve_model) + app.logger.debug("Created package structure") if len(articles_list.successful_articles()) > 0: - package = PreservationPackage(preserv.preservation_dir, job.user) - job.add_audit_message("Create preservation package") - tar_file = package.create_package() - app.logger.debug(f"Created tar file {tar_file}") - - job.add_audit_message("Create shasum") - sha256 = package.sha256() - - job.add_audit_message("Upload package") - response = package.upload_package(sha256) - app.logger.debug(f"Uploaded. Response{response.text}") + # Each subdirectory is a jornal and the directory name is ISSN of the journal + # iterate through the directories and upload each journal as an individual package + dirs = os.listdir(preserv.preservation_dir) + upload_failed = False + for sub_dir in dirs: + tar_file = os.path.join(preserv.preservation_dir, sub_dir) + package = PreservationPackage(tar_file, job.user) + job.add_audit_message("Create preservation package") + tar_file = package.create_package() + app.logger.debug(f"Created tar file {tar_file}") + + job.add_audit_message("Create shasum") + sha256 = package.sha256() + + job.add_audit_message("Upload package") + response = package.upload_package(sha256) + app.logger.debug(f"Uploaded. Response{response.text}") + + job.add_audit_message("Validate response") + self.validate_response(response, tar_file, sha256, preserve_model) + + if preserve_model.status == 'failed': + upload_failed = True + break + else: + articles_list.add_uploaded_journal(package.tar_file_name) - job.add_audit_message("Validate response") - self.validate_response(response, tar_file, sha256, preserve_model) + if not upload_failed: + preserve_model.uploaded_to_ia() # Check if the only few articles are successful if articles_list.is_partial_success(): @@ -278,6 +300,8 @@ def run(self): preserve_model.failed(FailedReasons.no_valid_article_available) preserve_model.save() + self.save_articles_list(articles_list, preserve_model) + except (PreservationException, Exception) as exp: # ~~-> PreservationException:Exception~~ preserve_model.failed(str(exp)) @@ -305,6 +329,8 @@ def save_articles_list(self, articles_list: ArticlesList, model: PreservationSta model.unbagged_articles(articles_list.unbagged_articles()) if len(articles_list.no_files_articles()) > 0: model.no_files_articles(articles_list.no_files_articles()) + if len(articles_list.uploaded_journals()) > 0: + model.uploaded_journals(articles_list.uploaded_journals()) model.save() def cleanup(self): @@ -345,8 +371,7 @@ def validate_response(self, response, tar_file, sha256, model): if res_filename and res_filename == tar_file: if res_shasum and res_shasum == sha256: - app.logger.info("successfully uploaded") - model.uploaded_to_ia() + app.logger.info("successfully uploaded " + tar_file) else: model.failed(FailedReasons.checksum_doesnot_match) else: @@ -379,7 +404,7 @@ def validate_response(self, response, tar_file, sha256, model): model.save() else: - app.logger.error(f"Upload failed {response.text}") + app.logger.error(f"Upload failed for {tar_file}. Reason - {response.text}") model.failed(response.text) model.save() @@ -680,7 +705,8 @@ class PreservationPackage: def __init__(self, directory, owner): self.package_dir = directory - self.tar_file = self.package_dir + ".tar.gz" + created_time = dates.format(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S") + self.tar_file = directory + "_" + created_time + ".tar.gz" self.tar_file_name = os.path.basename(self.tar_file) self.__owner = owner diff --git a/portality/templates/publisher/preservation.html b/portality/templates/publisher/preservation.html index b99c8e6c91..081f716f2a 100644 --- a/portality/templates/publisher/preservation.html +++ b/portality/templates/publisher/preservation.html @@ -144,6 +144,10 @@

History of uploads (showing last {{previous|length}})

{% endif %} {% if file.status == "uploaded" or file.status == "partial" and file.articles_info %} {% endblock %} diff --git a/portality/templates/account/login.html b/portality/templates/account/login.html index 247149641a..726831e9e3 100644 --- a/portality/templates/account/login.html +++ b/portality/templates/account/login.html @@ -3,7 +3,7 @@ {% block page_title %}Login to your account{% endblock %} {% block content %} -
+
@@ -17,5 +17,5 @@

Login

-
+ {% endblock %} diff --git a/portality/templates/account/login_to_apply.html b/portality/templates/account/login_to_apply.html index c701e9e024..33ac5666fd 100644 --- a/portality/templates/account/login_to_apply.html +++ b/portality/templates/account/login_to_apply.html @@ -3,7 +3,7 @@ {% block page_title %}Login to apply{% endblock %} {% block content %} -
+
@@ -46,5 +46,5 @@

Related help

-
+ {% endblock %} diff --git a/portality/templates/account/register.html b/portality/templates/account/register.html index 4f91d8250a..a497bd396a 100644 --- a/portality/templates/account/register.html +++ b/portality/templates/account/register.html @@ -12,7 +12,7 @@ {% endblock %} {% block content %} -
+
@@ -30,7 +30,7 @@

Register

-
+ {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/account/reset.html b/portality/templates/account/reset.html index 2b459de104..fdacc27620 100644 --- a/portality/templates/account/reset.html +++ b/portality/templates/account/reset.html @@ -4,7 +4,7 @@ {% block content %} -
+
@@ -20,6 +20,6 @@

Hi {{ account.name or account.email }}

-
+ {% endblock %} diff --git a/portality/templates/api/current/api_docs.html b/portality/templates/api/current/api_docs.html index 9c5a2bc8e5..42d3d14588 100644 --- a/portality/templates/api/current/api_docs.html +++ b/portality/templates/api/current/api_docs.html @@ -14,7 +14,7 @@ {% endblock %} {% block content %} -
+
{# todo: this nav was bumping into swagger @@ -58,7 +58,7 @@

API

-
+ {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/application_form/public_application.html b/portality/templates/application_form/public_application.html index af0e05d63e..31439b94d3 100644 --- a/portality/templates/application_form/public_application.html +++ b/portality/templates/application_form/public_application.html @@ -23,7 +23,7 @@ {% block content scoped %} -
+
{% include "application_form/_backend_validation.html" %}
@@ -64,7 +64,7 @@
-
+ {% endblock %} diff --git a/portality/templates/application_form/readonly_journal.html b/portality/templates/application_form/readonly_journal.html index 2fed49db61..fe429eea08 100644 --- a/portality/templates/application_form/readonly_journal.html +++ b/portality/templates/application_form/readonly_journal.html @@ -20,7 +20,7 @@ {% block content scoped %} -
+
@@ -47,7 +47,7 @@
-
+ {% endblock %} diff --git a/portality/templates/doaj/article.html b/portality/templates/doaj/article.html index 67720a8585..91f3f80200 100644 --- a/portality/templates/doaj/article.html +++ b/portality/templates/doaj/article.html @@ -67,7 +67,7 @@ {% set doi = bibjson.get_one_identifier("doi") %} {% set normalised_doi = article.get_normalised_doi() %} -
+

@@ -224,5 +224,5 @@

Published in {{jtitle}}

-
+ {% endblock %} diff --git a/portality/templates/doaj/articles_search.html b/portality/templates/doaj/articles_search.html index cea8977897..4a518fb3ae 100644 --- a/portality/templates/doaj/articles_search.html +++ b/portality/templates/doaj/articles_search.html @@ -10,10 +10,10 @@ {%- block meta_twitter_description -%}Find open access articles in DOAJ.{%- endblock -%} {% block content %} -
+
{% include "includes/search-help-modal.html" %} -
+ {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/doaj/contact.html b/portality/templates/doaj/contact.html index 2a4739dec8..2ffb86985f 100644 --- a/portality/templates/doaj/contact.html +++ b/portality/templates/doaj/contact.html @@ -1,7 +1,7 @@ {% extends "layouts/public_base.html" %} {% block content %} -
+

Submit your feedback and questions here. Feedback submitted about a particular journal is treated as confidential.

@@ -52,7 +52,7 @@
-
+ {% endblock %} diff --git a/portality/templates/doaj/index.html b/portality/templates/doaj/index.html index c1599f238a..43b11960c9 100644 --- a/portality/templates/doaj/index.html +++ b/portality/templates/doaj/index.html @@ -74,7 +74,7 @@

DOAJ in numbers

{% endblock %} {% block content %} -
+
@@ -242,6 +242,6 @@

Recently-added journals

-
+ {% endblock %} diff --git a/portality/templates/doaj/journals_search.html b/portality/templates/doaj/journals_search.html index e38bedd18e..8eef0d2c63 100644 --- a/portality/templates/doaj/journals_search.html +++ b/portality/templates/doaj/journals_search.html @@ -10,10 +10,10 @@ {%- block meta_twitter_description -%}Find open access journals in DOAJ.{%- endblock -%} {% block content %} -
+
{% include "includes/search-help-modal.html" %} -
+ {% endblock %} {% block extra_js_bottom %} diff --git a/portality/templates/doaj/toc.html b/portality/templates/doaj/toc.html index acf0077a7e..9d5302bc64 100644 --- a/portality/templates/doaj/toc.html +++ b/portality/templates/doaj/toc.html @@ -42,7 +42,7 @@ } %} -
+
{% if journal.last_manually_updated_since(days=30) %} @@ -450,7 +450,7 @@

Journal metadata

-
+ {% include "includes/_hotjar.html" %} {% endblock %} diff --git a/portality/templates/editor/editor_base.html b/portality/templates/editor/editor_base.html index 3d488e6e72..19a36ca7c7 100644 --- a/portality/templates/editor/editor_base.html +++ b/portality/templates/editor/editor_base.html @@ -7,7 +7,7 @@ {% block page_title %}Editor dashboard{% endblock %} {% block content %} -
+

Editor dashboard

{% include 'editor/nav.html' %} @@ -15,7 +15,7 @@

Editor dashboard

{% block editor_content %} {% endblock %} -
+ {% include "includes/_hotjar.html" %} {% endblock %} diff --git a/portality/templates/layouts/public_base.html b/portality/templates/layouts/public_base.html index 2e4cad9ddc..ef0a3ae4f0 100644 --- a/portality/templates/layouts/public_base.html +++ b/portality/templates/layouts/public_base.html @@ -2,6 +2,7 @@ {% block base_content %} +
Skip to main content