From 9575103b36357385576a7275df0fcc516babd932 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:33:07 +0000 Subject: [PATCH 01/21] Add is_deprecated flag to Tool model --- .../api/migrations/0048_tool_is_deprecated.py | 19 +++++++++++++++ controlpanel/api/models/tool.py | 2 ++ controlpanel/frontend/consumers.py | 2 +- controlpanel/frontend/forms.py | 1 + .../frontend/jinja2/release-detail.html | 23 +++++++++++++++++++ controlpanel/frontend/views/tool.py | 8 +++++-- 6 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 controlpanel/api/migrations/0048_tool_is_deprecated.py diff --git a/controlpanel/api/migrations/0048_tool_is_deprecated.py b/controlpanel/api/migrations/0048_tool_is_deprecated.py new file mode 100644 index 000000000..05eb68307 --- /dev/null +++ b/controlpanel/api/migrations/0048_tool_is_deprecated.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.2 on 2024-11-29 16:25 + +# Third-party +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0047_app_cloud_platform_role_arn"), + ] + + operations = [ + migrations.AddField( + model_name="tool", + name="is_deprecated", + field=models.BooleanField(default=False), + ), + ] diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index d090967ea..b4a90815a 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -51,6 +51,8 @@ class Tool(TimeStampedModel): default=None, ) + is_deprecated = models.BooleanField(default=False) + class Meta(TimeStampedModel.Meta): db_table = "control_panel_api_tool" ordering = ("name",) diff --git a/controlpanel/frontend/consumers.py b/controlpanel/frontend/consumers.py index f34c7f256..25c4516ab 100644 --- a/controlpanel/frontend/consumers.py +++ b/controlpanel/frontend/consumers.py @@ -196,7 +196,7 @@ def tool_restart(self, message): log.debug(f"Restarted {tool.name} for {user}") def get_tool_and_user(self, message): - tool = Tool.objects.get(pk=message["tool_id"]) + tool = Tool.objects.get(is_deprecated=False, pk=message["tool_id"]) if not tool: raise Exception(f"no Tool record found for query {message['tool_id']}") user = User.objects.get(auth0_id=message["user_id"]) diff --git a/controlpanel/frontend/forms.py b/controlpanel/frontend/forms.py index 65d2e93b2..5592bd5d4 100644 --- a/controlpanel/frontend/forms.py +++ b/controlpanel/frontend/forms.py @@ -544,6 +544,7 @@ class Meta: "is_restricted", "tool_domain", "description", + "is_deprecated", ] diff --git a/controlpanel/frontend/jinja2/release-detail.html b/controlpanel/frontend/jinja2/release-detail.html index 6bd054634..9da8f52af 100644 --- a/controlpanel/frontend/jinja2/release-detail.html +++ b/controlpanel/frontend/jinja2/release-detail.html @@ -143,6 +143,29 @@

{{ page_title }}

"value": target_users if target_users else "", "errorMessage": { "html": form.target_users_list.errors|join(". ") } if form.target_users_list.errors else {} }) }} + + {{ govukCheckboxes({ + "fieldset": { + "legend": { + "text": "Deprecate release", + "classes": "govuk-fieldset__legend--m", + }, + }, + "classes": "govuk-!-width-two-thirds", + "hint": { + "text": 'A flag to indicate the release is deprecated and will no longer be available to deploy' + }, + "name": "is_deprecated", + "items": [ + { + "value": "True", + "text": "Release is deprecated", + "checked": form.is_deprecated.value(), + }, + ], + "errorMessage": { "html": form.is_deprecated.errors|join(". ") } if form.is_deprecated.errors else {} + }) }} + {% endcall %} diff --git a/controlpanel/frontend/views/tool.py b/controlpanel/frontend/views/tool.py index 18f1dd85e..ac163c59b 100644 --- a/controlpanel/frontend/views/tool.py +++ b/controlpanel/frontend/views/tool.py @@ -45,7 +45,9 @@ def get_queryset(self): * The current user is in the beta tester group for the tool. """ - return Tool.objects.filter(Q(is_restricted=False) | Q(target_users=self.request.user.id)) + return Tool.objects.filter( + Q(is_restricted=False) | Q(target_users=self.request.user.id) + ).exclude(is_deprecated=True) def _locate_tool_box_by_chart_name(self, chart_name): tool_box = None @@ -67,7 +69,9 @@ def _find_related_tool_record(self, chart_name, chart_version, image_tag): memory, CPU etc, then the linkage will be confused although it won't affect people usage. """ - tool_set = Tool.objects.filter(chart_name=chart_name, version=chart_version) + tool_set = Tool.objects.filter( + chart_name=chart_name, version=chart_version, is_restricted=False + ) for item in tool_set: if item.image_tag == image_tag: return item From eb49afb538a9b0b0ade44c5dd1adc9255e5950a9 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:33:12 +0000 Subject: [PATCH 02/21] Add field to mark release retired and add deprecation message --- ...tool_deprecated_message_tool_is_retired.py | 24 ++++++++++++ controlpanel/api/models/tool.py | 2 + controlpanel/frontend/consumers.py | 2 +- controlpanel/frontend/forms.py | 2 + .../frontend/jinja2/release-detail.html | 37 ++++++++++++++++++- controlpanel/frontend/views/tool.py | 2 +- pyproject.toml | 3 +- 7 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py diff --git a/controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py b/controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py new file mode 100644 index 000000000..cd1920101 --- /dev/null +++ b/controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.2 on 2024-12-03 17:12 + +# Third-party +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0048_tool_is_deprecated"), + ] + + operations = [ + migrations.AddField( + model_name="tool", + name="deprecated_message", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="tool", + name="is_retired", + field=models.BooleanField(default=False), + ), + ] diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index b4a90815a..1303fbc2a 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -52,6 +52,8 @@ class Tool(TimeStampedModel): ) is_deprecated = models.BooleanField(default=False) + deprecated_message = models.TextField(blank=True) + is_retired = models.BooleanField(default=False) class Meta(TimeStampedModel.Meta): db_table = "control_panel_api_tool" diff --git a/controlpanel/frontend/consumers.py b/controlpanel/frontend/consumers.py index 25c4516ab..b51fdbdb8 100644 --- a/controlpanel/frontend/consumers.py +++ b/controlpanel/frontend/consumers.py @@ -196,7 +196,7 @@ def tool_restart(self, message): log.debug(f"Restarted {tool.name} for {user}") def get_tool_and_user(self, message): - tool = Tool.objects.get(is_deprecated=False, pk=message["tool_id"]) + tool = Tool.objects.get(is_retired=False, pk=message["tool_id"]) if not tool: raise Exception(f"no Tool record found for query {message['tool_id']}") user = User.objects.get(auth0_id=message["user_id"]) diff --git a/controlpanel/frontend/forms.py b/controlpanel/frontend/forms.py index 5592bd5d4..5e59affcc 100644 --- a/controlpanel/frontend/forms.py +++ b/controlpanel/frontend/forms.py @@ -545,6 +545,8 @@ class Meta: "tool_domain", "description", "is_deprecated", + "deprecated_message", + "is_retired", ] diff --git a/controlpanel/frontend/jinja2/release-detail.html b/controlpanel/frontend/jinja2/release-detail.html index 9da8f52af..0cd63c6c0 100644 --- a/controlpanel/frontend/jinja2/release-detail.html +++ b/controlpanel/frontend/jinja2/release-detail.html @@ -153,7 +153,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'A flag to indicate the release is deprecated and will no longer be available to deploy' + "text": 'A flag to indicate the release is deprecated and will soon become unavailable to deploy' }, "name": "is_deprecated", "items": [ @@ -165,6 +165,41 @@

{{ page_title }}

], "errorMessage": { "html": form.is_deprecated.errors|join(". ") } if form.is_deprecated.errors else {} }) }} + {{ govukInput({ + "label": { + "text": "Deprecation message", + "classes": "govuk-label--s", + }, + "classes": "govuk-!-width-two-thirds", + "hint": { + "text": 'A message to display to users if they select this tool' + }, + "name": "deprecated_message", + "value": form.deprecated_message.value(), + "errorMessage": { "html": form.deprecated_message.errors|join(". ") } if form.deprecated_message.errors else {} + }) }} + + {{ govukCheckboxes({ + "fieldset": { + "legend": { + "text": "Retire release", + "classes": "govuk-fieldset__legend--m", + }, + }, + "classes": "govuk-!-width-two-thirds", + "hint": { + "text": 'A flag to indicate the release is retired. Users will not be able to deploy this release.' + }, + "name": "is_retired", + "items": [ + { + "value": "True", + "text": "Release is retired", + "checked": form.is_retired.value(), + }, + ], + "errorMessage": { "html": form.is_retired.errors|join(". ") } if form.is_retired.errors else {} + }) }} diff --git a/controlpanel/frontend/views/tool.py b/controlpanel/frontend/views/tool.py index ac163c59b..3cd3f118f 100644 --- a/controlpanel/frontend/views/tool.py +++ b/controlpanel/frontend/views/tool.py @@ -47,7 +47,7 @@ def get_queryset(self): """ return Tool.objects.filter( Q(is_restricted=False) | Q(target_users=self.request.user.id) - ).exclude(is_deprecated=True) + ).exclude(is_retired=True) def _locate_tool_box_by_chart_name(self, chart_name): tool_box = None diff --git a/pyproject.toml b/pyproject.toml index 42ef8e96f..13b68951f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,5 +20,6 @@ import_heading_thirdparty = 'Third-party' line_length = 100 multi_line_output = 3 no_lines_before = 'LOCALFOLDER' -skip = "migrations,venv" +skip_glob = "migrations,venv" +filter_files = true virtual_env = "venv" From eccd8b87c25eb6c5346b34fa8418889099130677 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:44:37 +0000 Subject: [PATCH 03/21] Disable open tool button --- controlpanel/frontend/jinja2/tool-list.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index 21cf95eb2..8648aecf1 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -61,7 +61,7 @@

{{ tool_info.name }}

Status: - {% if deployment %} + {% if deployment and deployment.tool_id != -1 %} {{ deployment.status | default("") }} {% else %} Not deployed @@ -86,7 +86,7 @@

{{ tool_info.name }}

onclick="window.open('{{ tool_info['url'] }}', '_blank');" rel="noopener" target="_blank" - {% if not deployment %} disabled {% endif %}> + {% if not deployment or deployment.tool_id == -1 %} disabled {% endif %}> Open @@ -108,11 +108,11 @@

{{ tool_info.name }}

{% if deployment and deployment.tool_id == -1 %} +

- Your current deployment ({{ deployment.chart_name}}-{{ deployment.chart_version }}: {{ deployment.image_tag }}) - is not recognised as a current maintained tool release. You can still use it, - but it is recommended to switch to a new version from the dropdown list. + Your previous deployment ({{ deployment.chart_name}}-{{ deployment.chart_version }}: {{ deployment.image_tag }}) + is no longer maintained. You will need to deploy a new version from the dropdown list.

{% endif %} From 79923c6ee4fba2b0144b429d9dd62281d18fcd04 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:08:10 +0000 Subject: [PATCH 04/21] Handle retired and deprecated tools in the frontend - If a tool is depcrecated, show the deprecation message when it is selected - If a tool is retired, hide it and show a message that user must upgrade --- .../0050_alter_tool_deprecated_message.py | 21 +++++++++ controlpanel/api/models/tool.py | 15 +++++- .../frontend/jinja2/release-detail.html | 6 +-- controlpanel/frontend/jinja2/tool-list.html | 16 +++++-- .../static/javascripts/modules/tool-status.js | 16 +++++++ controlpanel/frontend/views/tool.py | 46 ++++++++++++------- 6 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 controlpanel/api/migrations/0050_alter_tool_deprecated_message.py diff --git a/controlpanel/api/migrations/0050_alter_tool_deprecated_message.py b/controlpanel/api/migrations/0050_alter_tool_deprecated_message.py new file mode 100644 index 000000000..8a7a2409a --- /dev/null +++ b/controlpanel/api/migrations/0050_alter_tool_deprecated_message.py @@ -0,0 +1,21 @@ +# Generated by Django 5.1.2 on 2024-12-09 15:07 + +# Third-party +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0049_tool_deprecated_message_tool_is_retired"), + ] + + operations = [ + migrations.AlterField( + model_name="tool", + name="deprecated_message", + field=models.TextField( + blank=True, help_text="If no message is provided, a default message will be used." + ), + ), + ] diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index 1303fbc2a..36560403e 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -27,6 +27,7 @@ class Tool(TimeStampedModel): "rstudio": "rstudio", "vscode": "vscode", } + DEFAULT_DEPRECATED_MESSAGE = "The selected release has been deprecated and will be retired soon. Please update to a more recent version." # noqa description = models.TextField(blank=True) chart_name = models.CharField(max_length=100, blank=False) @@ -52,7 +53,9 @@ class Tool(TimeStampedModel): ) is_deprecated = models.BooleanField(default=False) - deprecated_message = models.TextField(blank=True) + deprecated_message = models.TextField( + blank=True, help_text="If no message is provided, a default message will be used." + ) is_retired = models.BooleanField(default=False) class Meta(TimeStampedModel.Meta): @@ -83,6 +86,16 @@ def image_tag(self): "{}.image.tag".format(chart_image_key_name) ) + @property + def get_deprecated_message(self): + if not self.is_deprecated: + return "" + + if self.is_retired: + return "" + + return self.deprecated_message or self.DEFAULT_DEPRECATED_MESSAGE + class ToolDeploymentManager: """ diff --git a/controlpanel/frontend/jinja2/release-detail.html b/controlpanel/frontend/jinja2/release-detail.html index 0cd63c6c0..d74dea699 100644 --- a/controlpanel/frontend/jinja2/release-detail.html +++ b/controlpanel/frontend/jinja2/release-detail.html @@ -153,7 +153,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'A flag to indicate the release is deprecated and will soon become unavailable to deploy' + "text": 'Checking this will display a deprecation message to users when they select this release' }, "name": "is_deprecated", "items": [ @@ -172,7 +172,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'A message to display to users if they select this tool' + "text": form.deprecated_message.help_text }, "name": "deprecated_message", "value": form.deprecated_message.value(), @@ -188,7 +188,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'A flag to indicate the release is retired. Users will not be able to deploy this release.' + "text": 'Checking this will remove this release from all users dropdown options on the Your Tools page.' }, "name": "is_retired", "items": [ diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index 8648aecf1..3da29b75c 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -41,16 +41,22 @@

{{ tool_info.name }}

{% if deployment and deployment.tool_id > -1 %} {% set installed_chart_version = deployment.description %} {% set installed_release_version = deployment.tool_id %} - {% else %} {% endif %} {% for release_version, release_detail in tool_info["releases"].items(): %} {% if release_version != installed_release_version: %} - {% endif %} {% endfor %} @@ -107,6 +113,8 @@

{{ tool_info.name }}

+

{{ deployment.deprecated_message }}

+ {% if deployment and deployment.tool_id == -1 %}
diff --git a/controlpanel/frontend/static/javascripts/modules/tool-status.js b/controlpanel/frontend/static/javascripts/modules/tool-status.js index 2121c4b35..ddc61b8e9 100644 --- a/controlpanel/frontend/static/javascripts/modules/tool-status.js +++ b/controlpanel/frontend/static/javascripts/modules/tool-status.js @@ -156,5 +156,21 @@ moj.Modules.toolStatus = { // If "(not installed)" or "(installed)" version selected // the "Deploy" button needs to be disabled deployButton.disabled = notInstalledSelected || installedSelected; + + this.toggleDeprecationMessage(selected, targetTool); }, + + toggleDeprecationMessage(selected, targetTool) { + const isDeprecated = selected.attributes["data-is-deprecated"].value === "True"; + const deprecationMessageElement = document.getElementById(targetTool.value + "-deprecation-message"); + const deprecationMessage = selected.attributes["data-deprecated-message"].value; + + if (isDeprecated) { + deprecationMessageElement.firstChild.innerText = deprecationMessage; + deprecationMessageElement.classList.remove(this.hidden); + } else { + deprecationMessageElement.classList.add(this.hidden); + deprecationMessageElement.firstChild.innerText = ""; + } + } }; diff --git a/controlpanel/frontend/views/tool.py b/controlpanel/frontend/views/tool.py index 3cd3f118f..a78e416a1 100644 --- a/controlpanel/frontend/views/tool.py +++ b/controlpanel/frontend/views/tool.py @@ -71,13 +71,15 @@ def _find_related_tool_record(self, chart_name, chart_version, image_tag): """ tool_set = Tool.objects.filter( chart_name=chart_name, version=chart_version, is_restricted=False - ) - for item in tool_set: - if item.image_tag == image_tag: - return item - return tool_set.first() + ).exclude(is_retired=True) + for tool in tool_set: + if tool.image_tag == image_tag: + return tool + # If we cant find a tool with the same image tag, this must mean that it was retired or + # deleted. So return none, and let the calling function handle it + return None - def _add_new_item_to_tool_box(self, user, tool_box, tool, tools_info, charts_info): + def _add_new_item_to_tool_box(self, user, tool_box, tool, tools_info): if tool_box not in tools_info: tools_info[tool_box] = { "name": tool.name, @@ -85,16 +87,19 @@ def _add_new_item_to_tool_box(self, user, tool_box, tool, tools_info, charts_inf "deployment": None, "releases": {}, } - image_tag = tool.image_tag - if not image_tag: - image_tag = charts_info.get(tool.version, {}) or "unknown" + # TODO We should update model to always store an image tag + # image_tag = tool.image_tag + # if not image_tag: + # image_tag = charts_info.get(tool.version, {}) or "unknown" if tool.id not in tools_info[tool_box]["releases"]: tools_info[tool_box]["releases"][tool.id] = { "tool_id": tool.id, "chart_name": tool.chart_name, "description": tool.description, "chart_version": tool.version, - "image_tag": image_tag, + "image_tag": tool.image_tag, + "is_deprecated": tool.is_deprecated, + "deprecated_message": tool.get_deprecated_message, } def _get_tool_deployed_image_tag(self, containers): @@ -103,8 +108,10 @@ def _get_tool_deployed_image_tag(self, containers): return container.image.split(":")[1] return None - def _add_deployed_charts_info(self, tools_info, user, id_token, charts_info): + def _add_deployed_charts_info(self, tools_info, user, id_token): # Get list of deployed tools + # TODO this sets what tool the user currently has deployed. If we were to refactor to store + # deployed tools in the database, we could remove a lot of this logic deployments = cluster.ToolDeployment.get_deployments(user, id_token) for deployment in deployments: chart_name, chart_version = deployment.metadata.labels["chart"].rsplit("-", 1) @@ -119,7 +126,7 @@ def _add_deployed_charts_info(self, tools_info, user, id_token, charts_info): ) ) else: - self._add_new_item_to_tool_box(user, tool_box, tool, tools_info, charts_info) + self._add_new_item_to_tool_box(user, tool_box, tool, tools_info) if tool_box not in tools_info: # up to this stage, if the tool_box is still empty, it means # there is no tool release available in db @@ -131,21 +138,26 @@ def _add_deployed_charts_info(self, tools_info, user, id_token, charts_info): "image_tag": image_tag, "description": tool.description if tool else "Not available", "status": ToolDeployment(tool, user).get_status(id_token, deployment=deployment), + "is_deprecated": tool.is_deprecated if tool else False, + "deprecated_message": tool.get_deprecated_message if tool else "", + "is_retired": tool is None, } - def _retrieve_detail_tool_info(self, user, tools, charts_info): + def _retrieve_detail_tool_info(self, user, tools): + # TODO why do we need this? We could change so that all information required about available tools comes from the DB # noqa: E501 tools_info = {} for tool in tools: # Work out which bucket the chart should be in tool_box = self._locate_tool_box_by_chart_name(tool.chart_name) # No matching tool bucket for the given chart. So ignore. if tool_box: - self._add_new_item_to_tool_box(user, tool_box, tool, tools_info, charts_info) + self._add_new_item_to_tool_box(user, tool_box, tool, tools_info) return tools_info def _get_charts_info(self, tool_list): # We may need the default image_tag from helm chart # unless we configure it specifically in parameters of tool release + # TODO if we make sure that we always have an image_tag for a tool, then building charts_info is redundant and could be removed # noqa: E501 charts_info = {} chart_entries = None for tool in tool_list: @@ -229,14 +241,14 @@ def get_context_data(self, *args, **kwargs): context["managed_airflow_prod_url"] = f"{settings.AWS_SERVICE_URL}/?{args_airflow_prod_url}" # Arrange tools information - charts_info = self._get_charts_info(context["tools"]) - tools_info = self._retrieve_detail_tool_info(user, context["tools"], charts_info) + # charts_info = self._get_charts_info(context["tools"]) + tools_info = self._retrieve_detail_tool_info(user, context["tools"]) if "vscode" in tools_info: url = tools_info["vscode"]["url"] tools_info["vscode"]["url"] = f"{url}?folder=/home/analyticalplatform/workspace" - self._add_deployed_charts_info(tools_info, user, id_token, charts_info) + self._add_deployed_charts_info(tools_info, user, id_token) context["tools_info"] = tools_info return context From cda0c943521889036945553bb60c5aac303ea8bb Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:18:39 +0000 Subject: [PATCH 05/21] Display deprecation message as a warning --- controlpanel/frontend/jinja2/tool-list.html | 4 ++-- .../frontend/static/javascripts/modules/tool-status.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index 3da29b75c..c8ac20a9a 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -113,14 +113,14 @@

{{ tool_info.name }}

-

{{ deployment.deprecated_message }}

+
Warning{{ deployment.deprecated_message }}
{% if deployment and deployment.tool_id == -1 %}

Your previous deployment ({{ deployment.chart_name}}-{{ deployment.chart_version }}: {{ deployment.image_tag }}) - is no longer maintained. You will need to deploy a new version from the dropdown list. + has been retired. You will need to deploy a new version from the dropdown list.

{% endif %} diff --git a/controlpanel/frontend/static/javascripts/modules/tool-status.js b/controlpanel/frontend/static/javascripts/modules/tool-status.js index ddc61b8e9..79c1ce9ae 100644 --- a/controlpanel/frontend/static/javascripts/modules/tool-status.js +++ b/controlpanel/frontend/static/javascripts/modules/tool-status.js @@ -166,11 +166,11 @@ moj.Modules.toolStatus = { const deprecationMessage = selected.attributes["data-deprecated-message"].value; if (isDeprecated) { - deprecationMessageElement.firstChild.innerText = deprecationMessage; + deprecationMessageElement.lastChild.innerText = deprecationMessage; deprecationMessageElement.classList.remove(this.hidden); } else { deprecationMessageElement.classList.add(this.hidden); - deprecationMessageElement.firstChild.innerText = ""; + deprecationMessageElement.lastChild.innerText = ""; } } }; From d341d0e25c25b054867fbe7c2af4320d541e42ed Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:34:38 +0000 Subject: [PATCH 06/21] Show retired message as a warning --- controlpanel/frontend/jinja2/tool-list.html | 15 ++++++------ .../static/javascripts/modules/tool-status.js | 24 ++++++++++++++----- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index c8ac20a9a..80c458e4c 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -92,7 +92,7 @@

{{ tool_info.name }}

onclick="window.open('{{ tool_info['url'] }}', '_blank');" rel="noopener" target="_blank" - {% if not deployment or deployment.tool_id == -1 %} disabled {% endif %}> + {% if not deployment or deployment.is_retired %} disabled {% endif %}> Open @@ -106,7 +106,7 @@

{{ tool_info.name }}

{{ csrf_input }} @@ -115,13 +115,14 @@

{{ tool_info.name }}

Warning{{ deployment.deprecated_message }}
-{% if deployment and deployment.tool_id == -1 %} +{% if deployment and deployment.is_retired %} -
-

- Your previous deployment ({{ deployment.chart_name}}-{{ deployment.chart_version }}: {{ deployment.image_tag }}) +

+ + Warning + Your previous deployment ({{ deployment.chart_name}}-{{ deployment.chart_version }}: {{ deployment.image_tag }}) has been retired. You will need to deploy a new version from the dropdown list. -

+
{% endif %}
diff --git a/controlpanel/frontend/static/javascripts/modules/tool-status.js b/controlpanel/frontend/static/javascripts/modules/tool-status.js index 79c1ce9ae..31e58f4e8 100644 --- a/controlpanel/frontend/static/javascripts/modules/tool-status.js +++ b/controlpanel/frontend/static/javascripts/modules/tool-status.js @@ -161,16 +161,28 @@ moj.Modules.toolStatus = { }, toggleDeprecationMessage(selected, targetTool) { - const isDeprecated = selected.attributes["data-is-deprecated"].value === "True"; + const isDeprecated = selected.attributes["data-is-deprecated"]; const deprecationMessageElement = document.getElementById(targetTool.value + "-deprecation-message"); + if (isDeprecated === undefined) { + this.hideDeprecationMessage(deprecationMessageElement); + return; + } const deprecationMessage = selected.attributes["data-deprecated-message"].value; - if (isDeprecated) { - deprecationMessageElement.lastChild.innerText = deprecationMessage; - deprecationMessageElement.classList.remove(this.hidden); + if (isDeprecated.value === "True") { + this.showDeprecationMessage(deprecationMessageElement, deprecationMessage); } else { - deprecationMessageElement.classList.add(this.hidden); - deprecationMessageElement.lastChild.innerText = ""; + this.hideDeprecationMessage(deprecationMessageElement); } + }, + + showDeprecationMessage(element, message) { + element.classList.remove(this.hidden); + element.lastChild.innerText = message; + }, + + hideDeprecationMessage(element) { + element.classList.add(this.hidden); + element.lastChild.innerText = ""; } }; From 0570ae99dc60ece11630dfa717b38d2a89d0eeeb Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:42:59 +0000 Subject: [PATCH 07/21] Another change to use is_retired --- controlpanel/frontend/jinja2/tool-list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index 80c458e4c..ec034c882 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -67,7 +67,7 @@

{{ tool_info.name }}

Status: - {% if deployment and deployment.tool_id != -1 %} + {% if deployment and not deployment.is_retired %} {{ deployment.status | default("") }} {% else %} Not deployed From 9566bdc075ee7fb1ad3de53fd2fddb0e8e6c4441 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:46:36 +0000 Subject: [PATCH 08/21] Disable buttons based on selected release If a deployed release is selected, enable open and restart button. If a undeployed release is selected, enable deploy button only. --- controlpanel/frontend/jinja2/tool-list.html | 5 ++++- .../frontend/static/javascripts/modules/tool-status.js | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index ec034c882..06e6da73c 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -92,6 +92,7 @@

{{ tool_info.name }}

onclick="window.open('{{ tool_info['url'] }}', '_blank');" rel="noopener" target="_blank" + id="open-{{ chart_name }}" {% if not deployment or deployment.is_retired %} disabled {% endif %}> Open @@ -102,10 +103,12 @@

{{ tool_info.name }}

{% endif %} data-action-name="restart" method="post" - style="display: inline;"> + style="display: inline;" + > {{ csrf_input }} diff --git a/controlpanel/frontend/static/javascripts/modules/tool-status.js b/controlpanel/frontend/static/javascripts/modules/tool-status.js index 31e58f4e8..07945f782 100644 --- a/controlpanel/frontend/static/javascripts/modules/tool-status.js +++ b/controlpanel/frontend/static/javascripts/modules/tool-status.js @@ -152,10 +152,18 @@ moj.Modules.toolStatus = { const targetTool = target.attributes["data-action-target"]; const deployButton = document.getElementById("deploy-" + targetTool.value); + const openButton = document.getElementById("open-" + targetTool.value); + const restartButton = document.getElementById("restart-" + targetTool.value); + + // if (!installedSelected) { + // openButton.disabled = true; + // }; // If "(not installed)" or "(installed)" version selected // the "Deploy" button needs to be disabled deployButton.disabled = notInstalledSelected || installedSelected; + openButton.disabled = !installedSelected; + restartButton.disabled = !installedSelected; this.toggleDeprecationMessage(selected, targetTool); }, From 1d0bf5ae9ef2539ae330d6b9e7bfa26039142a1e Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 10 Dec 2024 09:20:52 +0000 Subject: [PATCH 09/21] Replace image_tag property with model field --- .../api/migrations/0051_tool_image_tag.py | 18 +++++++++++++ .../migrations/0052_add_image_tag_value.py | 26 +++++++++++++++++++ .../migrations/0053_alter_tool_image_tag.py | 19 ++++++++++++++ controlpanel/api/models/tool.py | 9 +------ pyproject.toml | 2 +- 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 controlpanel/api/migrations/0051_tool_image_tag.py create mode 100644 controlpanel/api/migrations/0052_add_image_tag_value.py create mode 100644 controlpanel/api/migrations/0053_alter_tool_image_tag.py diff --git a/controlpanel/api/migrations/0051_tool_image_tag.py b/controlpanel/api/migrations/0051_tool_image_tag.py new file mode 100644 index 000000000..937fd81c3 --- /dev/null +++ b/controlpanel/api/migrations/0051_tool_image_tag.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-12-10 09:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0050_alter_tool_deprecated_message"), + ] + + operations = [ + migrations.AddField( + model_name="tool", + name="image_tag", + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/controlpanel/api/migrations/0052_add_image_tag_value.py b/controlpanel/api/migrations/0052_add_image_tag_value.py new file mode 100644 index 000000000..a058e6f83 --- /dev/null +++ b/controlpanel/api/migrations/0052_add_image_tag_value.py @@ -0,0 +1,26 @@ +# Generated by Django 5.1.2 on 2024-12-10 09:01 + +from django.db import migrations + + +def add_image_tag(apps, schema_editor): + Tool = apps.get_model("api", "Tool") + for tool in Tool.objects.all(): + chart_image_key_name = tool.chart_name.split("-")[0] + values = tool.values or {} + image_tag = values.get("{}.tag".format(chart_image_key_name)) or values.get( + "{}.image.tag".format(chart_image_key_name) + ) + tool.image_tag = image_tag + tool.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0051_tool_image_tag"), + ] + + operations = [ + migrations.RunPython(code=add_image_tag, reverse_code=migrations.RunPython.noop), + ] diff --git a/controlpanel/api/migrations/0053_alter_tool_image_tag.py b/controlpanel/api/migrations/0053_alter_tool_image_tag.py new file mode 100644 index 000000000..a0903be6d --- /dev/null +++ b/controlpanel/api/migrations/0053_alter_tool_image_tag.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.2 on 2024-12-10 09:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0052_add_image_tag_value"), + ] + + operations = [ + migrations.AlterField( + model_name="tool", + name="image_tag", + field=models.CharField(default="", max_length=100), + preserve_default=False, + ), + ] diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index 36560403e..ae08d5366 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -57,6 +57,7 @@ class Tool(TimeStampedModel): blank=True, help_text="If no message is provided, a default message will be used." ) is_retired = models.BooleanField(default=False) + image_tag = models.CharField(max_length=100) class Meta(TimeStampedModel.Meta): db_table = "control_panel_api_tool" @@ -78,14 +79,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) return self - @property - def image_tag(self): - chart_image_key_name = self.chart_name.split("-")[0] - values = self.values or {} - return values.get("{}.tag".format(chart_image_key_name)) or values.get( - "{}.image.tag".format(chart_image_key_name) - ) - @property def get_deprecated_message(self): if not self.is_deprecated: diff --git a/pyproject.toml b/pyproject.toml index 13b68951f..e6029b63f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,6 @@ import_heading_thirdparty = 'Third-party' line_length = 100 multi_line_output = 3 no_lines_before = 'LOCALFOLDER' -skip_glob = "migrations,venv" +skip_glob = "**/migrations**,venv" filter_files = true virtual_env = "venv" From 1ec6793a219a78b827f0bfff2b2525b345e1fe9d Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:38:25 +0000 Subject: [PATCH 10/21] Rebuild migrations after rebase --- .../api/migrations/0048_tool_is_deprecated.py | 19 ----------------- .../0050_alter_tool_deprecated_message.py | 21 ------------------- ..._retired.py => 0050_tool_is_deprecated.py} | 11 +++++++--- .../api/migrations/0051_tool_image_tag.py | 2 +- 4 files changed, 9 insertions(+), 44 deletions(-) delete mode 100644 controlpanel/api/migrations/0048_tool_is_deprecated.py delete mode 100644 controlpanel/api/migrations/0050_alter_tool_deprecated_message.py rename controlpanel/api/migrations/{0049_tool_deprecated_message_tool_is_retired.py => 0050_tool_is_deprecated.py} (52%) diff --git a/controlpanel/api/migrations/0048_tool_is_deprecated.py b/controlpanel/api/migrations/0048_tool_is_deprecated.py deleted file mode 100644 index 05eb68307..000000000 --- a/controlpanel/api/migrations/0048_tool_is_deprecated.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-29 16:25 - -# Third-party -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("api", "0047_app_cloud_platform_role_arn"), - ] - - operations = [ - migrations.AddField( - model_name="tool", - name="is_deprecated", - field=models.BooleanField(default=False), - ), - ] diff --git a/controlpanel/api/migrations/0050_alter_tool_deprecated_message.py b/controlpanel/api/migrations/0050_alter_tool_deprecated_message.py deleted file mode 100644 index 8a7a2409a..000000000 --- a/controlpanel/api/migrations/0050_alter_tool_deprecated_message.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 5.1.2 on 2024-12-09 15:07 - -# Third-party -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("api", "0049_tool_deprecated_message_tool_is_retired"), - ] - - operations = [ - migrations.AlterField( - model_name="tool", - name="deprecated_message", - field=models.TextField( - blank=True, help_text="If no message is provided, a default message will be used." - ), - ), - ] diff --git a/controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py b/controlpanel/api/migrations/0050_tool_is_deprecated.py similarity index 52% rename from controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py rename to controlpanel/api/migrations/0050_tool_is_deprecated.py index cd1920101..8776573a2 100644 --- a/controlpanel/api/migrations/0049_tool_deprecated_message_tool_is_retired.py +++ b/controlpanel/api/migrations/0050_tool_is_deprecated.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-12-03 17:12 +# Generated by Django 5.1.2 on 2024-11-29 16:25 # Third-party from django.db import migrations, models @@ -7,14 +7,19 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0048_tool_is_deprecated"), + ("api", "0049_alter_feedback_suggestions"), ] operations = [ + migrations.AddField( + model_name="tool", + name="is_deprecated", + field=models.BooleanField(default=False), + ), migrations.AddField( model_name="tool", name="deprecated_message", - field=models.TextField(blank=True), + field=models.TextField(blank=True, help_text="If no message is provided, a default message will be used."), ), migrations.AddField( model_name="tool", diff --git a/controlpanel/api/migrations/0051_tool_image_tag.py b/controlpanel/api/migrations/0051_tool_image_tag.py index 937fd81c3..6be08f309 100644 --- a/controlpanel/api/migrations/0051_tool_image_tag.py +++ b/controlpanel/api/migrations/0051_tool_image_tag.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0050_alter_tool_deprecated_message"), + ("api", "0050_tool_is_deprecated"), ] operations = [ From 21a73dcdd1e7bd840f62d0ae58833a0d1a3a1ec0 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:43:50 +0000 Subject: [PATCH 11/21] Remove DEPRECATED message from options --- controlpanel/frontend/jinja2/tool-list.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index 06e6da73c..5c4ce3e0c 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -44,7 +44,7 @@

{{ tool_info.name }}

{% else %} @@ -56,7 +56,7 @@

{{ tool_info.name }}

data-is-deprecated="{{ release_detail.is_deprecated }}" data-deprecated-message="{{ release_detail.deprecated_message }}"> - {% if release_detail.is_deprecated %}DEPRECATED {% endif %}[{{ release_detail.chart_name }} {{ release_detail.image_tag }}] {{ release_detail.description or "Unknown" }} + [{{ release_detail.chart_name }} {{ release_detail.image_tag }}] {{ release_detail.description or "Unknown" }} {% endif %} {% endfor %} From 3e00b28618ea485513ffbf480e295d10189763d1 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:49:43 +0000 Subject: [PATCH 12/21] Placate super-linter --- controlpanel/api/migrations/0050_tool_is_deprecated.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/controlpanel/api/migrations/0050_tool_is_deprecated.py b/controlpanel/api/migrations/0050_tool_is_deprecated.py index 8776573a2..d87c3bf1f 100644 --- a/controlpanel/api/migrations/0050_tool_is_deprecated.py +++ b/controlpanel/api/migrations/0050_tool_is_deprecated.py @@ -19,7 +19,9 @@ class Migration(migrations.Migration): migrations.AddField( model_name="tool", name="deprecated_message", - field=models.TextField(blank=True, help_text="If no message is provided, a default message will be used."), + field=models.TextField( + blank=True, help_text="If no message is provided, a default message will be used." + ), ), migrations.AddField( model_name="tool", From 8f5a7c6d4bb9b679217b1b1133c06474e828d6f6 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:33:33 +0000 Subject: [PATCH 13/21] Add image_tag to the release detail, create pages Use the value stored in the DB when deploying the tool --- controlpanel/api/cluster.py | 9 +++++++++ controlpanel/api/models/tool.py | 11 +++++++++++ controlpanel/frontend/forms.py | 1 + .../frontend/jinja2/release-create.html | 17 +++++++++++++++++ .../frontend/jinja2/release-detail.html | 17 +++++++++++++++++ 5 files changed, 55 insertions(+) diff --git a/controlpanel/api/cluster.py b/controlpanel/api/cluster.py index 90b6ec86a..3f36b4b1f 100644 --- a/controlpanel/api/cluster.py +++ b/controlpanel/api/cluster.py @@ -970,6 +970,13 @@ def _delete_legacy_release(self): if old_release_name in helm.list_releases(old_release_name, self.k8s_namespace): helm.delete(self.k8s_namespace, old_release_name) + @property + def get_image_tag_helm_value(self): + """ + Return the image tag to be used in the helm chart values + """ + return {self.tool.image_tag_key: self.tool.image_tag} + def _set_values(self, **kwargs): """ Return the list of `--set KEY=VALUE` helm upgrade arguments @@ -991,6 +998,8 @@ def _set_values(self, **kwargs): values.update(self.tool.values) values.update(kwargs) + # override the tool image tag with the value stored in the DB + values.update(self.get_image_tag_helm_value) set_values = [] for key, val in values.items(): if val: diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index ae08d5366..620c133c4 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -89,6 +89,17 @@ def get_deprecated_message(self): return self.deprecated_message or self.DEFAULT_DEPRECATED_MESSAGE + @property + def image_tag_key(self): + mapping = { + "jupyter-lab-datascience-notebook": "jupyter.tag", + "jupyter-lab-all-spark": "jupyter.tag", + "jupyter-lab": "jupyterlab.image.tag", + "rstudio": "rstudio.image.tag", + "vscode": "vscode.image.tag", + } + return mapping["chart_name"] + class ToolDeploymentManager: """ diff --git a/controlpanel/frontend/forms.py b/controlpanel/frontend/forms.py index 5e59affcc..301c903f4 100644 --- a/controlpanel/frontend/forms.py +++ b/controlpanel/frontend/forms.py @@ -540,6 +540,7 @@ class Meta: "name", "chart_name", "version", + "image_tag", "values", "is_restricted", "tool_domain", diff --git a/controlpanel/frontend/jinja2/release-create.html b/controlpanel/frontend/jinja2/release-create.html index b63ed43e3..5b3679ea0 100644 --- a/controlpanel/frontend/jinja2/release-create.html +++ b/controlpanel/frontend/jinja2/release-create.html @@ -88,6 +88,23 @@

{{ page_title }}

"value": form.version.value(), "errorMessage": { "html": form.version.errors|join(". ") } if form.version.errors else {} }) }} + {{ govukInput({ + "label": { + "text": "Image Tag", + "classes": "govuk-label--m", + }, + "classes": "govuk-!-width-two-thirds", + "hint": { + "text": 'Docker image tag for this release. It will be passed to helm as a value when the tool is deployed.' + }, + "name": "image_tag", + "attributes": { + "pattern": "[a-z0-9.-]{1,60}", + "maxlength": "60", + }, + "value": form.image_tag.value(), + "errorMessage": { "html": form.image_tag.errors|join(". ") } if form.image_tag.errors else {} + }) }} {{ govukInput({ "label": { "text": "Custom Domain Name", diff --git a/controlpanel/frontend/jinja2/release-detail.html b/controlpanel/frontend/jinja2/release-detail.html index d74dea699..c73160fa8 100644 --- a/controlpanel/frontend/jinja2/release-detail.html +++ b/controlpanel/frontend/jinja2/release-detail.html @@ -91,6 +91,23 @@

{{ page_title }}

"value": form.version.value(), "errorMessage": { "html": form.version.errors|join(". ") } if form.version.errors else {} }) }} + {{ govukInput({ + "label": { + "text": "Image Tag", + "classes": "govuk-label--m", + }, + "classes": "govuk-!-width-two-thirds", + "hint": { + "text": 'Image tag for this release.' + }, + "name": "image_tag", + "attributes": { + "pattern": "[a-z0-9.-]{1,60}", + "maxlength": "60", + }, + "value": form.image_tag.value(), + "errorMessage": { "html": form.image_tag.errors|join(". ") } if form.image_tag.errors else {} + }) }} {{ govukInput({ "label": { "text": "Custom Domain Name", From fbfb8a5c2a3139107cd94b4cacd9416e9a7e5ca1 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:35:21 +0000 Subject: [PATCH 14/21] Make tool description a required field --- .../migrations/0054_alter_tool_description.py | 18 ++++++++++++++++++ controlpanel/api/models/tool.py | 2 +- .../frontend/jinja2/release-create.html | 4 ++-- .../frontend/jinja2/release-detail.html | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 controlpanel/api/migrations/0054_alter_tool_description.py diff --git a/controlpanel/api/migrations/0054_alter_tool_description.py b/controlpanel/api/migrations/0054_alter_tool_description.py new file mode 100644 index 000000000..235c92833 --- /dev/null +++ b/controlpanel/api/migrations/0054_alter_tool_description.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-12-11 14:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_tool_image_tag"), + ] + + operations = [ + migrations.AlterField( + model_name="tool", + name="description", + field=models.TextField(), + ), + ] diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index 620c133c4..ad7d7766c 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -29,7 +29,7 @@ class Tool(TimeStampedModel): } DEFAULT_DEPRECATED_MESSAGE = "The selected release has been deprecated and will be retired soon. Please update to a more recent version." # noqa - description = models.TextField(blank=True) + description = models.TextField(blank=False) chart_name = models.CharField(max_length=100, blank=False) name = models.CharField(max_length=100, blank=False) values = JSONField(default=dict) diff --git a/controlpanel/frontend/jinja2/release-create.html b/controlpanel/frontend/jinja2/release-create.html index 5b3679ea0..88ff38fc9 100644 --- a/controlpanel/frontend/jinja2/release-create.html +++ b/controlpanel/frontend/jinja2/release-create.html @@ -29,7 +29,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'Human readable name. 60 chars max' + "text": 'Human readable name. Only used in the Tool Releases admin page, not to standard users.' }, "name": "name", "attributes": { @@ -62,7 +62,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'The description for this version, if you leave it blank, the info from helm repo will be used instead' + "text": 'A brief description of this version. This is displayed to users in the dropdown selections.' }, "name": "description", "attributes": { diff --git a/controlpanel/frontend/jinja2/release-detail.html b/controlpanel/frontend/jinja2/release-detail.html index c73160fa8..62b960369 100644 --- a/controlpanel/frontend/jinja2/release-detail.html +++ b/controlpanel/frontend/jinja2/release-detail.html @@ -65,7 +65,7 @@

{{ page_title }}

}, "classes": "govuk-!-width-two-thirds", "hint": { - "text": 'The description about this version, if leave blank, the info from helm repo will be pulled out' + "text": 'A brief description of this version. This is displayed to users in the dropdown selections.' }, "name": "description", "attributes": { From da0200b88a79bf04d70ded3020f1f3c39b7c17c3 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:37:01 +0000 Subject: [PATCH 15/21] Fix queryset when looking form related tool Previously all restricted tools were excluded. However, allowing tools to be deprecated means we can update this logic. As some restricted tools may not be deprecated, so should not display an "unsupported" message. --- controlpanel/frontend/views/tool.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/controlpanel/frontend/views/tool.py b/controlpanel/frontend/views/tool.py index a78e416a1..f636bd462 100644 --- a/controlpanel/frontend/views/tool.py +++ b/controlpanel/frontend/views/tool.py @@ -69,10 +69,8 @@ def _find_related_tool_record(self, chart_name, chart_version, image_tag): memory, CPU etc, then the linkage will be confused although it won't affect people usage. """ - tool_set = Tool.objects.filter( - chart_name=chart_name, version=chart_version, is_restricted=False - ).exclude(is_retired=True) - for tool in tool_set: + tools = self.get_queryset().filter(chart_name=chart_name, version=chart_version) + for tool in tools: if tool.image_tag == image_tag: return tool # If we cant find a tool with the same image tag, this must mean that it was retired or From 32e1493bab109f602c7993b1032aad36ebae24eb Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:49:28 +0000 Subject: [PATCH 16/21] Fix tests --- controlpanel/api/models/tool.py | 2 +- tests/api/models/test_tool.py | 26 ++++++- tests/frontend/test_forms.py | 129 +++++++++----------------------- 3 files changed, 61 insertions(+), 96 deletions(-) diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index ad7d7766c..20ab02032 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -98,7 +98,7 @@ def image_tag_key(self): "rstudio": "rstudio.image.tag", "vscode": "vscode.image.tag", } - return mapping["chart_name"] + return mapping[self.chart_name] class ToolDeploymentManager: diff --git a/tests/api/models/test_tool.py b/tests/api/models/test_tool.py index 7e7466f02..3b47cedea 100644 --- a/tests/api/models/test_tool.py +++ b/tests/api/models/test_tool.py @@ -12,7 +12,7 @@ @pytest.fixture def tool(db): - return baker.make("api.Tool") + return baker.make("api.Tool", chart_name="rstudio", version="1.0.0", image_tag="0.0.1") def test_deploy_for_generic(helm, tool, users): @@ -46,6 +46,8 @@ def test_deploy_for_generic(helm, tool, users): f"aws.iamRole={user.iam_role_name}", "--set", f"toolsDomain={settings.TOOLS_DOMAIN}", + "--set", + f"rstudio.image.tag={tool.image_tag}", ) @@ -91,3 +93,25 @@ def test_home_directory_get_status(): hd._poll = MagicMock(return_value=cluster.HOME_RESETTING) hd._subprocess = True assert hd.get_status() == cluster.HOME_RESETTING + + +@pytest.mark.parametrize( + "chart_name, expected", + [ + ("jupyter-lab-datascience-notebook", "jupyter.tag"), + ("jupyter-lab-all-spark", "jupyter.tag"), + ("jupyter-lab", "jupyterlab.image.tag"), + ("rstudio", "rstudio.image.tag"), + ("vscode", "vscode.image.tag"), + ], + ids=[ + "jupyter-lab-datascience-notebook", + "jupyter-lab-all-spark", + "jupyter-lab", + "rstudio", + "vscode", + ], +) +def test_image_tag_key(tool, chart_name, expected): + tool.chart_name = chart_name + assert tool.image_tag_key == expected diff --git a/tests/frontend/test_forms.py b/tests/frontend/test_forms.py index b0fbc3d9d..d04c6f591 100644 --- a/tests/frontend/test_forms.py +++ b/tests/frontend/test_forms.py @@ -13,112 +13,53 @@ from controlpanel.frontend import forms -def test_tool_release_form_check_release_name(): - """ - Ensure valid chart names work, while invalid ones cause a helpful - exception. - """ - data = { +@pytest.fixture +def release_data(): + return { "name": "Test Release", "chart_name": "jupyter-lab", "version": "1.2.3", "values": {"foo": "bar"}, "is_restricted": False, + "image_tag": "1.0.0", + "description": "Test release description", } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "jupyter-lab-all-spark", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "rstudio", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "airflow-sqlite", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "vscode", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "invalid-chartname", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() is False -def test_tool_release_form_check_tool_domain(): +@pytest.mark.parametrize( + "chart_name, expected", + [ + ("jupyter-lab", True), + ("rstudio", True), + ("jupyter-lab-all-spark", True), + ("vscode", True), + ("jupyter-lab-datascience-notebook", True), + ("invalid-chartname", False), + ], +) +def test_tool_release_form_check_chart_name(release_data, chart_name, expected): + """ + Ensure valid chart names work, while invalid ones cause a helpful + exception. + """ + release_data["chart_name"] = chart_name + f = forms.ToolReleaseForm(release_data) + assert f.is_valid() is expected + + +@pytest.mark.parametrize( + "tool_domain, expected", + [("jupyter-lab", True), ("rstudio", True), ("vscode", True), ("invalid-tool-domain", False)], +) +def test_tool_release_form_check_tool_domain(release_data, tool_domain, expected): """ Ensure ONLY valid chart names work, while invalid ones cause a helpful exception. """ - data = { - "name": "Test Release", - "chart_name": "jupyter-lab", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - "tool_domain": "jupyter-lab", - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "jupyter-lab-all-spark", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - "tool_domain": "jupyter-lab", - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "vscode", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - "tool_domain": "vscode", - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() - data = { - "name": "Test Release", - "chart_name": "jupyter-lab-all-spark", - "version": "1.2.3", - "values": {"foo": "bar"}, - "is_restricted": False, - "tool_domain": "invalid-tool-domain", - } - f = forms.ToolReleaseForm(data) - assert f.is_valid() is False + release_data["tool_domain"] = tool_domain + + f = forms.ToolReleaseForm(release_data) + assert f.is_valid() is expected def test_tool_release_form_get_target_users(): From 97a2fbfcd23a1b1c6332cb9e3e485a7fc2692af6 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:39:18 +0000 Subject: [PATCH 17/21] Refactoring and add notes for further changes --- controlpanel/api/helm.py | 3 +++ controlpanel/api/models/tool.py | 2 ++ controlpanel/frontend/views/tool.py | 40 +++-------------------------- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/controlpanel/api/helm.py b/controlpanel/api/helm.py index e6b704961..e91512a0f 100644 --- a/controlpanel/api/helm.py +++ b/controlpanel/api/helm.py @@ -158,6 +158,7 @@ def update_helm_repository(force=False): _execute("repo", "update", timeout=None) # timeout = infinity. +# TODO no longer used, remove def get_default_image_tag_from_helm_chart(chart_url, chart_name): proc = _execute("show", "values", chart_url) if proc: @@ -171,6 +172,8 @@ def get_default_image_tag_from_helm_chart(chart_url, chart_name): return None +# TODO this is no longer called from the Your Tools page +# consider removing as part of further refactoring def get_helm_entries(): # Update repository metadata. update_helm_repository() diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index 20ab02032..e347bdb00 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -73,6 +73,8 @@ def url(self, user): def save(self, *args, **kwargs): helm.update_helm_repository(force=True) + # TODO description is now required when creating a release, so this is unlikely to be called + # Consider removing if not self.description: self.description = helm.get_chart_app_version(self.chart_name, self.version) or "" diff --git a/controlpanel/frontend/views/tool.py b/controlpanel/frontend/views/tool.py index f636bd462..091c16ddf 100644 --- a/controlpanel/frontend/views/tool.py +++ b/controlpanel/frontend/views/tool.py @@ -13,11 +13,6 @@ # First-party/Local from controlpanel.api import cluster -from controlpanel.api.helm import ( - get_chart_version_info, - get_default_image_tag_from_helm_chart, - get_helm_entries, -) from controlpanel.api.models import Tool, ToolDeployment from controlpanel.oidc import OIDCLoginRequiredMixin from controlpanel.utils import start_background_task @@ -85,10 +80,6 @@ def _add_new_item_to_tool_box(self, user, tool_box, tool, tools_info): "deployment": None, "releases": {}, } - # TODO We should update model to always store an image tag - # image_tag = tool.image_tag - # if not image_tag: - # image_tag = charts_info.get(tool.version, {}) or "unknown" if tool.id not in tools_info[tool_box]["releases"]: tools_info[tool_box]["releases"][tool.id] = { "tool_id": tool.id, @@ -110,6 +101,7 @@ def _add_deployed_charts_info(self, tools_info, user, id_token): # Get list of deployed tools # TODO this sets what tool the user currently has deployed. If we were to refactor to store # deployed tools in the database, we could remove a lot of this logic + # See https://github.com/ministryofjustice/analytical-platform/issues/6266 deployments = cluster.ToolDeployment.get_deployments(user, id_token) for deployment in deployments: chart_name, chart_version = deployment.metadata.labels["chart"].rsplit("-", 1) @@ -142,7 +134,8 @@ def _add_deployed_charts_info(self, tools_info, user, id_token): } def _retrieve_detail_tool_info(self, user, tools): - # TODO why do we need this? We could change so that all information required about available tools comes from the DB # noqa: E501 + # TODO when deployed tools are tracked in the DB this will not be needed + # see https://github.com/ministryofjustice/analytical-platform/issues/6266 # noqa: E501 tools_info = {} for tool in tools: # Work out which bucket the chart should be in @@ -152,31 +145,6 @@ def _retrieve_detail_tool_info(self, user, tools): self._add_new_item_to_tool_box(user, tool_box, tool, tools_info) return tools_info - def _get_charts_info(self, tool_list): - # We may need the default image_tag from helm chart - # unless we configure it specifically in parameters of tool release - # TODO if we make sure that we always have an image_tag for a tool, then building charts_info is redundant and could be removed # noqa: E501 - charts_info = {} - chart_entries = None - for tool in tool_list: - if tool.version in charts_info: - continue - - image_tag = tool.image_tag - if image_tag: - continue - - if chart_entries is None: - # load the potential massive helm chart yaml only it is necessary - chart_entries = get_helm_entries() - chart_app_version = get_chart_version_info(chart_entries, tool.chart_name, tool.version) - if chart_app_version: - image_tag = get_default_image_tag_from_helm_chart( - chart_app_version.chart_url, tool.chart_name - ) - charts_info[tool.version] = image_tag - return charts_info - def get_context_data(self, *args, **kwargs): """ Retrieve information about tools and arrange them for the @@ -238,8 +206,6 @@ def get_context_data(self, *args, **kwargs): context["managed_airflow_dev_url"] = f"{settings.AWS_SERVICE_URL}/?{args_airflow_dev_url}" context["managed_airflow_prod_url"] = f"{settings.AWS_SERVICE_URL}/?{args_airflow_prod_url}" - # Arrange tools information - # charts_info = self._get_charts_info(context["tools"]) tools_info = self._retrieve_detail_tool_info(user, context["tools"]) if "vscode" in tools_info: From 8caf31241f65669b1b142937b9715b3af560aa31 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:10:03 +0000 Subject: [PATCH 18/21] Add further tests --- controlpanel/api/cluster.py | 9 +----- controlpanel/api/models/tool.py | 15 ++++++---- controlpanel/frontend/jinja2/tool-list.html | 2 +- .../static/javascripts/modules/tool-status.js | 4 --- tests/api/cluster/test_tool_deployment.py | 30 +++++++++++++++++++ tests/api/models/test_tool.py | 10 +++++++ 6 files changed, 52 insertions(+), 18 deletions(-) diff --git a/controlpanel/api/cluster.py b/controlpanel/api/cluster.py index 3f36b4b1f..995c1cdae 100644 --- a/controlpanel/api/cluster.py +++ b/controlpanel/api/cluster.py @@ -970,13 +970,6 @@ def _delete_legacy_release(self): if old_release_name in helm.list_releases(old_release_name, self.k8s_namespace): helm.delete(self.k8s_namespace, old_release_name) - @property - def get_image_tag_helm_value(self): - """ - Return the image tag to be used in the helm chart values - """ - return {self.tool.image_tag_key: self.tool.image_tag} - def _set_values(self, **kwargs): """ Return the list of `--set KEY=VALUE` helm upgrade arguments @@ -999,7 +992,7 @@ def _set_values(self, **kwargs): values.update(self.tool.values) values.update(kwargs) # override the tool image tag with the value stored in the DB - values.update(self.get_image_tag_helm_value) + values.update({self.tool.image_tag_key: self.tool.image_tag}) set_values = [] for key, val in values.items(): if val: diff --git a/controlpanel/api/models/tool.py b/controlpanel/api/models/tool.py index e347bdb00..943ea8da3 100644 --- a/controlpanel/api/models/tool.py +++ b/controlpanel/api/models/tool.py @@ -28,6 +28,11 @@ class Tool(TimeStampedModel): "vscode": "vscode", } DEFAULT_DEPRECATED_MESSAGE = "The selected release has been deprecated and will be retired soon. Please update to a more recent version." # noqa + JUPYTER_DATASCIENCE_CHART_NAME = "jupyter-lab-datascience-notebook" + JUPYTER_ALL_SPARK_CHART_NAME = "jupyter-lab-all-spark" + JUPYTER_LAB_CHART_NAME = "jupyter-lab" + RSTUDIO_CHART_NAME = "rstudio" + VSCODE_CHART_NAME = "vscode" description = models.TextField(blank=False) chart_name = models.CharField(max_length=100, blank=False) @@ -94,11 +99,11 @@ def get_deprecated_message(self): @property def image_tag_key(self): mapping = { - "jupyter-lab-datascience-notebook": "jupyter.tag", - "jupyter-lab-all-spark": "jupyter.tag", - "jupyter-lab": "jupyterlab.image.tag", - "rstudio": "rstudio.image.tag", - "vscode": "vscode.image.tag", + self.JUPYTER_DATASCIENCE_CHART_NAME: "jupyter.tag", + self.JUPYTER_ALL_SPARK_CHART_NAME: "jupyter.tag", + self.JUPYTER_LAB_CHART_NAME: "jupyterlab.image.tag", + self.RSTUDIO_CHART_NAME: "rstudio.image.tag", + self.VSCODE_CHART_NAME: "vscode.image.tag", } return mapping[self.chart_name] diff --git a/controlpanel/frontend/jinja2/tool-list.html b/controlpanel/frontend/jinja2/tool-list.html index 5c4ce3e0c..1c38eb5bd 100644 --- a/controlpanel/frontend/jinja2/tool-list.html +++ b/controlpanel/frontend/jinja2/tool-list.html @@ -38,7 +38,7 @@

{{ tool_info.name }}

id="tools-{{ chart_name }}" name="version"> {% set installed_chart_version = None %} {% set installed_release_version = None %} - {% if deployment and deployment.tool_id > -1 %} + {% if deployment and not deployment.is_retired %} {% set installed_chart_version = deployment.description %} {% set installed_release_version = deployment.tool_id %}