From 15848b236d7fc80ba0134708fc00c161cc76cb5f Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 9 Sep 2020 19:04:15 +0300 Subject: [PATCH 001/447] Remove group tags --- ckan/lib/dictization/model_dictize.py | 10 -------- ckan/logic/action/get.py | 16 +------------ .../lib/dictization/test_model_dictize.py | 24 ------------------- 3 files changed, 1 insertion(+), 49 deletions(-) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index ecaa57652ca..1e4ba1547b0 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -32,7 +32,6 @@ def group_list_dictize(obj_list, context, sort_key=lambda x: x['display_name'], reverse=False, with_package_counts=True, include_groups=False, - include_tags=False, include_extras=False): group_dictize_context = dict(context.items()) @@ -42,7 +41,6 @@ def group_list_dictize(obj_list, context, 'packages_field': 'dataset_count' if with_package_counts else None, # don't allow packages_field='datasets' as it is too slow 'include_groups': include_groups, - 'include_tags': include_tags, 'include_extras': include_extras, 'include_users': False, # too slow - don't allow } @@ -276,7 +274,6 @@ def get_group_dataset_counts(): def group_dictize(group, context, include_groups=True, - include_tags=True, include_users=True, include_extras=True, packages_field='datasets', @@ -357,13 +354,6 @@ def get_packages_for_this_group(group_, just_the_count=False): result_dict['package_count'] = package_count - if include_tags: - # group tags are not creatable via the API yet, but that was(/is) a - # future intention (see kindly's commit 5c8df894 on 2011/12/23) - result_dict['tags'] = tag_list_dictize( - _get_members(context, group, 'tags'), - context) - if include_groups: # these sub-groups won't have tags or extras for speed result_dict['groups'] = group_list_dictize( diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 37b60949901..cc1bb056d7e 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -448,7 +448,7 @@ def _group_or_org_list(context, data_dict, is_org=False): group_list = [] for group in groups: data_dict['id'] = group.id - for key in ('include_extras', 'include_tags', 'include_users', + for key in ('include_extras', 'include_users', 'include_groups', 'include_followers'): if key not in data_dict: data_dict[key] = False @@ -495,9 +495,6 @@ def group_list(context, data_dict): :param include_extras: if all_fields, include the group extra fields (optional, default: ``False``) :type include_extras: bool - :param include_tags: if all_fields, include the group tags - (optional, default: ``False``) - :type include_tags: bool :param include_groups: if all_fields, include the groups the groups are in (optional, default: ``False``). :type include_groups: bool @@ -547,9 +544,6 @@ def organization_list(context, data_dict): :param include_extras: if all_fields, include the organization extra fields (optional, default: ``False``) :type include_extras: bool - :param include_tags: if all_fields, include the organization tags - (optional, default: ``False``) - :type include_tags: bool :param include_groups: if all_fields, include the organizations the organizations are in (optional, default: ``False``) @@ -1164,7 +1158,6 @@ def _group_or_org_show(context, data_dict, is_org=False): packages_field = None try: - include_tags = asbool(data_dict.get('include_tags', True)) if asbool(config.get('ckan.auth.public_user_details', True)): include_users = asbool(data_dict.get('include_users', True)) else: @@ -1189,7 +1182,6 @@ def _group_or_org_show(context, data_dict, is_org=False): group_dict = model_dictize.group_dictize(group, context, packages_field=packages_field, - include_tags=include_tags, include_extras=include_extras, include_groups=include_groups, include_users=include_users,) @@ -1246,9 +1238,6 @@ def group_show(context, data_dict): :param include_groups: include the group's sub groups (optional, default: ``True``) :type include_groups: bool - :param include_tags: include the group's tags - (optional, default: ``True``) - :type include_tags: bool :param include_followers: include the group's number of followers (optional, default: ``True``) :type include_followers: bool @@ -1281,9 +1270,6 @@ def organization_show(context, data_dict): :param include_groups: include the organization's sub groups (optional, default: ``True``) :type include_groups: bool - :param include_tags: include the organization's tags - (optional, default: ``True``) - :type include_tags: bool :param include_followers: include the organization's number of followers (optional, default: ``True``) :type include_followers: bool diff --git a/ckan/tests/lib/dictization/test_model_dictize.py b/ckan/tests/lib/dictization/test_model_dictize.py index 28a7cfbd181..06592291403 100644 --- a/ckan/tests/lib/dictization/test_model_dictize.py +++ b/ckan/tests/lib/dictization/test_model_dictize.py @@ -94,30 +94,6 @@ def test_group_list_dictize_including_extras(self): assert group_dicts[0]["extras"][0]["key"] == "k1" - def test_group_list_dictize_including_tags(self): - factories.Group() - # group tags aren't in the group_create schema, so its slightly more - # convoluted way to create them - group_obj = model.Session.query(model.Group).first() - tag = model.Tag(name="t1") - model.Session.add(tag) - model.Session.commit() - tag = model.Session.query(model.Tag).first() - group_obj = model.Session.query(model.Group).first() - member = model.Member( - group=group_obj, table_id=tag.id, table_name="tag" - ) - model.Session.add(member) - model.Session.commit() - group_list = model.Session.query(model.Group).filter_by().all() - context = {"model": model, "session": model.Session} - - group_dicts = model_dictize.group_list_dictize( - group_list, context, include_tags=True - ) - - assert group_dicts[0]["tags"][0]["name"] == "t1" - def test_group_list_dictize_including_groups(self): factories.Group(name="parent") factories.Group(name="child", groups=[{"name": "parent"}]) From 139ef7ebb4804811d6f5d36f29c9de7f61894e4c Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Sat, 12 Sep 2020 12:07:12 +0300 Subject: [PATCH 002/447] Remove "tags" mention from tests --- ckan/tests/legacy/lib/test_dictization_schema.py | 1 - ckan/tests/lib/dictization/test_model_dictize.py | 1 - ckan/tests/logic/action/test_get.py | 7 ++++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ckan/tests/legacy/lib/test_dictization_schema.py b/ckan/tests/legacy/lib/test_dictization_schema.py index 9e9ef046606..313a40b2c9b 100644 --- a/ckan/tests/legacy/lib/test_dictization_schema.py +++ b/ckan/tests/legacy/lib/test_dictization_schema.py @@ -132,7 +132,6 @@ def test_2_group_schema(self): # we don't want these here del data["groups"] del data["users"] - del data["tags"] del data["extras"] converted_data, errors = validate( diff --git a/ckan/tests/lib/dictization/test_model_dictize.py b/ckan/tests/lib/dictization/test_model_dictize.py index 06592291403..2f4d2a61fd5 100644 --- a/ckan/tests/lib/dictization/test_model_dictize.py +++ b/ckan/tests/lib/dictization/test_model_dictize.py @@ -122,7 +122,6 @@ def test_group_dictize(self): assert group["name"] == "test_dictize" assert group["packages"] == [] assert group["extras"] == [] - assert group["tags"] == [] assert group["groups"] == [] def test_group_dictize_group_with_dataset(self): diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index 7e53a64cea4..bfcb2ee3fdc 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -332,7 +332,7 @@ def test_group_list_all_fields(self): group_list = helpers.call_action("group_list", all_fields=True) expected_group = dict(group) - for field in ("users", "tags", "extras", "groups"): + for field in ("users", "extras", "groups"): del expected_group[field] assert group_list[0] == expected_group @@ -961,8 +961,9 @@ def test_user_list_order_by_created_datasets(self): ] datasets = [ factories.Dataset(user=users[1]), - factories.Dataset(user=users[1]) + factories.Dataset(user=users[1]), ] + factories.Dataset(user=users[2]) for dataset in datasets: dataset["title"] = "Edited title" helpers.call_action( @@ -971,7 +972,7 @@ def test_user_list_order_by_created_datasets(self): expected_names = [ u['name'] for u in [ users[0], # 0 packages created - users[2], # 0 packages created + users[2], # 1 packages created users[1], # 2 packages created ] ] From 0320961e7a8e3f36b1206a8c28f19849fb8bba68 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 14 Jul 2021 13:24:23 +0300 Subject: [PATCH 003/447] Allow patterns in devserver extra_paths --- ckan/cli/server.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ckan/cli/server.py b/ckan/cli/server.py index de061ed6c0f..babe16bb648 100644 --- a/ckan/cli/server.py +++ b/ckan/cli/server.py @@ -1,6 +1,8 @@ # encoding: utf-8 +import glob import logging +import itertools import click from werkzeug.serving import run_simple @@ -49,9 +51,12 @@ def run(ctx, host, port, disable_reloader, threaded, extra_files, processes, config_extra_files = tk.aslist( config.get(u"ckan.devserver.watch_patterns") ) - extra_files = list(extra_files) + [ - config[u"__file__"] - ] + config_extra_files + extra_files = itertools.chain( + [config[u"__file__"]], + *[glob.glob(path, recursive=True) + for path + in list(extra_files) + config_extra_files] + ) # Threads and processes threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded")) From 272508b5de2789f000066727347f1778edfb917f Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Wed, 14 Jul 2021 12:30:15 -0400 Subject: [PATCH 004/447] [#6118] custom url_type button for table designer --- ckanext/tabledesigner/plugin.py | 11 ++++++++++ .../snippets/resource_upload_field.html | 21 +++++++++++++++++++ setup.py | 1 + 3 files changed, 33 insertions(+) create mode 100644 ckanext/tabledesigner/plugin.py create mode 100644 ckanext/tabledesigner/templates/package/snippets/resource_upload_field.html diff --git a/ckanext/tabledesigner/plugin.py b/ckanext/tabledesigner/plugin.py new file mode 100644 index 00000000000..5c10f065d37 --- /dev/null +++ b/ckanext/tabledesigner/plugin.py @@ -0,0 +1,11 @@ +import ckan.plugins as plugins +import ckan.plugins.toolkit as toolkit + + +class TableDesignerPlugin(plugins.SingletonPlugin): + plugins.implements(plugins.IConfigurer) + + # IConfigurer + + def update_config(self, config_): + toolkit.add_template_directory(config_, "templates") diff --git a/ckanext/tabledesigner/templates/package/snippets/resource_upload_field.html b/ckanext/tabledesigner/templates/package/snippets/resource_upload_field.html new file mode 100644 index 00000000000..789d8927eba --- /dev/null +++ b/ckanext/tabledesigner/templates/package/snippets/resource_upload_field.html @@ -0,0 +1,21 @@ +{% ckan_extends %} + +{% block url_type_select %} + {{ super() }} + +{% endblock %} + +{% block url_type_fields %} + {{ super() }} + +
+ {{ remove_button() }} + +

{{ _('Use the Data Dictionary tab to customize this table.') }}

+
+{% endblock %} diff --git a/setup.py b/setup.py index 448e825ac80..cd172769c9f 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ 'chained_functions = ckanext.chained_functions.plugin:ChainedFunctionsPlugin', 'datastore = ckanext.datastore.plugin:DatastorePlugin', 'datapusher=ckanext.datapusher.plugin:DatapusherPlugin', + 'table_designer = ckanext.tabledesigner.plugin:TableDesignerPlugin', 'test_tag_vocab_plugin = ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin', 'resource_proxy = ckanext.resourceproxy.plugin:ResourceProxy', 'text_view = ckanext.textview.plugin:TextView', From 397d3a3d98c86554e42da4b32f41b16a4d1f7177 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 9 Sep 2021 17:05:07 +0300 Subject: [PATCH 005/447] Update changelog --- changes/6254.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/6254.feature diff --git a/changes/6254.feature b/changes/6254.feature new file mode 100644 index 00000000000..0758430e884 --- /dev/null +++ b/changes/6254.feature @@ -0,0 +1 @@ +`ckan.devserver.watch_patterns` config option supports glob patterns From cb9b405732782e44ef13913134d58b8c6beb1d9e Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 5 Oct 2021 18:30:18 -0400 Subject: [PATCH 006/447] [#6118] create table for table designer resources --- .../templates/datastore/dictionary.html | 2 +- .../datastore/snippets/dictionary_form.html | 16 ++++++----- ckanext/tabledesigner/actions.py | 28 +++++++++++++++++++ ckanext/tabledesigner/assets/style.css | 5 ++++ ckanext/tabledesigner/assets/webassets.yml | 5 ++++ ckanext/tabledesigner/plugin.py | 10 +++++++ .../datastore/snippets/dictionary_form.html | 8 ++++++ 7 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 ckanext/tabledesigner/actions.py create mode 100644 ckanext/tabledesigner/assets/style.css create mode 100644 ckanext/tabledesigner/assets/webassets.yml create mode 100644 ckanext/tabledesigner/templates/datastore/snippets/dictionary_form.html diff --git a/ckanext/datastore/templates/datastore/dictionary.html b/ckanext/datastore/templates/datastore/dictionary.html index c6e88a4ffb1..6faefe3a5ff 100644 --- a/ckanext/datastore/templates/datastore/dictionary.html +++ b/ckanext/datastore/templates/datastore/dictionary.html @@ -11,7 +11,7 @@
{% block dictionary_form %} {% for field in fields %} - {% snippet "datastore/snippets/dictionary_form.html", field=field, position=loop.index %} + {% snippet "datastore/snippets/dictionary_form.html", field=field, position=loop.index, res=res %} {% endfor %} {% endblock %} +
+{% endblock %} + diff --git a/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html b/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html new file mode 100644 index 00000000000..9e2c981b26d --- /dev/null +++ b/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html @@ -0,0 +1,5 @@ +{% import 'macros/form.html' as form %} + +{{ form.input('col__' ~ position ~ '__value', + label=field.get('info', {}).get('label', '') or field.id, classes=['control-full'], + id='field-' ~ position)}} diff --git a/ckanext/tabledesigner/views.py b/ckanext/tabledesigner/views.py index ec66cfed55c..ed66f77a440 100644 --- a/ckanext/tabledesigner/views.py +++ b/ckanext/tabledesigner/views.py @@ -7,7 +7,13 @@ tuplize_dict, parse_params, ) -from ckan.plugins.toolkit import get_action, h, _, request +from ckan.plugins.toolkit import ( + get_action, + h, + _, + request, + render, +) tabledesigner = Blueprint(u'tabledesigner', __name__) @@ -58,3 +64,39 @@ def post(self, id, resource_id): '/dataset//dictionary/', view_func=_TableDesignerDictionary.as_view('tabledesigner'), ) + + +class _TableDesignerAddRow(MethodView): + def post(self, id, resource_id): + pass + + def get(self, id, resource_id): + _datastore_view = DictionaryView() + data_dict = _datastore_view._prepare(id, resource_id) + return render('tabledesigner/add_row.html', data_dict) + + def post(self, id, resource_id): + _datastore_view = DictionaryView() + data_dict = _datastore_view._prepare(id, resource_id) + + data = dict_fns.unflatten(tuplize_dict(parse_params(request.form))) + col = data.get('col', []) + row = {} + for f, c in zip(data_dict['fields'], col): + row[f['id']] = c['value'] or None + + get_action('datastore_upsert')( + None, { + 'resource_id': resource_id, + 'method': 'insert', + 'force': True, # FIXME: don't require this for tabledesigner tables + 'records': [row], + } + ) + return h.redirect_to(data_dict['pkg_dict']['type'] + '_resource.read', id=id, resource_id=resource_id) + + +tabledesigner.add_url_rule( + '/dataset//tabledesigner//add-row', + view_func=_TableDesignerAddRow.as_view('add_row'), +) From 2091e8ecb9ef7162f28c0b9d8c4f650cedd01abc Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 20 Dec 2021 17:57:41 -0500 Subject: [PATCH 015/447] basic edit row functionality --- .../templates/package/resource_read.html | 19 +++++-- .../templates/tabledesigner/edit_row.html | 23 ++++++++ .../tabledesigner/snippets/row_field.html | 10 ++-- ckanext/tabledesigner/views.py | 52 +++++++++++++++++-- 4 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 ckanext/tabledesigner/templates/tabledesigner/edit_row.html diff --git a/ckanext/tabledesigner/templates/package/resource_read.html b/ckanext/tabledesigner/templates/package/resource_read.html index 07def694803..b644d5dc9e8 100644 --- a/ckanext/tabledesigner/templates/package/resource_read.html +++ b/ckanext/tabledesigner/templates/package/resource_read.html @@ -1,14 +1,27 @@ {% ckan_extends %} +{% import 'macros/form.html' as form %} + {% block resource_additional_information_inner %} {% if res.url_type == 'tabledesigner' and res.datastore_active %} {% block resource_tabledesigner %}

{{ _('Table Designer') }}

+ {% block resource_tabledesigner_edit %} +
{{ form.input('_id', label='_id', classes=['control-full'], id='_id', is_required=true) }} + +
+ {% endblock %} {% block resource_tabledesigner_add %} - {% block add_row_button_text %} {{ _('Add Row') }}{% endblock %} +

+ {% block add_row_button_text %} {{ _('Add Row') }}{% endblock %} +

{% endblock %}
{% endblock %} diff --git a/ckanext/tabledesigner/templates/tabledesigner/edit_row.html b/ckanext/tabledesigner/templates/tabledesigner/edit_row.html new file mode 100644 index 00000000000..2d2c2082b2d --- /dev/null +++ b/ckanext/tabledesigner/templates/tabledesigner/edit_row.html @@ -0,0 +1,23 @@ +{% extends "package/resource_edit_base.html" %} + +{% import 'macros/form.html' as form %} + +{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} + +{% block primary_content_inner %} + + {% set action = h.url_for('tabledesigner.edit_row', id=pkg.name, resource_id=res.id) %} +

{{ _('Edit row') }}

+ +
+ {% block add_row %} + {% for field in fields %} + {% snippet "tabledesigner/snippets/row_field.html", field=field, position=loop.index, res=res, value=row.get(field.id) %} + {% endfor %} + {% endblock %} + +
+{% endblock %} + diff --git a/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html b/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html index 9e2c981b26d..2e31065415c 100644 --- a/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html +++ b/ckanext/tabledesigner/templates/tabledesigner/snippets/row_field.html @@ -1,5 +1,9 @@ {% import 'macros/form.html' as form %} -{{ form.input('col__' ~ position ~ '__value', - label=field.get('info', {}).get('label', '') or field.id, classes=['control-full'], - id='field-' ~ position)}} +{{ + form.input('col__' ~ position ~ '__value', + label=field.get('info', {}).get('label', '') or field.id, classes=['control-full'], + id='field-' ~ position, + value=value or '', + ) +}} diff --git a/ckanext/tabledesigner/views.py b/ckanext/tabledesigner/views.py index ed66f77a440..822172241c8 100644 --- a/ckanext/tabledesigner/views.py +++ b/ckanext/tabledesigner/views.py @@ -7,6 +7,8 @@ tuplize_dict, parse_params, ) +from ckan.lib import base +from ckan.logic import NotAuthorized from ckan.plugins.toolkit import ( get_action, h, @@ -67,9 +69,6 @@ def post(self, id, resource_id): class _TableDesignerAddRow(MethodView): - def post(self, id, resource_id): - pass - def get(self, id, resource_id): _datastore_view = DictionaryView() data_dict = _datastore_view._prepare(id, resource_id) @@ -100,3 +99,50 @@ def post(self, id, resource_id): '/dataset//tabledesigner//add-row', view_func=_TableDesignerAddRow.as_view('add_row'), ) + + +class _TableDesignerEditRow(MethodView): + def get(self, id, resource_id): + _datastore_view = DictionaryView() + data_dict = _datastore_view._prepare(id, resource_id) + _id = request.params['_id'] + + try: + r = get_action('datastore_search')( + None, { + 'resource_id': resource_id, + 'filters': {'_id': _id}, + } + ) + except NotAuthorized: + return base.abort(403, _('Not authorized to see this page')) + if not r['records']: + return base.abort(404, _('Row not found')) + data_dict['row'] = r['records'][0] + return render('tabledesigner/edit_row.html', data_dict) + + def post(self, id, resource_id): + _datastore_view = DictionaryView() + data_dict = _datastore_view._prepare(id, resource_id) + _id = request.params['_id'] + + data = dict_fns.unflatten(tuplize_dict(parse_params(request.form))) + col = data.get('col', []) + row = {'_id': _id} + for f, c in zip(data_dict['fields'], col): + row[f['id']] = c['value'] or None + + get_action('datastore_upsert')( + None, { + 'resource_id': resource_id, + 'method': 'update', + 'force': True, # FIXME: don't require this for tabledesigner tables + 'records': [row], + } + ) + return h.redirect_to(data_dict['pkg_dict']['type'] + '_resource.read', id=id, resource_id=resource_id) + +tabledesigner.add_url_rule( + '/dataset//tabledesigner//edit-row', + view_func=_TableDesignerEditRow.as_view('edit_row'), +) From 826046a25cb13e090d64f6a94f8c8382f08964f1 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 21 Feb 2022 10:35:14 +0200 Subject: [PATCH 016/447] Update types --- ckan/cli/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/cli/server.py b/ckan/cli/server.py index 9585a50e9dd..d0d354834e1 100644 --- a/ckan/cli/server.py +++ b/ckan/cli/server.py @@ -5,7 +5,7 @@ import itertools import logging import warnings -from typing import Optional +from typing import Iterable, Optional import click from werkzeug.serving import run_simple @@ -47,7 +47,7 @@ " automatically generate a new one (on each server reload).") @click.pass_context def run(ctx: click.Context, host: str, port: str, disable_reloader: bool, - threaded: bool, extra_files: list[str], processes: int, + threaded: bool, extra_files: Iterable[str], processes: int, ssl_cert: Optional[str], ssl_key: Optional[str]): u"""Runs the Werkzeug development server""" From 1d1c083a695ad40ea4cd6184f219dc94a259097c Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 12 Sep 2022 13:36:11 -0400 Subject: [PATCH 017/447] [#6118] restore plugin after setup.py change --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 2df3ecc8487..e34dd48f3b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,6 +72,7 @@ ckan.plugins = recline_grid = ckanext.reclineview.plugin:ReclineGridView recline_graph = ckanext.reclineview.plugin:ReclineGraphView recline_map = ckanext.reclineview.plugin:ReclineMapView + tabledesigner = ckanext.tabledesigner.plugin:TableDesignerPlugin example_itemplatehelpers = ckanext.example_itemplatehelpers.plugin:ExampleITemplateHelpersPlugin example_idatasetform = ckanext.example_idatasetform.plugin:ExampleIDatasetFormPlugin example_idatasetform_inherit = ckanext.example_idatasetform.plugin:ExampleIDatasetFormInheritPlugin From 90d8c6f73e2599d93f3d8458c4cc45820a9ea134 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 26 Sep 2022 14:18:05 -0400 Subject: [PATCH 018/447] [#6118] display error on duplicate pk --- .../templates/tabledesigner/add_row.html | 2 +- .../templates/tabledesigner/edit_row.html | 2 +- .../tabledesigner/snippets/row_field.html | 1 + ckanext/tabledesigner/views.py | 35 ++++++++++++++----- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/ckanext/tabledesigner/templates/tabledesigner/add_row.html b/ckanext/tabledesigner/templates/tabledesigner/add_row.html index 4b898698af5..955118468aa 100644 --- a/ckanext/tabledesigner/templates/tabledesigner/add_row.html +++ b/ckanext/tabledesigner/templates/tabledesigner/add_row.html @@ -12,7 +12,7 @@

{{ _('Add row') }}

{% block add_row %} {% for field in fields %} - {% snippet "tabledesigner/snippets/row_field.html", field=field, position=loop.index, res=res %} + {% snippet "tabledesigner/snippets/row_field.html", field=field, position=loop.index, res=res, value=row.get(field.id), errors=errors %} {% endfor %} {% endblock %} + +
+
+ {{ _('Get 5 results containing "jones" in any field:') }} + {% for clang in code_languages %} + {{ code_examples[clang]['request_limit'] }} + {% endfor %} + + + {{ _('Get results with either "watershed" or "survey" as subject and "active" as its stage:') }} + {% for clang in code_languages %} + {{ code_examples[clang]['request_filter'] }} + {% endfor %} + + {% if h.datastore_search_sql_enabled() %} + {{ _('SQL query example:') }} + {% for clang in code_languages %} + {{ code_examples[clang]['request_sql'] }} + {% endfor %} + {% endif %} +
+
+ +
+

+ +

+
+
+ +
+
+
+
+

+ +

+
+
+ +
+
+
+ {% else %} + {{ super() }} + {% endif %} +{% endblock %} From 2c7f62882eb2ada70f534b6d6f07fcdf3366d863 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 30 May 2023 17:04:15 -0400 Subject: [PATCH 039/447] [#6118] dynamic query, insert, update, delete parameters --- .../templates/ajax_snippets/api_info.html | 10 +- .../datastore/api_examples/curl.html | 3 + .../datastore/api_examples/javascript.html | 3 + .../datastore/api_examples/powershell.html | 2 + .../datastore/api_examples/python.html | 2 + .../templates/datastore/api_examples/r.html | 3 + ckanext/tabledesigner/helpers.py | 27 ++++- .../templates/ajax_snippets/api_info.html | 37 +++++-- .../datastore/api_examples/curl.html | 61 +++++++++++ .../datastore/api_examples/javascript.html | 102 ++++++++++++++++++ .../datastore/api_examples/powershell.html | 75 +++++++++++++ .../datastore/api_examples/python.html | 66 ++++++++++++ .../templates/datastore/api_examples/r.html | 93 ++++++++++++++++ 13 files changed, 469 insertions(+), 15 deletions(-) create mode 100644 ckanext/tabledesigner/templates/datastore/api_examples/curl.html create mode 100644 ckanext/tabledesigner/templates/datastore/api_examples/javascript.html create mode 100644 ckanext/tabledesigner/templates/datastore/api_examples/powershell.html create mode 100644 ckanext/tabledesigner/templates/datastore/api_examples/python.html create mode 100644 ckanext/tabledesigner/templates/datastore/api_examples/r.html diff --git a/ckanext/datastore/templates/ajax_snippets/api_info.html b/ckanext/datastore/templates/ajax_snippets/api_info.html index ae484105358..aff89590153 100644 --- a/ckanext/datastore/templates/ajax_snippets/api_info.html +++ b/ckanext/datastore/templates/ajax_snippets/api_info.html @@ -20,10 +20,12 @@ {% set code_languages = ['curl', 'javascript', 'powershell', 'python', 'r'] %} -{# register examples by including templates here #} -{% for clang in code_languages %} - {% include "/datastore/api_examples/" + clang + ".html" %} -{% endfor %} +{% block register_examples %} + {# register examples by including templates here #} + {% for clang in code_languages %} + {% include "/datastore/api_examples/" + clang + ".html" %} + {% endfor %} +{% endblock %}
diff --git a/ckanext/datastore/templates/datastore/api_examples/curl.html b/ckanext/datastore/templates/datastore/api_examples/curl.html index 939bf0b959a..f2d01d5e2a0 100644 --- a/ckanext/datastore/templates/datastore/api_examples/curl.html +++ b/ckanext/datastore/templates/datastore/api_examples/curl.html @@ -38,3 +38,6 @@ } END {% endcall %} + + +{% block custom %}{% endblock %} diff --git a/ckanext/datastore/templates/datastore/api_examples/javascript.html b/ckanext/datastore/templates/datastore/api_examples/javascript.html index d0a065d5935..a9c021b0767 100644 --- a/ckanext/datastore/templates/datastore/api_examples/javascript.html +++ b/ckanext/datastore/templates/datastore/api_examples/javascript.html @@ -53,3 +53,6 @@ }) await resp.json() {% endcall %} + + +{% block custom %}{% endblock %} diff --git a/ckanext/datastore/templates/datastore/api_examples/powershell.html b/ckanext/datastore/templates/datastore/api_examples/powershell.html index c008d0adffc..5d4cd280b87 100644 --- a/ckanext/datastore/templates/datastore/api_examples/powershell.html +++ b/ckanext/datastore/templates/datastore/api_examples/powershell.html @@ -46,3 +46,5 @@ -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"} $response.result.records {% endcall %} + +{% block custom %}{% endblock %} diff --git a/ckanext/datastore/templates/datastore/api_examples/python.html b/ckanext/datastore/templates/datastore/api_examples/python.html index f15d2d934b2..be2aea145fb 100644 --- a/ckanext/datastore/templates/datastore/api_examples/python.html +++ b/ckanext/datastore/templates/datastore/api_examples/python.html @@ -50,3 +50,5 @@ print(result['records']) {% endcall %} + +{% block custom %}{% endblock %} diff --git a/ckanext/datastore/templates/datastore/api_examples/r.html b/ckanext/datastore/templates/datastore/api_examples/r.html index 4ba2d60f3e1..fc51c49aecd 100644 --- a/ckanext/datastore/templates/datastore/api_examples/r.html +++ b/ckanext/datastore/templates/datastore/api_examples/r.html @@ -49,3 +49,6 @@ resp_body_json {% endcall %} + + +{% block custom %}{% endblock %} diff --git a/ckanext/tabledesigner/helpers.py b/ckanext/tabledesigner/helpers.py index e59c2c97c65..46e2169917e 100644 --- a/ckanext/tabledesigner/helpers.py +++ b/ckanext/tabledesigner/helpers.py @@ -1,14 +1,31 @@ - from .column_types import column_types from ckan.plugins.toolkit import _ + def tabledesigner_column_type_options(): """ return list of {'value':..., 'text':...} dicts with the type name and label for all registered column types """ - return [ - {'value': k, 'text': _(v.label)} - for k,v in column_types.items() - ] + return [{"value": k, "text": _(v.label)} for k, v in column_types.items()] + + +def tabledesigner_data_api_examples(resource_id): + return { + "text_column_filters_object": { + "subject": ["watershed", "survey"], + "stage": "active", + }, + "text_column_name_sql": "title", + "insert_record_object": { + "subject": "watershed", + "stage": "active", + }, + "update_record_object": { + "_id": 1, + "subject": "survey", + "stage": "inactive", + }, + "unique_filter_object": {"_id": 1}, + } diff --git a/ckanext/tabledesigner/templates/ajax_snippets/api_info.html b/ckanext/tabledesigner/templates/ajax_snippets/api_info.html index c40c20eac4d..57e21403bd0 100644 --- a/ckanext/tabledesigner/templates/ajax_snippets/api_info.html +++ b/ckanext/tabledesigner/templates/ajax_snippets/api_info.html @@ -1,5 +1,17 @@ {% ckan_extends %} +{% block register_examples %} + {% if h.resource_url_type(resource_id) == 'tabledesigner' %} + {% set examples = h.tabledesigner_data_api_examples(resource_id) %} + + {% for clang in code_languages %} + {% include "/datastore/api_examples/" + clang + ".html" %} + {% endfor %} + {% else %} + {{ super() }} + {% endif %} +{% endblock %} + {% block query_examples %} {% if h.resource_url_type(resource_id) == 'tabledesigner' %}
@@ -10,19 +22,19 @@

{{ _('Get 5 results containing "jones" in any field:') }} {% for clang in code_languages %} - {{ code_examples[clang]['request_limit'] }} + {{ code_examples[clang].request_limit }} {% endfor %} - {{ _('Get results with either "watershed" or "survey" as subject and "active" as its stage:') }} + {{ _('Get results filtered by the contents of specific fields:') }} {% for clang in code_languages %} - {{ code_examples[clang]['request_filter'] }} + {{ code_examples[clang].request_filter_custom }} {% endfor %} {% if h.datastore_search_sql_enabled() %} {{ _('SQL query example:') }} {% for clang in code_languages %} - {{ code_examples[clang]['request_sql'] }} + {{ code_examples[clang].request_sql_custom }} {% endfor %} {% endif %}
@@ -34,7 +46,17 @@

- + {{ _('Insert a new record:') }} + {% for clang in code_languages %} + {{ code_examples[clang].request_insert }} + {% endfor %} + {{ _('Update an existing record:') }} + {% for clang in code_languages %} + {{ code_examples[clang].request_update }} + {% endfor %} + {% trans method="method", upsert="upsert" + %}"{{ method }}" defaults to "{{ upsert }}" i.e. records will be inserted or updated based on the primary key values passed{% + endtrans %}
@@ -44,7 +66,10 @@

- + {{ _('Delete a record:') }} + {% for clang in code_languages %} + {{ code_examples[clang].request_delete }} + {% endfor %}
diff --git a/ckanext/tabledesigner/templates/datastore/api_examples/curl.html b/ckanext/tabledesigner/templates/datastore/api_examples/curl.html new file mode 100644 index 00000000000..18af9431e73 --- /dev/null +++ b/ckanext/tabledesigner/templates/datastore/api_examples/curl.html @@ -0,0 +1,61 @@ +{% ckan_extends %} + +{% block custom %} + {{ super() }} + + {% call register_example('curl', 'request_filter_custom') %} + 
curl {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }} \
+-H"Authorization:$API_TOKEN" -d '
+{
+  "resource_id": "{{ resource_id }}",
+  "filters": {{ examples.text_column_filters_object | tojson(indent=2) | indent(2) }}
+}'
+ {% endcall %} + + {% call register_example('curl', 'request_sql_custom') %} + 
curl {{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }} \
+  -H"Authorization:$API_TOKEN" -d @- <<END
+{
+  "sql": "SELECT * FROM \"{{ resource_id }}\" WHERE {{ examples.text_column_name_sql }} LIKE 'jones'"
+}
+END
+ {% endcall %} + + {% call register_example('curl', 'request_insert') %} + 
curl {{ h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }} \
+-H"Authorization:$API_TOKEN" -d '
+{
+  "resource_id": "{{ resource_id }}",
+  "method": "insert",
+  "records: [
+    {{ examples.insert_record_object | tojson(indent=2) | indent(4) }}
+  ]
+}'
+ {% endcall %} + + {% call register_example('curl', 'request_update') %} + 
curl {{ h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }} \
+-H"Authorization:$API_TOKEN" -d '
+{
+  "resource_id": "{{ resource_id }}",
+  "method": "update",
+  "records: [
+    {{ examples.update_record_object | tojson(indent=2) | indent(4) }}
+  ]
+}'
+ {% endcall %} + + {% call register_example('curl', 'request_delete') %} + 
curl {{ h.url_for('api.action', logic_function='datastore_records_delete', qualified=True) }} \
+-H"Authorization:$API_TOKEN" -d '
+{
+  "resource_id": "{{ resource_id }}",
+  "filters": {{ examples.unique_filter_object | tojson(indent=2) | indent(2) }}
+}'
+ {% endcall %} +{% endblock %} diff --git a/ckanext/tabledesigner/templates/datastore/api_examples/javascript.html b/ckanext/tabledesigner/templates/datastore/api_examples/javascript.html new file mode 100644 index 00000000000..ba4f56ec02c --- /dev/null +++ b/ckanext/tabledesigner/templates/datastore/api_examples/javascript.html @@ -0,0 +1,102 @@ +{% ckan_extends %} + +{% block custom %} + {{ super() }} + + {% macro js_data(obj) -%} + {% for k, v in obj.items() %}{{ k }}: {{ v | tojson }} + {%- if not loop.last %}, +{% endif -%} + {% endfor %} + {%- endmacro %} + + {% call register_example('javascript', 'request_filter_custom') %} +
const resp = await fetch(`{{
+  h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`, {
+    method: 'POST',
+    headers: {
+        'content-type': 'application/json',
+        authorization: API_TOKEN
+    },
+    body: JSON.stringify({
+        resource_id: '{{ resource_id }}',
+        filters: {
+            {{ js_data(examples.text_column_filters_object) | indent(12) }}
+        }
+    })
+await resp.json()
+ {% endcall %} + + {% call register_example('javascript', 'request_sql_custom') %} +
const resp = await fetch(`{{
+  h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}`, {
+    method: 'POST',
+    headers: {
+        'content-type': 'application/json',
+        authorization: API_TOKEN
+    },
+    body: JSON.stringify({
+        sql: `SELECT * FROM "{{ resource_id }}" WHERE {{ examples.text_column_name_sql }} LIKE 'jones'`
+    })
+})
+await resp.json()
+ {% endcall %} + + {% call register_example('javascript', 'request_insert') %} +
const resp = await fetch(`{{
+  h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }}`, {
+    method: 'POST',
+    headers: {
+        'content-type': 'application/json',
+        authorization: API_TOKEN
+    },
+    body: JSON.stringify({
+        resource_id: '{{ resource_id }}',
+        method: "insert",
+        records: [{
+{{ js_data(examples.insert_record_object) | indent(12, first=True) }}
+        }]
+    })
+await resp.json()
+ {% endcall %} + + {% call register_example('javascript', 'request_update') %} +
const resp = await fetch(`{{
+  h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }}`, {
+    method: 'POST',
+    headers: {
+        'content-type': 'application/json',
+        authorization: API_TOKEN
+    },
+    body: JSON.stringify({
+        resource_id: '{{ resource_id }}',
+        method: "update",
+        records: [{
+{{ js_data(examples.update_record_object) | indent(12, first=True) }}
+        }]
+    })
+await resp.json()
+ {% endcall %} + +{% call register_example('javascript', 'request_delete') %} +
const resp = await fetch(`{{
+  h.url_for('api.action', logic_function='datastore_records_delete', qualified=True) }}`, {
+    method: 'POST',
+    headers: {
+        'content-type': 'application/json',
+        authorization: API_TOKEN
+    },
+    body: JSON.stringify({
+        resource_id: '{{ resource_id }}',
+        filters: {
+{{ js_data(examples.unique_filter_object) | indent(12, first=True) }}
+        }
+    })
+await resp.json()
+ {% endcall %} +{% endblock %} diff --git a/ckanext/tabledesigner/templates/datastore/api_examples/powershell.html b/ckanext/tabledesigner/templates/datastore/api_examples/powershell.html new file mode 100644 index 00000000000..5d740d95090 --- /dev/null +++ b/ckanext/tabledesigner/templates/datastore/api_examples/powershell.html @@ -0,0 +1,75 @@ +{% ckan_extends %} + +{% block custom %} + {{ super() }} + + {% call register_example('powershell', 'request_filter_custom') %} +
$json = @'
+{
+  "resource_id": "{{resource_id}}",
+  "filters": {{ examples.text_column_filters_object | tojson(indent=2) | indent(2) }}
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}`
+  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+ {% endcall %} + + {% call register_example('powershell', 'request_sql_custom') %} +
$json = @'
+{
+  "sql": "SELECT * from \"{{resource_id}}\" WHERE {{ examples.text_column_name_sql }} LIKE 'jones'"
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}`
+  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+ {% endcall %} + + {% call register_example('powershell', 'request_insert') %} +
$json = @'
+{
+  "resource_id": "{{resource_id}}",
+  "method": "insert",
+  "records": [
+    {{ examples.insert_record_object | tojson(indent=2) | indent(4) }}
+  ]
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }}`
+  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+ {% endcall %} + + {% call register_example('powershell', 'request_update') %} +
$json = @'
+{
+  "resource_id": "{{resource_id}}",
+  "method": "update",
+  "records": [
+    {{ examples.update_record_object | tojson(indent=2) | indent(4) }}
+  ]
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }}`
+  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+ {% endcall %} + +{% call register_example('powershell', 'request_delete') %} +
$json = @'
+{
+  "resource_id": "{{resource_id}}",
+  "filters": {{ examples.unique_filter_object | tojson(indent=2) | indent(2) }}
+}
+'@
+$response = Invoke-RestMethod {{ h.url_for('api.action', logic_function='datastore_records_delete', qualified=True) }}`
+  -Method Post -Body $json -Headers @{"Authorization"="$API_TOKEN"}
+$response.result.records
+ {% endcall %} +{% endblock %} diff --git a/ckanext/tabledesigner/templates/datastore/api_examples/python.html b/ckanext/tabledesigner/templates/datastore/api_examples/python.html new file mode 100644 index 00000000000..52ec162f30f --- /dev/null +++ b/ckanext/tabledesigner/templates/datastore/api_examples/python.html @@ -0,0 +1,66 @@ +{% ckan_extends %} + +{% block custom %} + {{ super() }} + + {% call register_example('python', 'request_filter_custom') %} +
from ckanapi import RemoteCKAN
+
+ck = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = ck.action.datastore_search(
+    resource_id="{{ resource_id }}",
+    filters={{ examples.text_column_filters_object | tojson(indent=4) | indent(4) }},
+)
+print(result['records'])
+ {% endcall %} + + {% call register_example('python', 'request_sql_custom') %} +
from ckanapi import RemoteCKAN
+
+ck = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+rresult = ck.action.datastore_search_sql(
+    sql="""SELECT * from "{{resource_id}}" WHERE {{ examples.text_column_name_sql }} LIKE 'jones'"""
+)
+print(result['records'])
+ {% endcall %} + + {% call register_example('python', 'request_insert') %} +
from ckanapi import RemoteCKAN
+
+ck = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = ck.action.datastore_upsert(
+    resource_id="{{ resource_id }}",
+    method="insert",
+    records=[{{ examples.insert_record_object | tojson(indent=4) | indent(4) }}]
+)
+
+ {% endcall %} + + {% call register_example('python', 'request_update') %} +
from ckanapi import RemoteCKAN
+
+ck = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = ck.action.datastore_upsert(
+    resource_id="{{ resource_id }}",
+    method="update",
+    records=[{{ examples.update_record_object | tojson(indent=4) | indent(4) }}]
+)
+
+ {% endcall %} + + {% call register_example('python', 'request_delete') %} +
from ckanapi import RemoteCKAN
+
+ck = RemoteCKAN('{{ h.url_for('home.index', qualified=True) }}', apikey=API_TOKEN)
+result = ck.action.datastore_records_delete(
+    resource_id="{{ resource_id }}",
+    filters={{ examples.unique_filter_object | tojson(indent=4) | indent(4) }}
+)
+
+ {% endcall %} +{% endblock %} diff --git a/ckanext/tabledesigner/templates/datastore/api_examples/r.html b/ckanext/tabledesigner/templates/datastore/api_examples/r.html new file mode 100644 index 00000000000..4baeebd6e1b --- /dev/null +++ b/ckanext/tabledesigner/templates/datastore/api_examples/r.html @@ -0,0 +1,93 @@ +{% ckan_extends %} + +{% block custom %} + {{ super() }} + + {% macro r_data(obj) -%} + {% for k, v in obj.items() %}{{ k }} = {% + if v is iterable and v is not string + %}list({% for lv in v %}{{ lv | tojson }}{% + if not loop.last %}, {% endif %}{% endfor %}){% + else %}{{ v | tojson }}{% endif %} + {%- if not loop.last %}, +{% endif -%} + {% endfor %} + {%- endmacro %} + + {% call register_example('r', 'request_filter_custom') %} +
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_search', qualified=True) }}")
+result <- req %>% 
+    req_headers(Authorization = API_TOKEN) %>% 
+    req_body_json(list(
+        resource_id = '{{ resource_id }}', 
+        filters = list(
+            {{ r_data(examples.text_column_filters_object) | indent(12) }})))
+    req_perform %>% 
+    resp_body_json
+ {% endcall %} + + {% call register_example('r', 'request_sql_custom') %} +
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_search_sql', qualified=True) }}")
+result <- req %>% 
+    req_headers(Authorization = API_TOKEN) %>% 
+    req_body_json(list(
+        sql = "SELECT * FROM \"{{resource_id}}\" WHERE {{ examples.text_column_name_sql }} LIKE 'jones'"))
+    req_perform %>% 
+    resp_body_json
+
+{% endcall %} + + {% call register_example('r', 'request_insert') %} +
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }}")
+result <- req %>% 
+    req_headers(Authorization = API_TOKEN) %>% 
+    req_body_json(list(
+        resource_id = '{{ resource_id }}', 
+        method = 'insert',
+        records = list(list(
+            {{ r_data(examples.insert_record_object) | indent(12) }}))))
+    req_perform %>% 
+    resp_body_json
+ {% endcall %} + + {% call register_example('r', 'request_update') %} +
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_upsert', qualified=True) }}")
+result <- req %>% 
+    req_headers(Authorization = API_TOKEN) %>% 
+    req_body_json(list(
+        resource_id = '{{ resource_id }}', 
+        method = 'update',
+        records = list(list(
+            {{ r_data(examples.update_record_object) | indent(12) }}))))
+    req_perform %>% 
+    resp_body_json
+ {% endcall %} + + {% call register_example('r', 'request_delete') %} +
library(httr2)
+
+req <- request("{{ h.url_for('api.action', logic_function='datastore_records_delete', qualified=True) }}")
+result <- req %>% 
+    req_headers(Authorization = API_TOKEN) %>% 
+    req_body_json(list(
+        resource_id = '{{ resource_id }}', 
+        filters = list(
+            {{ r_data(examples.unique_filter_object) | indent(12) }})))
+    req_perform %>% 
+    resp_body_json
+ {% endcall %} + +{% endblock %} From 9effead419ef5fef69dceb95299eadd9df82a4f7 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 6 Jun 2023 14:17:23 +0300 Subject: [PATCH 040/447] feat: Automatically register all factories as fixtures --- changes/7631.feature | 1 + ckan/tests/factories.py | 8 +++- ckan/tests/pytest_ckan/fixtures.py | 70 ++++++++---------------------- 3 files changed, 26 insertions(+), 53 deletions(-) create mode 100644 changes/7631.feature diff --git a/changes/7631.feature b/changes/7631.feature new file mode 100644 index 00000000000..6e7b04b6280 --- /dev/null +++ b/changes/7631.feature @@ -0,0 +1 @@ +`user_with_token` / `sysadmin_with_token` factory fixtures added to test suite. diff --git a/ckan/tests/factories.py b/ckan/tests/factories.py index dc8d65bc788..4686a121c76 100644 --- a/ckan/tests/factories.py +++ b/ckan/tests/factories.py @@ -304,8 +304,8 @@ class Meta: type = "organization" -class Dataset(CKANFactory): - """A factory class for creating CKAN datasets.""" +class Package(CKANFactory): + """A factory class for creating CKAN packages.""" class Meta: model = ckan.model.Package @@ -316,6 +316,10 @@ class Meta: notes = factory.LazyFunction(lambda: fake.text(max_nb_chars=60)) +class Dataset(Package): + """A factory class for creating CKAN datasets.""" + + class Vocabulary(CKANFactory): """A factory class for creating tag vocabularies.""" diff --git a/ckan/tests/pytest_ckan/fixtures.py b/ckan/tests/pytest_ckan/fixtures.py index f7edd1a92be..a95f9410626 100644 --- a/ckan/tests/pytest_ckan/fixtures.py +++ b/ckan/tests/pytest_ckan/fixtures.py @@ -27,14 +27,19 @@ `_ """ +from __future__ import annotations import smtplib from io import BytesIO import copy +import inspect +from typing import Any, Type +from typing_extensions import TypeGuard import pytest import rq +import inflection from werkzeug.datastructures import FileStorage as FlaskFileStorage from pytest_factoryboy import register @@ -49,61 +54,24 @@ from ckan.common import config, aslist -@register -class UserFactory(factories.User): - pass - - -@register -class ResourceFactory(factories.Resource): - pass - - -@register -class ResourceViewFactory(factories.ResourceView): - pass - - -@register -class GroupFactory(factories.Group): - pass - - -@register -class PackageFactory(factories.Dataset): - pass - - -@register -class VocabularyFactory(factories.Vocabulary): - pass - - -@register -class TagFactory(factories.Tag): - pass - - -@register -class SystemInfoFactory(factories.SystemInfo): - pass - - -@register -class APITokenFactory(factories.APIToken): - pass - - -class SysadminFactory(factories.Sysadmin): - pass +def _is_factory_class(m: Any) -> TypeGuard[Type[factories.CKANFactory]]: + """Select classes that can be used as factory fixtures. + """ + return ( + isinstance(m, type) + and issubclass(m, factories.CKANFactory) + and m is not factories.CKANFactory + ) -class OrganizationFactory(factories.Organization): - pass +for (name, factory) in inspect.getmembers(factories, _is_factory_class): + if not name.endswith("Factory"): + name += "Factory" + cls = type(name, (factory, ), {}) -register(SysadminFactory, "sysadmin") -register(OrganizationFactory, "organization") + # strip `Factory` suffix from instance name + register(cls, inflection.underscore(name[:-7])) @pytest.fixture From 1cb8025e8d1382dd71e73e0921a76aa55dc8f234 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 6 Jun 2023 17:47:21 +0300 Subject: [PATCH 041/447] chore: add comments --- ckan/tests/pytest_ckan/fixtures.py | 126 ++++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 12 deletions(-) diff --git a/ckan/tests/pytest_ckan/fixtures.py b/ckan/tests/pytest_ckan/fixtures.py index a95f9410626..534d906df16 100644 --- a/ckan/tests/pytest_ckan/fixtures.py +++ b/ckan/tests/pytest_ckan/fixtures.py @@ -3,11 +3,9 @@ All fixtures below available anywhere under the root of CKAN repository. Any external CKAN extension should be able to include them -by adding next lines under root `conftest.py` +automatically by installing `pytest-ckan` package. -.. literalinclude:: /../conftest.py - -There are three type of fixtures available in CKAN: +There are five types of fixtures available in CKAN: * Fixtures that have some side-effect. They don't return any useful value and generally should be injected via @@ -16,12 +14,25 @@ * Fixtures that provide value. Ex. `app` -* Fixtures that provide factory function. They are rarely needed, so - prefer using 'side-effect' or 'value' fixtures. Main use-case when - one may use function-fixture - late initialization or repeatable - execution(ex.: cleaning database more than once in a single - test). But presence of these fixtures in test usually signals that - is's a good time to refactor this test. +* Fixtures that prvovide a callable for certain task. They are rarely needed, + so prefer using 'side-effect' or 'value' fixtures. Main use-case when one may + use function-fixture - late initialization or repeatable execution(ex.: + cleaning database more than once in a single test). But presence of these + fixtures in test usually signals that is's a good time to refactor this test. + +* Fixtures data-factories. All of them has `*_factory` suffix. They represent + data-factories from ``ckan.tests.factories`` modules. For example, CKAN + registers fixture ``user_factory`` which is basically an alias for + ``ckan.tests.factories.User``, ``dataset_factory`` which is an alias for + ``ckan.tests.factories.Dataset``, etc. + +* Fixtures data-instances. Every ``ENTITY_factory`` fixture has corresponding + ``ENTITY`` fixture. I.e. ``user_factory``/``user``, + ``dataset_factory``/``dataset``. These are shortcuts for creating a single + instance using data-factory. For example, instead of adding ``user_factory`` + to the test and calling it like this: ``user = user_factory()``, it's + possible to include ``user`` fixture to test. This fixture will create an + instance, generated by the corresponding factory-fixture. Deeper explanation can be found in `official documentation `_ @@ -64,14 +75,105 @@ def _is_factory_class(m: Any) -> TypeGuard[Type[factories.CKANFactory]]: ) +# Register every subclass of CKANFactory as a fixture. It gives us two new +# fixtures for each factory. First fixture is created using the factory class +# name converted from camel-case to underscored form(`FirstFixtureClass` -> +# `first_fixture_class`). It's just an alias for original factory +# class(`first_fixture_class is FirstFixtureClass`). Second fixture is created +# by calling the first fixture without arguments(`second_fixture = +# first_fixture_class()`). We have to specify it's name explicitely. Let's take +# a look on artifacts that will be produced by +# `ckan.tests.factories.User`. We'll append `Factory` suffix to this class +# before registering the fixture, that's why first fixture will take the name +# `user_factory`. +# +# +# # user_factory - factory-fixture +# +# it's just an alias for factories.User. Basic usage of factory class looks +# like this:: +# +# from ckan.tests import factories +# def test_user(): +# user = factories.User() +# +# Factory fixture changes it to pytest-way:: +# +# def test_user(user_factory): +# user = user_factory() +# +# Apart from different style, it gives you flexibility of pytest fixtures. You +# can define module-local version of ``user_factory`` or use parametrization. +# +# Because ``user_factory`` is just an alias for the factory class, you still +# can use all the features of factory, as: +# * custom properties: user_factory(name="Test", password="123") +# * get model instead of a dict: user_factory.model(name="Tes") # model.User +# * batch generation: user_factory.create_batch(100) # create 100 users +# +# +# # user - instance-fixture +# +# This is a shortcut for creating an instance using factory. Roughly, this:: +# +# def test_something(user_factory): +# user = user_factory() +# ... +# +# is identical to this:: +# +# def test_something(user): +# ... +# +# You just save one line and 8 characters(``_factory`` suffix). But apart from +# that it's still a pytest fixture. It means that all cool effects of fixtures +# is applicable to it. For example, if there is a fixture that depends on +# ``user``:: +# +# @pytest.fixture() +# def user_from_somewhere(user): +# return user +# +# and a test that requires both ``user`` and this fixture from above, they will +# share the same ``user``:: +# +# def test_identity(user, user_from_somewhere): +# assert user is user_from_somewhere +# +# +# Details about factories and reference to pytest-factoryboy can be found in +# ckan.test.factories module for (name, factory) in inspect.getmembers(factories, _is_factory_class): + # Add `Factory` suffix to the factory class name. Class name will be + # converted into a name for factory-fixture. We want it to be different + # from the name of the instance-fixture. For end-user it will be more + # clear, if factory-fixture use name `something_factory`, while instance + # fixtures will use just `something`. + # + # If we use original class name, HelloWorld class will turn into + # `hello_world` factory-fixture, that will be **an alias** for HelloWorld + # class. It means, we won't be able to use `hello_world` identifier as and + # instance name. We'll have to come up with something different, like + # `hello_world_instance`. It's not so cool, because people want shorter + # names for instances. + # + # That's why we are using `Factory` suffix. It will give us + # `hello_world_factory` fixture and we'll be able to use `hello_world` as a + # name for instance. if not name.endswith("Factory"): name += "Factory" + # Create a new class with the "right" name(suffixed by `Factory`). cls = type(name, (factory, ), {}) - # strip `Factory` suffix from instance name - register(cls, inflection.underscore(name[:-7])) + # this function creates both fixtures + register( + # name of the factory-fixture will be obtained from `cls.__name__` + cls, + # name of instance-fixture must be passed explicitly. Strip `Factory` + # suffix from instance name and convert it to underscore-form. + inflection.underscore(name[:-7]) + ) @pytest.fixture From 27dc5533dfa40ed7d0b3a308008a47d0fba46489 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 12 Jun 2023 18:13:57 -0400 Subject: [PATCH 042/447] [#6118] fix for empty validation rules --- ckanext/tabledesigner/datastore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/tabledesigner/datastore.py b/ckanext/tabledesigner/datastore.py index a08ae685e39..44655a26bb2 100644 --- a/ckanext/tabledesigner/datastore.py +++ b/ckanext/tabledesigner/datastore.py @@ -64,6 +64,6 @@ def create_table(resource_id, info): } for i in info], 'triggers': [ {'function': f'{resource_id}_tabledesigner_validate'} - ] if validate_def else [], + ] if validate_rules else [], } ) From b3f7b74ce3c49336aacf4e58f6d1e37f5b4538bc Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Mon, 12 Jun 2023 18:19:42 -0400 Subject: [PATCH 043/447] [#6118] fix render empty data dictionary --- ckanext/datastore/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/datastore/blueprint.py b/ckanext/datastore/blueprint.py index 5cdad04b7f8..3e7c12b0bb2 100644 --- a/ckanext/datastore/blueprint.py +++ b/ckanext/datastore/blueprint.py @@ -106,7 +106,7 @@ def _prepare(self, id: str, resource_id: str) -> dict[str, Any]: u'pkg_dict': pkg_dict, u'resource': resource, u'fields': [ - f for f in rec[u'fields'] if not f[u'id'].startswith(u'_') + f for f in rec.get('fields', []) if not f[u'id'].startswith(u'_') ] } From a40bf576b7bb58f7198cc87f78c08c312c4a703f Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 13 Jun 2023 16:42:01 -0400 Subject: [PATCH 044/447] [#6118] scheming template compat --- ckanext/tabledesigner/templates/package/resource_read.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ckanext/tabledesigner/templates/package/resource_read.html b/ckanext/tabledesigner/templates/package/resource_read.html index b644d5dc9e8..8b72c89c36c 100644 --- a/ckanext/tabledesigner/templates/package/resource_read.html +++ b/ckanext/tabledesigner/templates/package/resource_read.html @@ -2,7 +2,8 @@ {% import 'macros/form.html' as form %} -{% block resource_additional_information_inner %} +{% block resource_additional_information %} +
{% if res.url_type == 'tabledesigner' and res.datastore_active %} {% block resource_tabledesigner %}
@@ -26,5 +27,6 @@

{{ _('Table Designer') }}

{% endblock %} {% endif %} +
{{ super() }} {% endblock %} From 19c2b5743e0a426d018f89a11764b860469fe82d Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 27 Jun 2023 17:51:03 -0400 Subject: [PATCH 045/447] [#6118] api data examples from actual data --- ckanext/tabledesigner/datastore.py | 2 +- ckanext/tabledesigner/helpers.py | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ckanext/tabledesigner/datastore.py b/ckanext/tabledesigner/datastore.py index 44655a26bb2..56b96e03292 100644 --- a/ckanext/tabledesigner/datastore.py +++ b/ckanext/tabledesigner/datastore.py @@ -56,7 +56,7 @@ def create_table(resource_id, info): 'primary_key': [f for f, typ in primary_key], 'fields': [{ 'id': i['id'], - 'type': i['type'], + 'type': column_types[i['type']].datastore_type, 'info': { k:v for (k, v) in i.items() if k != 'id' and k != 'type' diff --git a/ckanext/tabledesigner/helpers.py b/ckanext/tabledesigner/helpers.py index 46e2169917e..db7094064c3 100644 --- a/ckanext/tabledesigner/helpers.py +++ b/ckanext/tabledesigner/helpers.py @@ -1,6 +1,6 @@ from .column_types import column_types -from ckan.plugins.toolkit import _ +from ckan.plugins.toolkit import _, NotAuthorized, ObjectNotFound, get_action def tabledesigner_column_type_options(): @@ -12,6 +12,31 @@ def tabledesigner_column_type_options(): def tabledesigner_data_api_examples(resource_id): + resp = None + try: + resp = get_action('datastore_search')({}, + { + 'resource_id': resource_id, + 'limit': 1, + } + ) + except (ObjectNotFound, NotAuthorized): + pass + if resp and resp['records']: + record = resp['records'][0] + fields = [f['id'] for f in resp['fields']] + filtr = {k:record[k] for k in fields[1:3]} + txtcols = [f['id'] for f in resp['fields'] if f['type'] == 'text'] + if filtr and txtcols: + return { + "text_column_filters_object": filtr, + "text_column_name_sql": txtcols[0], + "insert_record_object": { + k:v for k,v in record.items() if k != '_id' + }, + "update_record_object": record, + "unique_filter_object": {"_id": 1}, + } return { "text_column_filters_object": { "subject": ["watershed", "survey"], From f4c5f5887cd6780956737f72d3d1e7b39234c172 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 6 Jul 2023 22:27:02 -0400 Subject: [PATCH 046/447] [#7683] datastore_upsert, datastore_create error fixes datastore_upsert method=insert: prevent 500 on invalid data datastore_create datastore_create: invalid data errors now reported against records value (not "message") --- changes/7683.bugfix | 4 ++++ ckanext/datastore/backend/postgres.py | 8 ++------ ckanext/datastore/tests/test_create.py | 2 +- ckanext/datastore/tests/test_upsert.py | 26 ++++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 changes/7683.bugfix diff --git a/changes/7683.bugfix b/changes/7683.bugfix new file mode 100644 index 00000000000..18870a7efdb --- /dev/null +++ b/changes/7683.bugfix @@ -0,0 +1,4 @@ +datastore_upsert, datastore_create error fixes + +datastore_upsert method=insert: prevent 500 on invalid data datastore_create +datastore_create: invalid data errors now reported against records value (not "message") diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index 3e4cc57aa49..b66637a56c5 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -36,7 +36,7 @@ from psycopg2.extras import register_default_json, register_composite import distutils.version from sqlalchemy.exc import (ProgrammingError, IntegrityError, - DBAPIError, DataError) + DBAPIError, DataError, DatabaseError) import ckan.plugins as plugins from ckan.common import CKANConfig, config @@ -1156,11 +1156,7 @@ def upsert_data(context: Context, data_dict: dict[str, Any]): try: context['connection'].execute(sql_string, rows) - except sqlalchemy.exc.DataError as err: - raise InvalidDataError( - toolkit._("The data was invalid: {}" - ).format(_programming_error_summary(err))) - except sqlalchemy.exc.DatabaseError as err: + except (DatabaseError, DataError) as err: raise ValidationError( {u'records': [_programming_error_summary(err)]}) diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 8b9c13ddc9c..fe38bd090fc 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -1193,7 +1193,7 @@ def test_datastore_create_with_invalid_data_value(self, app): assert res_dict["success"] is False assert res_dict["error"]["__type"] == "Validation Error" - assert res_dict["error"]["message"].startswith("The data was invalid") + assert 'invalid input syntax for type "numeric"' in str(res_dict["error"]) diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index dfd3f3b97cd..38478034840 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -780,6 +780,32 @@ def test_empty_string_instead_of_null(self): rec = search_result["records"][0] assert rec == {'_id': 1, 'pk': '1000', 'n': None, 'd': None} + @pytest.mark.ckan_config("ckan.plugins", "datastore") + @pytest.mark.usefixtures("clean_datastore", "with_plugins") + def test_insert_wrong_type(self): + resource = factories.Resource() + data = { + "resource_id": resource["id"], + "force": True, + "fields": [ + {"id": "num", "type": "int"}, + ], + } + helpers.call_action("datastore_create", **data) + + data = { + "resource_id": resource["id"], + "force": True, + "method": "insert", + "records": [ + {"num": "notanumber"} + ], + } + + with pytest.raises(ValidationError) as context: + helpers.call_action("datastore_upsert", **data) + assert u'invalid input syntax for type integer: "notanumber"' in str(context.value) + class TestDatastoreUpdate(object): # Test action 'datastore_upsert' with 'method': 'update' From ce44e28b1eb02b2eb1add219ce301aaf7cd65b26 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 6 Jul 2023 22:37:01 -0400 Subject: [PATCH 047/447] [#7683] remove InvalidDataError and duplicative test --- ckanext/datastore/backend/__init__.py | 10 ------- ckanext/datastore/backend/postgres.py | 8 ------ ckanext/datastore/logic/action.py | 9 ++----- ckanext/datastore/tests/test_db.py | 39 +-------------------------- 4 files changed, 3 insertions(+), 63 deletions(-) diff --git a/ckanext/datastore/backend/__init__.py b/ckanext/datastore/backend/__init__.py index 7710ad82589..af9615294b1 100644 --- a/ckanext/datastore/backend/__init__.py +++ b/ckanext/datastore/backend/__init__.py @@ -53,16 +53,6 @@ class DatastoreException(Exception): pass -class InvalidDataError(Exception): - """Exception that's raised if you try to add invalid data to the datastore. - - For example if you have a column with type "numeric" and then you try to - add a non-numeric value like "foo" to it, this exception should be raised. - - """ - pass - - class DatastoreBackend: """Base class for all datastore backends. diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index b66637a56c5..badc8104dbb 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -46,7 +46,6 @@ DatastoreException, _parse_sort_clause ) -from ckanext.datastore.backend import InvalidDataError log = logging.getLogger(__name__) @@ -1108,11 +1107,6 @@ def alter_table(context: Context, data_dict: dict[str, Any]): def insert_data(context: Context, data_dict: dict[str, Any]): - """ - - :raises InvalidDataError: if there is an invalid value in the given data - - """ data_dict['method'] = _INSERT result = upsert_data(context, data_dict) return result @@ -2003,8 +1997,6 @@ def create(self, context: Context, data_dict: dict[str, Any]): nor can the ordering of them be changed. They can be extended though. Any error results in total failure! For now pass back the actual error. Should be transactional. - :raises InvalidDataError: if there is an invalid value in the given - data ''' engine = get_write_engine() context['connection'] = engine.connect() diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index ddf1f541a81..d5f426cbf31 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -14,9 +14,7 @@ import ckan.plugins as p import ckanext.datastore.logic.schema as dsschema import ckanext.datastore.helpers as datastore_helpers -from ckanext.datastore.backend import ( - DatastoreBackend, InvalidDataError -) +from ckanext.datastore.backend import DatastoreBackend from ckanext.datastore.backend.postgres import identifier log = logging.getLogger(__name__) @@ -154,10 +152,7 @@ def datastore_create(context: Context, data_dict: dict[str, Any]): 'alias': [u'"{0}" is not a valid alias name'.format(alias)] }) - try: - result = backend.create(context, data_dict) - except InvalidDataError as err: - raise p.toolkit.ValidationError({'message': str(err)}) + result = backend.create(context, data_dict) if data_dict.get('calculate_record_count', False): backend.calculate_record_count(data_dict['resource_id']) # type: ignore diff --git a/ckanext/datastore/tests/test_db.py b/ckanext/datastore/tests/test_db.py index 66f7f4625f9..86dc3b09f61 100644 --- a/ckanext/datastore/tests/test_db.py +++ b/ckanext/datastore/tests/test_db.py @@ -10,6 +10,7 @@ import ckan.tests.helpers as helpers import ckanext.datastore.backend as backend import ckanext.datastore.backend.postgres as db +from ckan.logic import ValidationError @pytest.mark.ckan_config("ckan.plugins", "datastore") @@ -128,44 +129,6 @@ def _assert_created_index_on( ) -@mock.patch("ckanext.datastore.backend.postgres._get_fields") -def test_upsert_with_insert_method_and_invalid_data(mock_get_fields_function): - """upsert_data() should raise InvalidDataError if given invalid data. - - If the type of a field is numeric and upsert_data() is given a whitespace - value like " ", it should raise DataError. - - In this case we're testing with "method": "insert" in the data_dict. - - """ - mock_connection = mock.Mock() - mock_connection.execute.side_effect = sqlalchemy.exc.DataError( - "statement", "params", "orig", connection_invalidated=False - ) - - context = {"connection": mock_connection} - data_dict = { - "fields": [{"id": "value", "type": "numeric"}], - "records": [ - {"value": 0}, - {"value": 1}, - {"value": 2}, - {"value": 3}, - {"value": " "}, # Invalid numeric value. - {"value": 5}, - {"value": 6}, - {"value": 7}, - ], - "method": "insert", - "resource_id": "fake-resource-id", - } - - mock_get_fields_function.return_value = data_dict["fields"] - - with pytest.raises(backend.InvalidDataError): - db.upsert_data(context, data_dict) - - class TestGetAllResourcesIdsInDatastore(object): @pytest.mark.ckan_config(u"ckan.plugins", u"datastore") @pytest.mark.usefixtures(u"with_plugins", u"clean_db") From 97ae5933e73b52d7c12e139e9eff7a7ec0174637 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 6 Jul 2023 22:46:22 -0400 Subject: [PATCH 048/447] [#7683] remove unused import --- ckanext/datastore/tests/test_db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckanext/datastore/tests/test_db.py b/ckanext/datastore/tests/test_db.py index 86dc3b09f61..505ba36f9b7 100644 --- a/ckanext/datastore/tests/test_db.py +++ b/ckanext/datastore/tests/test_db.py @@ -2,7 +2,6 @@ import unittest.mock as mock import pytest -import sqlalchemy.exc import ckan.lib.jobs as jobs import ckan.plugins as p From 1f39dcd5a43d93b80f91a502653829863ea8070f Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 6 Jul 2023 23:05:58 -0400 Subject: [PATCH 049/447] [#6118] form validation error display for invalid data --- ckanext/tabledesigner/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckanext/tabledesigner/views.py b/ckanext/tabledesigner/views.py index dd9084d7cce..b9e380ce05e 100644 --- a/ckanext/tabledesigner/views.py +++ b/ckanext/tabledesigner/views.py @@ -91,6 +91,9 @@ def post(self, id, resource_id): errors={ k:[_('Duplicate primary key exists')] for k in pk_fields } + elif rec_err.startswith('invalid input syntax'): + bad_data = rec_err.split('"', 1)[1].rstrip('"') + errors = {f: [_("Invalid input")] for f in row if row[f] == bad_data} elif rec_err.startswith('TAB-DELIMITED\t'): errors = {} erriter = iter(rec_err.split('\t')[1:]) From 9910948e42c9f111de3002bf3d189a9187faf172 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 21 Jul 2023 15:36:04 -0400 Subject: [PATCH 050/447] [#6118] delete rows from datatable view --- .../js/tabledesigner-datatables-buttons.js | 22 +++++++++- .../templates/datatables/datatables_view.html | 7 +++- ckanext/tabledesigner/views.py | 42 +++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/ckanext/tabledesigner/assets/js/tabledesigner-datatables-buttons.js b/ckanext/tabledesigner/assets/js/tabledesigner-datatables-buttons.js index db09ae85ba0..4cbdf5630e8 100644 --- a/ckanext/tabledesigner/assets/js/tabledesigner-datatables-buttons.js +++ b/ckanext/tabledesigner/assets/js/tabledesigner-datatables-buttons.js @@ -1,4 +1,4 @@ -this.ckan.module('tabledesigner_datatables_buttons', function($, _) { +this.ckan.module('tabledesigner_datatables_edit', function($, _) { return { initialize: function() { var defn = $(this)[0].el; @@ -17,3 +17,23 @@ this.ckan.module('tabledesigner_datatables_buttons', function($, _) { } } }); +this.ckan.module('tabledesigner_datatables_delete', function($, _) { + return { + initialize: function() { + var defn = $(this)[0].el; + var editText = defn.data('edit-text'); + var editUrl = defn.data('edit-url'); + + const table = $('#dtprv').DataTable(); + table.button().add(0, { + extend: "selected", + text: editText, + action: function ( e, dt, button, config ){ + var _ids = dt.rows( { selected: true } ).data().map( + e=>encodeURIComponent(e._id)).join('&_id='); + window.parent.location = editUrl + '?_id=' + _ids; + } + }); + } + } +}); diff --git a/ckanext/tabledesigner/templates/datatables/datatables_view.html b/ckanext/tabledesigner/templates/datatables/datatables_view.html index c894d6c7123..d5e2457a5ba 100644 --- a/ckanext/tabledesigner/templates/datatables/datatables_view.html +++ b/ckanext/tabledesigner/templates/datatables/datatables_view.html @@ -3,7 +3,12 @@ {{ super() }} {% if resource.url_type == 'tabledesigner' %}
+
diff --git a/ckanext/tabledesigner/views.py b/ckanext/tabledesigner/views.py index b9e380ce05e..b278c3f9e12 100644 --- a/ckanext/tabledesigner/views.py +++ b/ckanext/tabledesigner/views.py @@ -180,3 +180,45 @@ def post(self, id, resource_id): '/dataset//tabledesigner//edit-row', view_func=_TableDesignerEditRow.as_view('edit_row'), ) + +class _TableDesignerDeleteRows(MethodView): + def get(self, id, resource_id): + _datastore_view = DictionaryView() + data_dict = _datastore_view._prepare(id, resource_id) + _ids = request.params.getlist('_id') + + try: + r = get_action('datastore_search')( + None, { + 'resource_id': resource_id, + 'filters': {'_id': _ids}, + } + ) + except NotAuthorized: + return base.abort(403, _('Not authorized to see this page')) + if len(r['records']) != len(_ids): + return base.abort(404, _('Row(s) not found')) + data_dict['records'] = r['records'] + return render('tabledesigner/delete_rows.html', data_dict) + + def post(self, id, resource_id): + _datastore_view = DictionaryView() + data_dict = _datastore_view._prepare(id, resource_id) + _ids = request.params.getlist('_id') + + try: + r = get_action('datastore_delete')( + None, { + 'resource_id': resource_id, + 'filters': {'_id': _ids}, + 'force': True, # FIXME: don't require this for tabledesigner tables + } + ) + except NotAuthorized: + return base.abort(403, _('Not authorized to see this page')) + return h.redirect_to(data_dict['pkg_dict']['type'] + '_resource.read', id=id, resource_id=resource_id) + +tabledesigner.add_url_rule( + '/dataset//tabledesigner//delete-rows', + view_func=_TableDesignerDeleteRows.as_view('delete_rows'), +) From 481d8824223cc9ed566de218a72879a9f36193a6 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 3 Aug 2023 19:17:37 -0400 Subject: [PATCH 051/447] [#6118] separate tdtype from datastore type --- .../assets/js/tabledesigner-fields.js | 12 +++++---- ckanext/tabledesigner/datastore.py | 10 +++---- ckanext/tabledesigner/helpers.py | 5 ++++ .../datastore/snippets/dictionary_form.html | 26 ++++++++++++------- .../tabledesigner/snippets/design_fields.html | 22 ++++++++++++---- 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/ckanext/tabledesigner/assets/js/tabledesigner-fields.js b/ckanext/tabledesigner/assets/js/tabledesigner-fields.js index 4b38f38e1ae..20cbdcdedf9 100644 --- a/ckanext/tabledesigner/assets/js/tabledesigner-fields.js +++ b/ckanext/tabledesigner/assets/js/tabledesigner-fields.js @@ -3,16 +3,18 @@ this.ckan.module('tabledesigner-fields', function($, _) { initialize: function() { var container = this; var $this = $(this.el); - var $template = $this.children('div[name="tabledesigner-template"]'); - var template = $template.html(); + var $templates = $this.children('div[name="tabledesigner-template"]'); + var templates = Object.fromEntries( + $templates.toArray().map( x => [x.dataset.tdtype, x.outerHTML]) + ); var $add = $this.find('a[name="tabledesigner-add"]'); - $template.remove(); + $templates.remove(); $add.on('click', function(e) { var $last = $this.find('.tabledesigner-field').last(); var group = ($last.data('index') + 1) || 0; - var $copy = $( - template.replace(/TABLEDESIGNER-INDEX0/g, group) + var $copy = $(templates[e.target.dataset.tdtype] + .replace(/TABLEDESIGNER-INDEX0/g, group) .replace(/TABLEDESIGNER-INDEX1/g, group + 1)); $this.find('.tabledesigner-fields-group').append($copy); $copy.hide().show(100); diff --git a/ckanext/tabledesigner/datastore.py b/ckanext/tabledesigner/datastore.py index 56b96e03292..3b1c6b878bd 100644 --- a/ckanext/tabledesigner/datastore.py +++ b/ckanext/tabledesigner/datastore.py @@ -9,11 +9,11 @@ def create_table(resource_id, info): ''' Set up datastore table + validation ''' - primary_key = [(f['id'], f['type']) for f in info if f.get('pk')] + primary_key = [(f['id'], f['tdtype']) for f in info if f.get('pk')] validate_rules = [] - for colname, typ in primary_key: - ct = column_types[typ] + for colname, tdtype in primary_key: + ct = column_types[tdtype] condition = ct.sql_is_empty.format( column=f'NEW.{identifier(colname)}' ) @@ -56,10 +56,10 @@ def create_table(resource_id, info): 'primary_key': [f for f, typ in primary_key], 'fields': [{ 'id': i['id'], - 'type': column_types[i['type']].datastore_type, + 'type': column_types[i['tdtype']].datastore_type, 'info': { k:v for (k, v) in i.items() - if k != 'id' and k != 'type' + if k != 'id' }, } for i in info], 'triggers': [ diff --git a/ckanext/tabledesigner/helpers.py b/ckanext/tabledesigner/helpers.py index db7094064c3..1667dd5aa14 100644 --- a/ckanext/tabledesigner/helpers.py +++ b/ckanext/tabledesigner/helpers.py @@ -10,6 +10,11 @@ def tabledesigner_column_type_options(): """ return [{"value": k, "text": _(v.label)} for k, v in column_types.items()] +def tabledesigner_column_type(tdtype): + """ + return column type object (fall back to text if not found) + """ + return column_types.get(tdtype, column_types['text']) def tabledesigner_data_api_examples(resource_id): resp = None diff --git a/ckanext/tabledesigner/templates/datastore/snippets/dictionary_form.html b/ckanext/tabledesigner/templates/datastore/snippets/dictionary_form.html index 8a109ea4d82..fe6e79d0f77 100644 --- a/ckanext/tabledesigner/templates/datastore/snippets/dictionary_form.html +++ b/ckanext/tabledesigner/templates/datastore/snippets/dictionary_form.html @@ -1,10 +1,14 @@ {% ckan_extends %} {% block field_heading %} - {% if res.url_type != 'tabledesigner' or field.id %} + {% if res.url_type != 'tabledesigner' %} {{ super() }} {% else %} -

{{ _("Field {num}.").format(num=position) }}

+

{{ _("Field {num}.").format(num=position) }} + {{ field.id or '' }} + {{ 0/0 }} + ({{ h.tabledesigner_column_type(field.tdtype or field.type).label }}) +

{% if is_new %}