From f92967f5c79a9d7107f5f366bc8fae2e2e243771 Mon Sep 17 00:00:00 2001 From: jacquelinemorrissette Date: Wed, 24 Feb 2021 22:45:03 -0500 Subject: [PATCH 01/10] Added version information to formList --- onadata/libs/serializers/xform_serializer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index ee48a53da..bd50265e8 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals, print_function, division, absolute_import +import json + from rest_framework import serializers from rest_framework.reverse import reverse @@ -82,7 +84,12 @@ class XFormListSerializer(serializers.Serializer): manifestUrl = serializers.SerializerMethodField('get_manifest_url') def get_version(self, obj): - return None + # Returns version data + # The data returned may vary depending on the contents of the + # version field in the settings of the XLS file when the asset was + # created or updated + obj_json = json.loads(obj.json) + return obj_json.get('version') @check_obj def get_hash(self, obj): From 8326aef0bbfddcc1b046cd4873fb7cb4014128b1 Mon Sep 17 00:00:00 2001 From: jacquelinemorrissette Date: Wed, 3 Mar 2021 20:41:43 -0500 Subject: [PATCH 02/10] Fixed requirements because newest version of twill is not compatible with python2 --- requirements/dev.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.pip b/requirements/dev.pip index 6c677c786..6876e1d83 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -8,7 +8,7 @@ ipdb ipython shell_command statsd -twill +twill==2.0.3 Werkzeug sqlparse pytest==3.10 From cfd4fd57b64cfb9e67a8c86808b2b8202ff8b278 Mon Sep 17 00:00:00 2001 From: JacquelineMorrissette <74968475+JacquelineMorrissette@users.noreply.github.com> Date: Thu, 20 May 2021 13:29:52 -0400 Subject: [PATCH 03/10] Update dev.pip --- requirements/dev.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.pip b/requirements/dev.pip index 6876e1d83..6c677c786 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -8,7 +8,7 @@ ipdb ipython shell_command statsd -twill==2.0.3 +twill Werkzeug sqlparse pytest==3.10 From fe03dcd6ff8a7608689c09cb8f21cfd08284d1b4 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 19:34:25 -0400 Subject: [PATCH 04/10] Upgrade to pyxform 1.5.1 * Recognize select questions from the question elements themselves instead of checking the bind type; see XLSForm/pyxform#168 * Recognize new error message when a group name matches the form name See XLSForm/pyxform#510 * Set `allow_choice_duplicates` to `yes` in cascade test XLSForms See XLSForm/pyxform#23; https://github.com/XLSForm/pyxform/issues/373#issuecomment-580572668 * Update tests to use `default_name` instead of relying on the XLSForm file name See XLSForm/pyxform#130 * Update expected XML in tests to match new pyxform behavior: * `` becomes `` * See XLSForm/pyxform#393 * The bind type for `select` and `select1` becomes `string` * See XLSForm/pyxform#168 * `calculate="concat('uuid:', uuid())"` becomes `jr:preload="uid"` * See XLSForm/pyxform#94 This squashed set of changes includes work from kobotoolbox/kobocat#699. Thanks to @duvld for that contribution. --- .../tests/fixtures/Transportation Form.xml | 24 +++++++++--------- .../forms/contributions/contributions.xml | 2 +- .../bug_fixes/new_cascading_select.xls | Bin 9728 -> 9216 bytes .../new_cascading_select.xls | Bin 9728 -> 9216 bytes .../new_cascading_select.xml | 2 +- .../main/tests/fixtures/exp_line_break.xml | 2 +- .../transportation/Transportation Form.xml | 2 +- .../transportation/transportation.xml | 24 +++++++++--------- .../transportation/transportation2.xml | 2 +- onadata/apps/main/tests/test_past_bugs.py | 2 +- onadata/apps/viewer/pandas_mongo_bridge.py | 9 +++---- .../apps/viewer/tests/test_export_builder.py | 15 +++++++---- onadata/libs/utils/export_tools.py | 5 ++-- requirements/base.pip | 2 +- requirements/travis.pip | 2 +- 15 files changed, 48 insertions(+), 45 deletions(-) diff --git a/onadata/apps/api/tests/fixtures/Transportation Form.xml b/onadata/apps/api/tests/fixtures/Transportation Form.xml index 376923e80..9aa6c7149 100644 --- a/onadata/apps/api/tests/fixtures/Transportation Form.xml +++ b/onadata/apps/api/tests/fixtures/Transportation Form.xml @@ -2,7 +2,7 @@ transportation_2011_07_25 - + @@ -46,18 +46,18 @@ - + - - - - - - - - - - + + + + + + + + + + diff --git a/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml b/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml index 14f205f56..263628b88 100644 --- a/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml +++ b/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml @@ -21,7 +21,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/bug_fixes/new_cascading_select.xls b/onadata/apps/main/tests/fixtures/bug_fixes/new_cascading_select.xls index 8cfd43372ef105c2bbd53db2f79de7ef5b8e081b..3afd4f10f0e6fd2542a8515b227c2374fef277e6 100644 GIT binary patch delta 991 zcmYk5&uddb5XZm!l9!j?zhg|Ospged)Yd9eMG)*I6xxcXA{0qu5-}#&(zI$&NiIEk zd1eLw0S_L$Xvjej^-%l+^dJ#L(33Y0J?No{Gf8nD%e>{YGqc~>dAqHp)`B%FLa)vk zn%*A_2I0=U_Fg<$bG~+d(%c#VfjuF=x^`XVJLSznt@GTtp-q_vdEi2Qd-HC&v4ari zeY|3zcx$~kMx)A{KAA<{{pP^+!q%a5Y-rR|McrC8W1 z*X0ZUtj_^7%Jl)fn33Q7f0}MQf}gj?L~ZT+LRjV_a{*x*U0Xl|5%g>vMHFUSZ4g5YW{sf-Z!7mYnk$6u?)AdMO9tJ)3OrkV<3On5|Bh?OjxU zmNic+X4#yrOqR9eROtAUgG*IB)tOPrxDH zq}GH>eh(*tToIpxXHDx|N=WW4*YeIP?K-F}>%8|C4)$VY^{Ii7JZPE@i#uYM8om@_UvjJMj^`}DLL b*`|2f`Ac8%QZ^wNnUz73LQ delta 1000 zcmYjQOK1~O6g_W}$t2VKKaB~|q)IGk#VTzPL5M<0z*@zH2vKaTffiybZIlL)M6ml# zpSTiq;i@|wh#OIm;z|*8XLs&gcISsA-kVh4%y8b!x%a&}=iPUwRcg&_jEnHg6Ncss z0qFL6y@-8AdnaDGzi3`TA%QK(Z=P+>^z8KIvorP!xt&s6gNB+T?}7`~gvz_&M*FbhI&@fv6!(x_Yro z8H$@^IG<5`2()EWaX)RMom4!40BU~4g9xG+P&`Cc1LZIL+i{p`)^<#YAc7qSk0OeE zTzw#h81f0l<8)YUyAb-Sl~qZdid<4Fq84yTi($pNq)r1|(tJ$i^}fJTw4@6|zD`aB zK{e(M;Y-)$pX6@Y_x~_=hSf?|PB^Tx^3G|MXXX5H#aX%NaLUS-ET*nczU{Y)hD85% zqeG{{1`;?zpmOFAz9YBKQb%1d~uBaQm+jxRD&}@@m%aZEan=b9?!dkPqDxvXyPw&(an4ePc7X eCJM5VYl>_3FB-*1**JJ#XU}zMrO(T4_nW_+-jP%Q diff --git a/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xls b/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xls index 99bb0735156bb280060e3c7682064114fdf6ac91..2ea8cf331fb046ce16b225df18bcccb465f45727 100644 GIT binary patch delta 975 zcmYk5&ubGw6vw}_`LRhhX*NG=LT$Axt*EV0ib_xB5(HcEA{2_WF$q`_Y;9UK_#;8+ zRn|=ZgdRM2(U5~6)kEfWV=UU%dz3(!J7VzS7#$?`YGOP9Ats-QIjqs_h_* ztidZf3U}An3Z?1}Or)c7CU8kW76Z%rWo?Q4v#n2oA0j;p)|g@g26}$QLE57|syKuY%0a~= z7(p(iI81Hr8DT8)k+}eqM%xh(K?EHKM-hb;R~y6-gOyMmr@z^5oe1v>nMx<+q1j(n zt#vAMYwd_i3Lkru?3P(2eLbuS1}dQ~@jyb5zCxoq6SF0qeYC7RWgY zVgXxDr*1EHsp+8X1kTXBKOh%Tf-=}AXf=Ve^c|8QI`^MAZ{{dZz$M?MR^@jyX>dJ! z^=WR+r(cNXSbXq2^rPo-x;g&)))NZx4#I#dH2cY}l5s%`Wc(ud((~0Vl7+}N$aw2t zU0|GMy6U(%H_XFsEz_0vC*ra;VfeDo_YyyEei8DdJ1##?G@6r-=qc4E>teU{m%ie) Ttetx7;R3c~K7CytxZnQ+5aE=} delta 980 zcmYk4PiPZS5XQf^*<_PV()>?Mh$d}`g<7#{6;XsJloYH5J%|v+#u{ibw9rOr5KRPo z@Aio&K@Xn1=|WCYL5e3u@Mvz{JbDnSki?m^`j+tR=J#gyee>o`Yo;~3IVr+#jvJaU z1fbXN_apXs?Y(%bf7QIS0)f|%-#j~>>AC5v7iR5O#%*oPWpK!YwR@}gtBnnWFzn|Q z0}G2QOAFPt4TSM0B9EG z?E8Orouo=&gM_0pHt3vHaW*I%Q_Kb>M?q|0$x`~}sqMjJXh;lxI=XZTf*O{ zyl{kwa|JmilVVNgSFX`AG95lT;(|sIQZFXrGXRq)if={pbRR_Sfbr^~K6^{^DwNsZL9Sbr-!) zoZQ>Lb?(Y=cYgs_Mv#{P diff --git a/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml b/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml index 1962c50d5..bdfd13a37 100644 --- a/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml +++ b/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml @@ -150,7 +150,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/exp_line_break.xml b/onadata/apps/main/tests/fixtures/exp_line_break.xml index a817a9885..a4ed68fa1 100644 --- a/onadata/apps/main/tests/fixtures/exp_line_break.xml +++ b/onadata/apps/main/tests/fixtures/exp_line_break.xml @@ -49,7 +49,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml b/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml index 5288295ec..17b6687bc 100644 --- a/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml +++ b/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml @@ -57,7 +57,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/transportation/transportation.xml b/onadata/apps/main/tests/fixtures/transportation/transportation.xml index 05a7aae24..ed90c375e 100644 --- a/onadata/apps/main/tests/fixtures/transportation/transportation.xml +++ b/onadata/apps/main/tests/fixtures/transportation/transportation.xml @@ -2,7 +2,7 @@ transportation_2011_07_25 - + @@ -46,18 +46,18 @@ - + - - - - - - - - - - + + + + + + + + + + diff --git a/onadata/apps/main/tests/fixtures/transportation/transportation2.xml b/onadata/apps/main/tests/fixtures/transportation/transportation2.xml index aa05548fa..80ba236d0 100644 --- a/onadata/apps/main/tests/fixtures/transportation/transportation2.xml +++ b/onadata/apps/main/tests/fixtures/transportation/transportation2.xml @@ -55,7 +55,7 @@ - + diff --git a/onadata/apps/main/tests/test_past_bugs.py b/onadata/apps/main/tests/test_past_bugs.py index d4166a5c3..6ed0c19a8 100644 --- a/onadata/apps/main/tests/test_past_bugs.py +++ b/onadata/apps/main/tests/test_past_bugs.py @@ -19,7 +19,7 @@ def test_uniqueness_of_group_names_enforced(self): self._create_user_and_login() response = self._publish_xls_file( 'fixtures/group_names_must_be_unique.xls') - message = 'There are two sections with the name group_names_must_be_unique.' + message = 'The name "group_names_must_be_unique" is the same as the form name' response_content = smart_text(response.content) self.assertTrue(message in response_content) self.assertEqual(XForm.objects.count(), pre_count) diff --git a/onadata/apps/viewer/pandas_mongo_bridge.py b/onadata/apps/viewer/pandas_mongo_bridge.py index e279f86c4..1feaaffdb 100644 --- a/onadata/apps/viewer/pandas_mongo_bridge.py +++ b/onadata/apps/viewer/pandas_mongo_bridge.py @@ -13,6 +13,7 @@ except ImportError: from pandas import ExcelWriter +from pyxform.constants import SELECT_ALL_THAT_APPLY from pyxform.survey_element import SurveyElement from pyxform.section import Section, RepeatingSection from pyxform.question import Question @@ -40,8 +41,6 @@ # this is Mongo Collection where we will store the parsed submissions xform_instances = settings.MONGO_DB.instances -# the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = "select" GEOPOINT_BIND_TYPE = "geopoint" # column group delimiters @@ -123,7 +122,7 @@ def _collect_select_multiples(cls, dd): return dict([(e.get_abbreviated_xpath(), [c.get_abbreviated_xpath() for c in e.children]) for e in dd.get_survey_elements() - if e.bind.get("type") == "select"]) + if e.type == SELECT_ALL_THAT_APPLY]) @classmethod def _split_select_multiples(cls, record, select_multiples, @@ -438,9 +437,9 @@ def _build_sections_recursive(self, section_name, element, child_bind_type = child.bind.get("type") if isinstance(child, Question) and not \ question_types_to_exclude(child.type)\ - and not child_bind_type == MULTIPLE_SELECT_BIND_TYPE: + and not child.type == SELECT_ALL_THAT_APPLY: self._add_column_to_section(section_name, child) - elif child_bind_type == MULTIPLE_SELECT_BIND_TYPE: + elif child.type == SELECT_ALL_THAT_APPLY: self.select_multiples[child.get_abbreviated_xpath()] = \ [option.get_abbreviated_xpath() for option in child.children] diff --git a/onadata/apps/viewer/tests/test_export_builder.py b/onadata/apps/viewer/tests/test_export_builder.py index 5f803336e..5744c999a 100644 --- a/onadata/apps/viewer/tests/test_export_builder.py +++ b/onadata/apps/viewer/tests/test_export_builder.py @@ -197,7 +197,8 @@ class TestExportBuilder(TestBase): def _create_childrens_survey(self): return create_survey_from_xls(_logger_fixture_path( - 'childrens_survey.xls')) + 'childrens_survey.xls'), + default_name='childrens_survey') def test_build_sections_from_survey(self): survey = self._create_childrens_survey() @@ -817,7 +818,8 @@ def test_get_valid_sheet_name_catches_long_duplicate_names(self): def test_to_xls_export_generates_valid_sheet_names(self): survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xls')) + 'childrens_survey_with_a_very_long_name.xls'), + default_name='childrens_survey_with_a_very_long_name') export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') @@ -836,7 +838,8 @@ def test_to_xls_export_generates_valid_sheet_names(self): def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xls')) + 'childrens_survey_with_a_very_long_name.xls'), + default_name='childrens_survey_with_a_very_long_name.xls') export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xlsx') @@ -896,7 +899,8 @@ def test_type_conversion(self): } survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xls')) + 'test_data_types/test_data_types.xls'), + default_name='test_data_types/test_data_types') export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export @@ -919,7 +923,8 @@ def test_type_conversion(self): def test_xls_convert_dates_before_1900(self): survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xls')) + 'test_data_types/test_data_types.xls'), + default_name='test_data_types/test_data_types.xls') export_builder = ExportBuilder() export_builder.set_survey(survey) data = [ diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index aafadfdcb..d8fa2cd27 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -20,6 +20,7 @@ from django.utils.text import slugify from openpyxl.date_time import SharedDate from openpyxl.workbook import Workbook +from pyxform.constants import SELECT_ALL_THAT_APPLY from pyxform.question import Question from pyxform.section import Section, RepeatingSection from savReaderWriter import SavWriter @@ -51,8 +52,6 @@ QUESTION_TYPES_TO_EXCLUDE = [ 'note', ] -# the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = "select" GEOPOINT_BIND_TYPE = "geopoint" @@ -265,7 +264,7 @@ def build_sections( {child_xpath: MongoHelper.encode(child_xpath)}) # if its a select multiple, make columns out of its choices - if child.bind.get("type") == MULTIPLE_SELECT_BIND_TYPE\ + if child.type == SELECT_ALL_THAT_APPLY\ and self.SPLIT_SELECT_MULTIPLES: for c in child.children: _xpath = c.get_abbreviated_xpath() diff --git a/requirements/base.pip b/requirements/base.pip index c59f47a28..0c5d98f1b 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -13,7 +13,7 @@ poster==0.8.1 psycopg2-binary==2.7.7 pymongo==3.7.2 lxml==3.4.0 -pyxform==0.13.1 +-e git+https://github.com/jnm/pyxform@v1.5.1-py2-jnm#egg=pyxform django-reversion==2.0.8 xlrd==1.1.0 xlwt==1.3.0 diff --git a/requirements/travis.pip b/requirements/travis.pip index b3778692e..af7cb1b77 100644 --- a/requirements/travis.pip +++ b/requirements/travis.pip @@ -85,7 +85,7 @@ python-digest==1.7 -e git+https://github.com/onaio/python-json2xlsclient.git@5a39387752d819cb6387f75569dbea9a5288aa6f#egg=python_json2xlsclient python-slugify==1.2.6 pytz==2018.9 -pyxform==0.13.1 +-e git+https://github.com/jnm/pyxform@v1.5.1-py2-jnm#egg=pyxform recaptcha-client==1.0.6 redis==3.2.0 requests==2.21.0 From cd0a15152cbf12d87881d2d75ed55423e522756c Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 21:06:02 -0400 Subject: [PATCH 05/10] Revert "Fixed XML comparison" This reverts commit 63abebc59cb5fb86bf3c6ee75d42d96abd7c1431. --- .../api/tests/viewsets/test_xform_list_api.py | 9 +++--- .../api/tests/viewsets/test_xform_viewset.py | 6 ++-- ...016_remove_conflicting_view_permissions.py | 2 +- onadata/apps/main/tests/test_process.py | 6 ++-- onadata/libs/tests/utils/xml.py | 32 ------------------- onadata/libs/utils/common_tags.py | 2 +- 6 files changed, 12 insertions(+), 45 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_list_api.py b/onadata/apps/api/tests/viewsets/test_xform_list_api.py index 733caac60..1f50c06af 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_list_api.py +++ b/onadata/apps/api/tests/viewsets/test_xform_list_api.py @@ -13,7 +13,7 @@ CAN_ADD_SUBMISSIONS, CAN_VIEW_XFORM ) -from onadata.libs.tests.utils.xml import pyxform_version_agnostic, is_equal_xml +from onadata.libs.tests.utils.xml import pyxform_version_agnostic class TestXFormListApi(TestAbstractViewSet): @@ -222,7 +222,7 @@ def test_get_xform_list_with_formid_parameter(self): with open(path) as f: form_list_xml = f.read().strip() data = {"hash": self.xform.hash, "pk": self.xform.pk} - content = response.render().content.decode() + content = response.render().content self.assertEqual(content, form_list_xml % data) def test_retrieve_xform_xml(self): @@ -252,9 +252,8 @@ def test_retrieve_xform_xml(self): form_xml = f.read().strip() data = {"form_uuid": self.xform.uuid} content = smart_str(response.render().content).strip() - - is_equal_xml(pyxform_version_agnostic(content), - pyxform_version_agnostic(form_xml % data)) + self.assertEqual(pyxform_version_agnostic(content), + pyxform_version_agnostic(form_xml % data)) def _load_metadata(self, xform=None): data_value = "screenshot.png" diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 7fd5e4eb9..7757e4968 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -15,7 +15,7 @@ CAN_VIEW_XFORM ) from onadata.libs.serializers.xform_serializer import XFormSerializer -from onadata.libs.tests.utils.xml import pyxform_version_agnostic, is_equal_xml +from onadata.libs.tests.utils.xml import pyxform_version_agnostic class TestXFormViewSet(TestAbstractViewSet): @@ -170,8 +170,8 @@ def test_form_format(self): uuid_node.setAttribute("calculate", "''") # check content without UUID - is_equal_xml(pyxform_version_agnostic(response_doc.toxml()), - pyxform_version_agnostic(expected_doc.toxml())) + self.assertEqual(pyxform_version_agnostic(response_doc.toxml()), + pyxform_version_agnostic(expected_doc.toxml())) def test_form_tags(self): self.publish_xls_form() diff --git a/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py b/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py index d6a0f80ce..2ff1ec21e 100644 --- a/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py +++ b/onadata/apps/logger/migrations/0016_remove_conflicting_view_permissions.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): """ Re-apply parts of `0015_add_delete_data_permission.py` for upgrade of existing installations. - See `0015_add_delete_data_permission.py` for detailsf + See `0015_add_delete_data_permission.py` for details """ dependencies = [ diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index aa4b317ab..fbcee48d2 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -19,7 +19,7 @@ from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.libs.tests.utils.xml import pyxform_version_agnostic, is_equal_xml +from onadata.libs.tests.utils.xml import pyxform_version_agnostic from onadata.libs.utils.common_tags import UUID, SUBMISSION_TIME from .test_base import TestBase @@ -178,8 +178,8 @@ def _download_xform(self): uuid_node.setAttribute("calculate", "''") # check content without UUID - is_equal_xml(pyxform_version_agnostic(response_doc.toxml()), - pyxform_version_agnostic(expected_doc.toxml())) + self.assertEqual(pyxform_version_agnostic(response_doc.toxml()), + pyxform_version_agnostic(expected_doc.toxml())) def _check_csv_export(self): self._check_data_dictionary() diff --git a/onadata/libs/tests/utils/xml.py b/onadata/libs/tests/utils/xml.py index d0e296ec9..ded1ce9c8 100644 --- a/onadata/libs/tests/utils/xml.py +++ b/onadata/libs/tests/utils/xml.py @@ -1,5 +1,4 @@ # coding: utf-8 -from xml.dom import minidom PYXFORM_CHANGED_STRINGS = [ ' jr:preload="uid"', @@ -13,34 +12,3 @@ def pyxform_version_agnostic(xml): for str_ in PYXFORM_CHANGED_STRINGS: xml = xml.replace(str_, '') return xml - - -def is_equal_xml(source: str, target: str) -> bool: - """ - Validates if `source` and `target` are identical whatever the order - of node attributes - - See: https://stackoverflow.com/a/321941/1141214 - """ - - def is_equal_element(a, b): - if a.tagName != b.tagName: - return False - if sorted(a.attributes.items()) != sorted(b.attributes.items()): - return False - if len(a.childNodes) != len(b.childNodes): - return False - for ac, bc in zip(a.childNodes, b.childNodes): - if ac.nodeType != bc.nodeType: - return False - if ac.nodeType == ac.TEXT_NODE and ac.data != bc.data: - return False - if ( - ac.nodeType == ac.ELEMENT_NODE - and not is_equal_element(ac, bc) - ): - return False - return True - - da, db = minidom.parseString(source), minidom.parseString(target) - return is_equal_element(da.documentElement, db.documentElement) diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index 9a5f635c0..17de45dcd 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -19,7 +19,7 @@ END_TIME = "end_time" END = "end" -# value of INSTANCE_DOC_NAME that indicates a registration form +# value of INSTANCE_DOC_NAME that indicates a regisration form REGISTRATION = "registration" # keys that we'll look for in the registration form NAME = "name" From c4998cf9657bc199ae3f381367d3d65a9664b049 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 21:25:15 -0400 Subject: [PATCH 06/10] Remove `pyxform_version_agnostic` test workaround --- .../apps/api/tests/viewsets/test_xform_list_api.py | 4 +--- .../apps/api/tests/viewsets/test_xform_viewset.py | 4 +--- onadata/apps/main/tests/test_process.py | 4 +--- onadata/libs/tests/utils/xml.py | 14 -------------- onadata/libs/utils/common_tags.py | 2 +- 5 files changed, 4 insertions(+), 24 deletions(-) delete mode 100644 onadata/libs/tests/utils/xml.py diff --git a/onadata/apps/api/tests/viewsets/test_xform_list_api.py b/onadata/apps/api/tests/viewsets/test_xform_list_api.py index 1f50c06af..703af27ac 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_list_api.py +++ b/onadata/apps/api/tests/viewsets/test_xform_list_api.py @@ -13,7 +13,6 @@ CAN_ADD_SUBMISSIONS, CAN_VIEW_XFORM ) -from onadata.libs.tests.utils.xml import pyxform_version_agnostic class TestXFormListApi(TestAbstractViewSet): @@ -252,8 +251,7 @@ def test_retrieve_xform_xml(self): form_xml = f.read().strip() data = {"form_uuid": self.xform.uuid} content = smart_str(response.render().content).strip() - self.assertEqual(pyxform_version_agnostic(content), - pyxform_version_agnostic(form_xml % data)) + self.assertEqual(content, form_xml % data) def _load_metadata(self, xform=None): data_value = "screenshot.png" diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 7757e4968..c48c6f30a 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -15,7 +15,6 @@ CAN_VIEW_XFORM ) from onadata.libs.serializers.xform_serializer import XFormSerializer -from onadata.libs.tests.utils.xml import pyxform_version_agnostic class TestXFormViewSet(TestAbstractViewSet): @@ -170,8 +169,7 @@ def test_form_format(self): uuid_node.setAttribute("calculate", "''") # check content without UUID - self.assertEqual(pyxform_version_agnostic(response_doc.toxml()), - pyxform_version_agnostic(expected_doc.toxml())) + self.assertEqual(response_doc.toxml(), expected_doc.toxml()) def test_form_tags(self): self.publish_xls_form() diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index fbcee48d2..cec2c0363 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -19,7 +19,6 @@ from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.viewer.models.data_dictionary import DataDictionary -from onadata.libs.tests.utils.xml import pyxform_version_agnostic from onadata.libs.utils.common_tags import UUID, SUBMISSION_TIME from .test_base import TestBase @@ -178,8 +177,7 @@ def _download_xform(self): uuid_node.setAttribute("calculate", "''") # check content without UUID - self.assertEqual(pyxform_version_agnostic(response_doc.toxml()), - pyxform_version_agnostic(expected_doc.toxml())) + self.assertEqual(response_doc.toxml(), expected_doc.toxml()) def _check_csv_export(self): self._check_data_dictionary() diff --git a/onadata/libs/tests/utils/xml.py b/onadata/libs/tests/utils/xml.py deleted file mode 100644 index ded1ce9c8..000000000 --- a/onadata/libs/tests/utils/xml.py +++ /dev/null @@ -1,14 +0,0 @@ -# coding: utf-8 - -PYXFORM_CHANGED_STRINGS = [ - ' jr:preload="uid"', - ''' calculate="concat('uuid:', uuid())"''', - # ' xmlns:odk="http://www.opendatakit.org/xforms"', - # '''GPS coordinates can only be collected when outside.''', -] - - -def pyxform_version_agnostic(xml): - for str_ in PYXFORM_CHANGED_STRINGS: - xml = xml.replace(str_, '') - return xml diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index 17de45dcd..9a5f635c0 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -19,7 +19,7 @@ END_TIME = "end_time" END = "end" -# value of INSTANCE_DOC_NAME that indicates a regisration form +# value of INSTANCE_DOC_NAME that indicates a registration form REGISTRATION = "registration" # keys that we'll look for in the registration form NAME = "name" From db7bda8977ea1b74bb482c6d21204dc4ba467205 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 22:19:36 -0400 Subject: [PATCH 07/10] Add a shell script shortcut for pip-compile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …to avoid running pip-compile three times manually whenever there's a Python dependency change --- pip-compile.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 pip-compile.sh diff --git a/pip-compile.sh b/pip-compile.sh new file mode 100755 index 000000000..8d1fd9d31 --- /dev/null +++ b/pip-compile.sh @@ -0,0 +1,7 @@ +#!/bin/bash +for in_file in dependencies/pip/*.in +do + # pass any arguments to pip-compile + # useful for switches like `--upgrade-package` + pip-compile "$@" "$in_file" +done From 09e7da8c52cc0ffa1237036fbbdb4833f9e80b1a Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 23:23:20 -0400 Subject: [PATCH 08/10] Update pyxform requirement to 1.5.1 Includes a humongous diff thanks to jazzband/pip-tools#1237, which places each "via" package on a separate line --- dependencies/pip/dev.txt | 481 ++++++++++++++++++++++-------- dependencies/pip/prod.txt | 377 ++++++++++++++++------- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 374 ++++++++++++++++------- 4 files changed, 902 insertions(+), 332 deletions(-) diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index 82740d4ab..17fb248ad 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -4,136 +4,357 @@ # # pip-compile dependencies/pip/dev.in # --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # via -r dependencies/pip/requirements.in --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==2.6.0 # via -r dependencies/pip/requirements.in, kombu -argparse==1.4.0 # via unittest2 -attrs==19.3.0 # via jsonschema, pytest -backcall==0.2.0 # via ipython -backports.csv==1.0.7 # via formpack -begins==0.9 # via formpack -billiard==3.6.3.0 # via celery -boto3==1.14.44 # via -r dependencies/pip/requirements.in -botocore==1.17.44 # via boto3, s3transfer -celery[redis]==4.4.6 # via -r dependencies/pip/requirements.in, django-celery-beat -certifi==2020.6.20 # via requests, sentry-sdk -chardet==3.0.4 # via requests -cssselect==1.1.0 # via pyquery -decorator==4.4.2 # via ipython, traitlets -defusedxml==0.6.0 # via djangorestframework-xml -dict2xml==1.7.0 # via -r dependencies/pip/requirements.in -dj-database-url==0.5.0 # via -r dependencies/pip/requirements.in -django-celery-beat==2.0.0 # via -r dependencies/pip/requirements.in -django-cors-headers==3.4.0 # via -r dependencies/pip/requirements.in -django-db-readonly==0.6.0 # via -r dependencies/pip/requirements.in -django-extensions==3.0.3 # via -r dependencies/pip/requirements.in -django-filter==2.3.0 # via -r dependencies/pip/requirements.in -django-guardian==2.3.0 # via -r dependencies/pip/requirements.in, djangorestframework-guardian -django-nose==1.4.7 # via -r dependencies/pip/dev.in -django-oauth-toolkit==1.3.2 # via -r dependencies/pip/requirements.in -django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in -django-registration-redux==2.8 # via -r dependencies/pip/requirements.in -django-render-block==0.7 # via django-templated-email -django-reversion==3.0.1 # via -r dependencies/pip/requirements.in -django-taggit==1.3.0 # via -r dependencies/pip/requirements.in -django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in -django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield -djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in -djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in -djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in -djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -djangorestframework==3.11.0 # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian -docutils==0.15.2 # via botocore, statistics -dpath==2.0.1 # via -r dependencies/pip/requirements.in -ecdsa==0.15 # via tlslite-ng -elaphe3==0.2.0 # via -r dependencies/pip/requirements.in -et-xmlfile==1.0.1 # via openpyxl -formencode==1.3.1 # via pyxform -git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack # via -r dependencies/pip/requirements.in -future==0.18.2 # via celery -gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -geojson-rewind==0.2.0 # via formpack -gitdb==4.0.5 # via gitpython -gitpython==3.1.7 # via transifex-client -httmock==1.3.0 # via -r dependencies/pip/dev.in -idna==2.10 # via requests -ipdb==0.13.3 # via -r dependencies/pip/dev.in -ipython-genutils==0.2.0 # via traitlets -ipython==7.16.1 # via -r dependencies/pip/dev.in, ipdb -jdcal==1.4.1 # via openpyxl -jedi==0.17.2 # via ipython -jmespath==0.10.0 # via boto3, botocore -jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -jsonschema==3.2.0 # via formpack -kombu==4.6.11 # via celery -linecache2==1.0.0 # via traceback2 -lxml==4.5.2 # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery -markdown==3.2.2 # via -r dependencies/pip/requirements.in -mock==4.0.2 # via -r dependencies/pip/dev.in -modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -mongomock==3.19.0 # via -r dependencies/pip/dev.in -more-itertools==8.4.0 # via pytest -nose==1.3.7 # via django-nose -numpy==1.19.0 # via pandas -oauthlib==3.1.0 # via django-oauth-toolkit -openpyxl==3.0.4 # via -r dependencies/pip/requirements.in -packaging==20.4 # via pytest -pandas==1.0.5 # via -r dependencies/pip/requirements.in -parso==0.7.0 # via jedi -path.py==12.4.0 # via formpack -path==13.1.0 # via path.py -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 -pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.5 # via ipython -psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in -ptyprocess==0.6.0 # via pexpect -py==1.9.0 # via pytest -pygments==2.6.1 # via ipython -pymongo==3.10.1 # via -r dependencies/pip/requirements.in -pyparsing==2.4.7 # via packaging -pyquery==1.4.1 # via formpack -pyrsistent==0.16.0 # via jsonschema -pytest-django==3.9.0 # via -r dependencies/pip/dev.in -pytest-env==0.6.2 # via -r dependencies/pip/dev.in -pytest==5.4.3 # via -r dependencies/pip/dev.in, pytest-django, pytest-env -python-crontab==2.5.1 # via django-celery-beat -python-dateutil==2.8.1 # via botocore, pandas, python-crontab -python-slugify==4.0.1 # via transifex-client -pytz==2020.1 # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas -pyxform==0.15.1 # via -r dependencies/pip/requirements.in, formpack -redis==3.5.3 # via -r dependencies/pip/requirements.in, celery, django-redis-sessions -requests==2.24.0 # via django-oauth-toolkit, httmock, transifex-client -s3transfer==0.3.3 # via boto3 -https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # via -r dependencies/pip/requirements.in -sentinels==1.0.0 # via mongomock -sentry-sdk==0.16.5 # via -r dependencies/pip/requirements.in -shell-command==0.1 # via -r dependencies/pip/dev.in -simplejson==3.17.2 # via -r dependencies/pip/requirements.in -six==1.15.0 # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, mongomock, packaging, pyrsistent, python-dateutil, ssrf-protect, traitlets, transifex-client, unittest2 -smmap==3.0.4 # via gitdb -sqlparse==0.3.1 # via -r dependencies/pip/dev.in, django -statistics==1.0.3.5 # via formpack -text-unidecode==1.3 # via python-slugify -tlslite-ng==0.7.5 # via gdata-python3 -traceback2==1.4.0 # via unittest2 -traitlets==4.3.3 # via ipython -transifex-client==0.13.11 # via -r dependencies/pip/requirements.in -unicodecsv==0.14.1 # via djangorestframework-csv, pyxform -unittest2==1.1.0 # via pyxform -urllib3==1.25.9 # via botocore, requests, sentry-sdk, transifex-client -vine==1.3.0 # via amqp, celery -wcwidth==0.2.5 # via prompt-toolkit, pytest -werkzeug==1.0.1 # via -r dependencies/pip/dev.in -xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform -xlsxwriter==1.2.9 # via formpack -xlwt==1.3.0 # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest + # via -r dependencies/pip/requirements.in +-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages + # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest + # via -r dependencies/pip/requirements.in +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client + # via -r dependencies/pip/requirements.in +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect + # via -r dependencies/pip/requirements.in +amqp==2.6.0 + # via + # -r dependencies/pip/requirements.in + # kombu +argparse==1.4.0 + # via unittest2 +attrs==19.3.0 + # via + # jsonschema + # pytest +backcall==0.2.0 + # via ipython +backports.csv==1.0.7 + # via formpack +begins==0.9 + # via formpack +billiard==3.6.3.0 + # via celery +boto3==1.14.44 + # via -r dependencies/pip/requirements.in +botocore==1.17.44 + # via + # boto3 + # s3transfer +celery[redis]==4.4.6 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat +certifi==2020.6.20 + # via + # requests + # sentry-sdk +chardet==3.0.4 + # via requests +cssselect==1.1.0 + # via pyquery +decorator==4.4.2 + # via + # ipython + # traitlets +defusedxml==0.6.0 + # via djangorestframework-xml +dict2xml==1.7.0 + # via -r dependencies/pip/requirements.in +dj-database-url==0.5.0 + # via -r dependencies/pip/requirements.in +django-celery-beat==2.0.0 + # via -r dependencies/pip/requirements.in +django-cors-headers==3.4.0 + # via -r dependencies/pip/requirements.in +django-db-readonly==0.6.0 + # via -r dependencies/pip/requirements.in +django-extensions==3.0.3 + # via -r dependencies/pip/requirements.in +django-filter==2.3.0 + # via -r dependencies/pip/requirements.in +django-guardian==2.3.0 + # via + # -r dependencies/pip/requirements.in + # djangorestframework-guardian +django-nose==1.4.7 + # via -r dependencies/pip/dev.in +django-oauth-toolkit==1.3.2 + # via -r dependencies/pip/requirements.in +django-pure-pagination==0.3.0 + # via -r dependencies/pip/requirements.in +django-redis-sessions==0.6.1 + # via -r dependencies/pip/requirements.in +django-registration-redux==2.8 + # via -r dependencies/pip/requirements.in +django-render-block==0.7 + # via django-templated-email +django-reversion==3.0.1 + # via -r dependencies/pip/requirements.in +django-taggit==1.3.0 + # via -r dependencies/pip/requirements.in +django-templated-email==2.3.0 + # via -r dependencies/pip/requirements.in +django-timezone-field==4.0 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat +django==2.2.14 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat + # django-cors-headers + # django-filter + # django-guardian + # django-oauth-toolkit + # django-render-block + # django-reversion + # django-storages + # django-taggit + # django-timezone-field + # djangorestframework + # djangorestframework-guardian + # jsonfield +djangorestframework-csv==2.1.0 + # via -r dependencies/pip/requirements.in +djangorestframework-guardian==0.3.0 + # via -r dependencies/pip/requirements.in +djangorestframework-jsonp==1.0.2 + # via -r dependencies/pip/requirements.in +djangorestframework-xml==2.0.0 + # via -r dependencies/pip/requirements.in +djangorestframework==3.11.0 + # via + # -r dependencies/pip/requirements.in + # djangorestframework-csv + # djangorestframework-guardian +docutils==0.15.2 + # via + # botocore + # statistics +dpath==2.0.1 + # via -r dependencies/pip/requirements.in +ecdsa==0.15 + # via tlslite-ng +elaphe3==0.2.0 + # via -r dependencies/pip/requirements.in +et-xmlfile==1.0.1 + # via openpyxl +formencode==1.3.1 + # via pyxform +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack + # via -r dependencies/pip/requirements.in +future==0.18.2 + # via celery +gdata-python3==3.0.1 + # via -r dependencies/pip/requirements.in +geojson-rewind==0.2.0 + # via formpack +gitdb==4.0.5 + # via gitpython +gitpython==3.1.7 + # via transifex-client +httmock==1.3.0 + # via -r dependencies/pip/dev.in +idna==2.10 + # via requests +ipdb==0.13.3 + # via -r dependencies/pip/dev.in +ipython-genutils==0.2.0 + # via traitlets +ipython==7.16.1 + # via + # -r dependencies/pip/dev.in + # ipdb +jdcal==1.4.1 + # via openpyxl +jedi==0.17.2 + # via ipython +jmespath==0.10.0 + # via + # boto3 + # botocore +jsonfield==3.1.0 + # via -r dependencies/pip/requirements.in +jsonschema==3.2.0 + # via formpack +kombu==4.6.11 + # via celery +linecache2==1.0.0 + # via traceback2 +lxml==4.5.2 + # via + # -r dependencies/pip/requirements.in + # formpack + # gdata-python3 + # pyquery +markdown==3.2.2 + # via -r dependencies/pip/requirements.in +mock==4.0.2 + # via -r dependencies/pip/dev.in +modilabs-python-utils==0.1.5 + # via -r dependencies/pip/requirements.in +mongomock==3.19.0 + # via -r dependencies/pip/dev.in +more-itertools==8.4.0 + # via pytest +nose==1.3.7 + # via django-nose +numpy==1.19.0 + # via pandas +oauthlib==3.1.0 + # via django-oauth-toolkit +openpyxl==3.0.4 + # via -r dependencies/pip/requirements.in +packaging==20.4 + # via pytest +pandas==1.0.5 + # via -r dependencies/pip/requirements.in +parso==0.7.0 + # via jedi +path.py==12.4.0 + # via formpack +path==13.1.0 + # via path.py +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==7.2.0 + # via + # -r dependencies/pip/requirements.in + # elaphe3 +pluggy==0.13.1 + # via pytest +prompt-toolkit==3.0.5 + # via ipython +psycopg2-binary==2.8.5 + # via -r dependencies/pip/requirements.in +ptyprocess==0.6.0 + # via pexpect +py==1.9.0 + # via pytest +pygments==2.6.1 + # via ipython +pymongo==3.10.1 + # via -r dependencies/pip/requirements.in +pyparsing==2.4.7 + # via packaging +pyquery==1.4.1 + # via formpack +pyrsistent==0.16.0 + # via jsonschema +pytest-django==3.9.0 + # via -r dependencies/pip/dev.in +pytest-env==0.6.2 + # via -r dependencies/pip/dev.in +pytest==5.4.3 + # via + # -r dependencies/pip/dev.in + # pytest-django + # pytest-env +python-crontab==2.5.1 + # via django-celery-beat +python-dateutil==2.8.1 + # via + # botocore + # pandas + # python-crontab +python-slugify==4.0.1 + # via transifex-client +pytz==2020.1 + # via + # -r dependencies/pip/requirements.in + # celery + # django + # django-timezone-field + # pandas +pyxform==1.5.1 + # via + # -r dependencies/pip/requirements.in + # formpack +redis==3.5.3 + # via + # -r dependencies/pip/requirements.in + # celery + # django-redis-sessions +requests==2.24.0 + # via + # django-oauth-toolkit + # httmock + # transifex-client +s3transfer==0.3.3 + # via boto3 +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter + # via -r dependencies/pip/requirements.in +sentinels==1.0.0 + # via mongomock +sentry-sdk==0.16.5 + # via -r dependencies/pip/requirements.in +shell-command==0.1 + # via -r dependencies/pip/dev.in +simplejson==3.17.2 + # via -r dependencies/pip/requirements.in +six==1.15.0 + # via + # django-extensions + # django-templated-email + # djangorestframework-csv + # ecdsa + # jsonschema + # mongomock + # packaging + # pyrsistent + # python-dateutil + # ssrf-protect + # traitlets + # transifex-client + # unittest2 +smmap==3.0.4 + # via gitdb +sqlparse==0.3.1 + # via + # -r dependencies/pip/dev.in + # django +statistics==1.0.3.5 + # via formpack +text-unidecode==1.3 + # via python-slugify +tlslite-ng==0.7.5 + # via gdata-python3 +traceback2==1.4.0 + # via unittest2 +traitlets==4.3.3 + # via ipython +transifex-client==0.13.11 + # via -r dependencies/pip/requirements.in +unicodecsv==0.14.1 + # via + # djangorestframework-csv + # pyxform +unittest2==1.1.0 + # via pyxform +urllib3==1.25.9 + # via + # botocore + # requests + # sentry-sdk + # transifex-client +vine==1.3.0 + # via + # amqp + # celery +wcwidth==0.2.5 + # via + # prompt-toolkit + # pytest +werkzeug==1.0.1 + # via -r dependencies/pip/dev.in +xlrd==1.2.0 + # via + # -r dependencies/pip/requirements.in + # pyxform +xlsxwriter==1.2.9 + # via formpack +xlwt==1.3.0 + # via -r dependencies/pip/requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index ac5903c12..b1f7022ff 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -4,107 +4,282 @@ # # pip-compile dependencies/pip/prod.in # --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # via -r dependencies/pip/requirements.in --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==2.6.0 # via -r dependencies/pip/requirements.in, kombu -argparse==1.4.0 # via unittest2 -attrs==19.3.0 # via jsonschema -backports.csv==1.0.7 # via formpack -begins==0.9 # via formpack -billiard==3.6.3.0 # via celery -boto3==1.14.44 # via -r dependencies/pip/requirements.in -botocore==1.17.44 # via boto3, s3transfer -celery[redis]==4.4.6 # via -r dependencies/pip/requirements.in, django-celery-beat -certifi==2020.6.20 # via requests, sentry-sdk -chardet==3.0.4 # via requests -cssselect==1.1.0 # via pyquery -defusedxml==0.6.0 # via djangorestframework-xml -dict2xml==1.7.0 # via -r dependencies/pip/requirements.in -dj-database-url==0.5.0 # via -r dependencies/pip/requirements.in -django-celery-beat==2.0.0 # via -r dependencies/pip/requirements.in -django-cors-headers==3.4.0 # via -r dependencies/pip/requirements.in -django-db-readonly==0.6.0 # via -r dependencies/pip/requirements.in -django-extensions==3.0.3 # via -r dependencies/pip/requirements.in -django-filter==2.3.0 # via -r dependencies/pip/requirements.in -django-guardian==2.3.0 # via -r dependencies/pip/requirements.in, djangorestframework-guardian -django-oauth-toolkit==1.3.2 # via -r dependencies/pip/requirements.in -django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in -django-registration-redux==2.8 # via -r dependencies/pip/requirements.in -django-render-block==0.7 # via django-templated-email -django-reversion==3.0.1 # via -r dependencies/pip/requirements.in -django-taggit==1.3.0 # via -r dependencies/pip/requirements.in -django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in -django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield -djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in -djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in -djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in -djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -djangorestframework==3.11.0 # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian -docutils==0.15.2 # via botocore, statistics -dpath==2.0.1 # via -r dependencies/pip/requirements.in -ecdsa==0.15 # via tlslite-ng -elaphe3==0.2.0 # via -r dependencies/pip/requirements.in -et-xmlfile==1.0.1 # via openpyxl -formencode==1.3.1 # via pyxform -git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack # via -r dependencies/pip/requirements.in -future==0.18.2 # via celery -gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -geojson-rewind==0.2.0 # via formpack -gitdb==4.0.5 # via gitpython -gitpython==3.1.7 # via transifex-client -idna==2.10 # via requests -jdcal==1.4.1 # via openpyxl -jmespath==0.10.0 # via boto3, botocore -jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -jsonschema==3.2.0 # via formpack -kombu==4.6.11 # via celery -linecache2==1.0.0 # via traceback2 -lxml==4.5.2 # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery -markdown==3.2.2 # via -r dependencies/pip/requirements.in -modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -numpy==1.19.0 # via pandas -oauthlib==3.1.0 # via django-oauth-toolkit -openpyxl==3.0.4 # via -r dependencies/pip/requirements.in -pandas==1.0.5 # via -r dependencies/pip/requirements.in -path.py==12.5.0 # via formpack -path==15.0.0 # via path.py -pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 -psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in -pymongo==3.10.1 # via -r dependencies/pip/requirements.in -pyquery==1.4.1 # via formpack -pyrsistent==0.16.0 # via jsonschema -python-crontab==2.5.1 # via django-celery-beat -python-dateutil==2.8.1 # via botocore, pandas, python-crontab -python-slugify==4.0.1 # via transifex-client -pytz==2020.1 # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas -pyxform==0.15.1 # via -r dependencies/pip/requirements.in, formpack -redis==3.5.3 # via -r dependencies/pip/requirements.in, celery, django-redis-sessions -requests==2.24.0 # via django-oauth-toolkit, transifex-client -s3transfer==0.3.3 # via boto3 -https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # via -r dependencies/pip/requirements.in -sentry-sdk==0.16.5 # via -r dependencies/pip/requirements.in -simplejson==3.17.2 # via -r dependencies/pip/requirements.in -six==1.15.0 # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, pyrsistent, python-dateutil, ssrf-protect, transifex-client, unittest2 -smmap==3.0.4 # via gitdb -sqlparse==0.3.1 # via django -statistics==1.0.3.5 # via formpack -text-unidecode==1.3 # via python-slugify -tlslite-ng==0.7.5 # via gdata-python3 -traceback2==1.4.0 # via unittest2 -transifex-client==0.13.11 # via -r dependencies/pip/requirements.in -unicodecsv==0.14.1 # via djangorestframework-csv, pyxform -unittest2==1.1.0 # via pyxform -urllib3==1.25.9 # via botocore, requests, sentry-sdk, transifex-client -uwsgi==2.0.19.1 # via -r dependencies/pip/prod.in -vine==1.3.0 # via amqp, celery -xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform -xlsxwriter==1.2.9 # via formpack -xlwt==1.3.0 # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest + # via -r dependencies/pip/requirements.in +-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages + # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest + # via -r dependencies/pip/requirements.in +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client + # via -r dependencies/pip/requirements.in +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect + # via -r dependencies/pip/requirements.in +amqp==2.6.0 + # via + # -r dependencies/pip/requirements.in + # kombu +argparse==1.4.0 + # via unittest2 +attrs==19.3.0 + # via jsonschema +backports.csv==1.0.7 + # via formpack +begins==0.9 + # via formpack +billiard==3.6.3.0 + # via celery +boto3==1.14.44 + # via -r dependencies/pip/requirements.in +botocore==1.17.44 + # via + # boto3 + # s3transfer +celery[redis]==4.4.6 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat +certifi==2020.6.20 + # via + # requests + # sentry-sdk +chardet==3.0.4 + # via requests +cssselect==1.1.0 + # via pyquery +defusedxml==0.6.0 + # via djangorestframework-xml +dict2xml==1.7.0 + # via -r dependencies/pip/requirements.in +dj-database-url==0.5.0 + # via -r dependencies/pip/requirements.in +django-celery-beat==2.0.0 + # via -r dependencies/pip/requirements.in +django-cors-headers==3.4.0 + # via -r dependencies/pip/requirements.in +django-db-readonly==0.6.0 + # via -r dependencies/pip/requirements.in +django-extensions==3.0.3 + # via -r dependencies/pip/requirements.in +django-filter==2.3.0 + # via -r dependencies/pip/requirements.in +django-guardian==2.3.0 + # via + # -r dependencies/pip/requirements.in + # djangorestframework-guardian +django-oauth-toolkit==1.3.2 + # via -r dependencies/pip/requirements.in +django-pure-pagination==0.3.0 + # via -r dependencies/pip/requirements.in +django-redis-sessions==0.6.1 + # via -r dependencies/pip/requirements.in +django-registration-redux==2.8 + # via -r dependencies/pip/requirements.in +django-render-block==0.7 + # via django-templated-email +django-reversion==3.0.1 + # via -r dependencies/pip/requirements.in +django-taggit==1.3.0 + # via -r dependencies/pip/requirements.in +django-templated-email==2.3.0 + # via -r dependencies/pip/requirements.in +django-timezone-field==4.0 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat +django==2.2.14 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat + # django-cors-headers + # django-filter + # django-guardian + # django-oauth-toolkit + # django-render-block + # django-reversion + # django-storages + # django-taggit + # django-timezone-field + # djangorestframework + # djangorestframework-guardian + # jsonfield +djangorestframework-csv==2.1.0 + # via -r dependencies/pip/requirements.in +djangorestframework-guardian==0.3.0 + # via -r dependencies/pip/requirements.in +djangorestframework-jsonp==1.0.2 + # via -r dependencies/pip/requirements.in +djangorestframework-xml==2.0.0 + # via -r dependencies/pip/requirements.in +djangorestframework==3.11.0 + # via + # -r dependencies/pip/requirements.in + # djangorestframework-csv + # djangorestframework-guardian +docutils==0.15.2 + # via + # botocore + # statistics +dpath==2.0.1 + # via -r dependencies/pip/requirements.in +ecdsa==0.15 + # via tlslite-ng +elaphe3==0.2.0 + # via -r dependencies/pip/requirements.in +et-xmlfile==1.0.1 + # via openpyxl +formencode==1.3.1 + # via pyxform +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack + # via -r dependencies/pip/requirements.in +future==0.18.2 + # via celery +gdata-python3==3.0.1 + # via -r dependencies/pip/requirements.in +geojson-rewind==0.2.0 + # via formpack +gitdb==4.0.5 + # via gitpython +gitpython==3.1.7 + # via transifex-client +idna==2.10 + # via requests +jdcal==1.4.1 + # via openpyxl +jmespath==0.10.0 + # via + # boto3 + # botocore +jsonfield==3.1.0 + # via -r dependencies/pip/requirements.in +jsonschema==3.2.0 + # via formpack +kombu==4.6.11 + # via celery +linecache2==1.0.0 + # via traceback2 +lxml==4.5.2 + # via + # -r dependencies/pip/requirements.in + # formpack + # gdata-python3 + # pyquery +markdown==3.2.2 + # via -r dependencies/pip/requirements.in +modilabs-python-utils==0.1.5 + # via -r dependencies/pip/requirements.in +numpy==1.19.0 + # via pandas +oauthlib==3.1.0 + # via django-oauth-toolkit +openpyxl==3.0.4 + # via -r dependencies/pip/requirements.in +pandas==1.0.5 + # via -r dependencies/pip/requirements.in +path.py==12.5.0 + # via formpack +path==15.0.0 + # via path.py +pillow==7.2.0 + # via + # -r dependencies/pip/requirements.in + # elaphe3 +psycopg2-binary==2.8.5 + # via -r dependencies/pip/requirements.in +pymongo==3.10.1 + # via -r dependencies/pip/requirements.in +pyquery==1.4.1 + # via formpack +pyrsistent==0.16.0 + # via jsonschema +python-crontab==2.5.1 + # via django-celery-beat +python-dateutil==2.8.1 + # via + # botocore + # pandas + # python-crontab +python-slugify==4.0.1 + # via transifex-client +pytz==2020.1 + # via + # -r dependencies/pip/requirements.in + # celery + # django + # django-timezone-field + # pandas +pyxform==1.5.1 + # via + # -r dependencies/pip/requirements.in + # formpack +redis==3.5.3 + # via + # -r dependencies/pip/requirements.in + # celery + # django-redis-sessions +requests==2.24.0 + # via + # django-oauth-toolkit + # transifex-client +s3transfer==0.3.3 + # via boto3 +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter + # via -r dependencies/pip/requirements.in +sentry-sdk==0.16.5 + # via -r dependencies/pip/requirements.in +simplejson==3.17.2 + # via -r dependencies/pip/requirements.in +six==1.15.0 + # via + # django-extensions + # django-templated-email + # djangorestframework-csv + # ecdsa + # jsonschema + # pyrsistent + # python-dateutil + # ssrf-protect + # transifex-client + # unittest2 +smmap==3.0.4 + # via gitdb +sqlparse==0.3.1 + # via django +statistics==1.0.3.5 + # via formpack +text-unidecode==1.3 + # via python-slugify +tlslite-ng==0.7.5 + # via gdata-python3 +traceback2==1.4.0 + # via unittest2 +transifex-client==0.13.11 + # via -r dependencies/pip/requirements.in +unicodecsv==0.14.1 + # via + # djangorestframework-csv + # pyxform +unittest2==1.1.0 + # via pyxform +urllib3==1.25.9 + # via + # botocore + # requests + # sentry-sdk + # transifex-client +uwsgi==2.0.19.1 + # via -r dependencies/pip/prod.in +vine==1.3.0 + # via + # amqp + # celery +xlrd==1.2.0 + # via + # -r dependencies/pip/requirements.in + # pyxform +xlsxwriter==1.2.9 + # via formpack +xlwt==1.3.0 + # via -r dependencies/pip/requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index dc5c63529..3e8da9ca3 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -37,7 +37,7 @@ Pillow psycopg2-binary pymongo lxml -pyxform==0.15.1 # same version as KPI. issues with 1.1.0 +pyxform django-reversion<3.0.2 xlrd xlwt diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 157e2a5ec..563a1643b 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -4,106 +4,280 @@ # # pip-compile dependencies/pip/requirements.in # --e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages # via -r dependencies/pip/requirements.in --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==2.6.0 # via -r dependencies/pip/requirements.in, kombu -argparse==1.4.0 # via unittest2 -attrs==19.3.0 # via jsonschema -backports.csv==1.0.7 # via formpack -begins==0.9 # via formpack -billiard==3.6.3.0 # via celery -boto3==1.14.44 # via -r dependencies/pip/requirements.in -botocore==1.17.44 # via boto3, s3transfer -celery[redis]==4.4.6 # via -r dependencies/pip/requirements.in, django-celery-beat -certifi==2020.6.20 # via requests, sentry-sdk -chardet==3.0.4 # via requests -cssselect==1.1.0 # via pyquery -defusedxml==0.6.0 # via djangorestframework-xml -dict2xml==1.7.0 # via -r dependencies/pip/requirements.in -dj-database-url==0.5.0 # via -r dependencies/pip/requirements.in -django-celery-beat==2.0.0 # via -r dependencies/pip/requirements.in -django-cors-headers==3.4.0 # via -r dependencies/pip/requirements.in -django-db-readonly==0.6.0 # via -r dependencies/pip/requirements.in -django-extensions==3.0.3 # via -r dependencies/pip/requirements.in -django-filter==2.3.0 # via -r dependencies/pip/requirements.in -django-guardian==2.3.0 # via -r dependencies/pip/requirements.in, djangorestframework-guardian -django-oauth-toolkit==1.3.2 # via -r dependencies/pip/requirements.in -django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in -django-registration-redux==2.8 # via -r dependencies/pip/requirements.in -django-render-block==0.7 # via django-templated-email -django-reversion==3.0.1 # via -r dependencies/pip/requirements.in -django-taggit==1.3.0 # via -r dependencies/pip/requirements.in -django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in -django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield -djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in -djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in -djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in -djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -djangorestframework==3.11.0 # via -r dependencies/pip/requirements.in, djangorestframework-csv, djangorestframework-guardian -docutils==0.15.2 # via botocore, statistics -dpath==2.0.1 # via -r dependencies/pip/requirements.in -ecdsa==0.15 # via tlslite-ng -elaphe3==0.2.0 # via -r dependencies/pip/requirements.in -et-xmlfile==1.0.1 # via openpyxl -formencode==1.3.1 # via pyxform -git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack # via -r dependencies/pip/requirements.in -future==0.18.2 # via celery -gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -geojson-rewind==0.2.0 # via formpack -gitdb==4.0.5 # via gitpython -gitpython==3.1.7 # via transifex-client -idna==2.10 # via requests -jdcal==1.4.1 # via openpyxl -jmespath==0.10.0 # via boto3, botocore -jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -jsonschema==3.2.0 # via formpack -kombu==4.6.11 # via celery -linecache2==1.0.0 # via traceback2 -lxml==4.5.2 # via -r dependencies/pip/requirements.in, formpack, gdata-python3, pyquery -markdown==3.2.2 # via -r dependencies/pip/requirements.in -modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -numpy==1.19.0 # via pandas -oauthlib==3.1.0 # via django-oauth-toolkit -openpyxl==3.0.4 # via -r dependencies/pip/requirements.in -pandas==1.0.5 # via -r dependencies/pip/requirements.in -path.py==12.4.0 # via formpack -path==13.1.0 # via path.py -pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 -psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in -pymongo==3.10.1 # via -r dependencies/pip/requirements.in -pyquery==1.4.1 # via formpack -pyrsistent==0.16.0 # via jsonschema -python-crontab==2.5.1 # via django-celery-beat -python-dateutil==2.8.1 # via botocore, pandas, python-crontab -python-slugify==4.0.1 # via transifex-client -pytz==2020.1 # via -r dependencies/pip/requirements.in, celery, django, django-timezone-field, pandas -pyxform==0.15.1 # via -r dependencies/pip/requirements.in, formpack -redis==3.5.3 # via -r dependencies/pip/requirements.in, celery, django-redis-sessions -requests==2.24.0 # via django-oauth-toolkit, transifex-client -s3transfer==0.3.3 # via boto3 -https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter # via -r dependencies/pip/requirements.in -sentry-sdk==0.16.5 # via -r dependencies/pip/requirements.in -simplejson==3.17.2 # via -r dependencies/pip/requirements.in -six==1.15.0 # via django-extensions, django-templated-email, djangorestframework-csv, ecdsa, jsonschema, pyrsistent, python-dateutil, ssrf-protect, transifex-client, unittest2 -smmap==3.0.4 # via gitdb -sqlparse==0.3.1 # via django -statistics==1.0.3.5 # via formpack -text-unidecode==1.3 # via python-slugify -tlslite-ng==0.7.5 # via gdata-python3 -traceback2==1.4.0 # via unittest2 -transifex-client==0.13.11 # via -r dependencies/pip/requirements.in -unicodecsv==0.14.1 # via djangorestframework-csv, pyxform -unittest2==1.1.0 # via pyxform -urllib3==1.25.9 # via botocore, requests, sentry-sdk, transifex-client -vine==1.3.0 # via amqp, celery -xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform -xlsxwriter==1.2.9 # via formpack -xlwt==1.3.0 # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest + # via -r dependencies/pip/requirements.in +-e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages + # via -r dependencies/pip/requirements.in +-e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest + # via -r dependencies/pip/requirements.in +-e git+https://github.com/dresiu/recaptcha-client-1.0.6-py3@48078f8131e2f3c2054d2099ef48cfe9a5745d0c#egg=recaptcha-client + # via -r dependencies/pip/requirements.in +-e git+https://github.com/kobotoolbox/ssrf-protect@755efe16694273ce66060a51e04f973dc034ca4e#egg=ssrf_protect + # via -r dependencies/pip/requirements.in +amqp==2.6.0 + # via + # -r dependencies/pip/requirements.in + # kombu +argparse==1.4.0 + # via unittest2 +attrs==19.3.0 + # via jsonschema +backports.csv==1.0.7 + # via formpack +begins==0.9 + # via formpack +billiard==3.6.3.0 + # via celery +boto3==1.14.44 + # via -r dependencies/pip/requirements.in +botocore==1.17.44 + # via + # boto3 + # s3transfer +celery[redis]==4.4.6 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat +certifi==2020.6.20 + # via + # requests + # sentry-sdk +chardet==3.0.4 + # via requests +cssselect==1.1.0 + # via pyquery +defusedxml==0.6.0 + # via djangorestframework-xml +dict2xml==1.7.0 + # via -r dependencies/pip/requirements.in +dj-database-url==0.5.0 + # via -r dependencies/pip/requirements.in +django-celery-beat==2.0.0 + # via -r dependencies/pip/requirements.in +django-cors-headers==3.4.0 + # via -r dependencies/pip/requirements.in +django-db-readonly==0.6.0 + # via -r dependencies/pip/requirements.in +django-extensions==3.0.3 + # via -r dependencies/pip/requirements.in +django-filter==2.3.0 + # via -r dependencies/pip/requirements.in +django-guardian==2.3.0 + # via + # -r dependencies/pip/requirements.in + # djangorestframework-guardian +django-oauth-toolkit==1.3.2 + # via -r dependencies/pip/requirements.in +django-pure-pagination==0.3.0 + # via -r dependencies/pip/requirements.in +django-redis-sessions==0.6.1 + # via -r dependencies/pip/requirements.in +django-registration-redux==2.8 + # via -r dependencies/pip/requirements.in +django-render-block==0.7 + # via django-templated-email +django-reversion==3.0.1 + # via -r dependencies/pip/requirements.in +django-taggit==1.3.0 + # via -r dependencies/pip/requirements.in +django-templated-email==2.3.0 + # via -r dependencies/pip/requirements.in +django-timezone-field==4.0 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat +django==2.2.14 + # via + # -r dependencies/pip/requirements.in + # django-celery-beat + # django-cors-headers + # django-filter + # django-guardian + # django-oauth-toolkit + # django-render-block + # django-reversion + # django-storages + # django-taggit + # django-timezone-field + # djangorestframework + # djangorestframework-guardian + # jsonfield +djangorestframework-csv==2.1.0 + # via -r dependencies/pip/requirements.in +djangorestframework-guardian==0.3.0 + # via -r dependencies/pip/requirements.in +djangorestframework-jsonp==1.0.2 + # via -r dependencies/pip/requirements.in +djangorestframework-xml==2.0.0 + # via -r dependencies/pip/requirements.in +djangorestframework==3.11.0 + # via + # -r dependencies/pip/requirements.in + # djangorestframework-csv + # djangorestframework-guardian +docutils==0.15.2 + # via + # botocore + # statistics +dpath==2.0.1 + # via -r dependencies/pip/requirements.in +ecdsa==0.15 + # via tlslite-ng +elaphe3==0.2.0 + # via -r dependencies/pip/requirements.in +et-xmlfile==1.0.1 + # via openpyxl +formencode==1.3.1 + # via pyxform +git+https://github.com/kobotoolbox/formpack.git@52f77e3519fd1079ce3451f9a5c7c27002e9f3f0#egg=formpack + # via -r dependencies/pip/requirements.in +future==0.18.2 + # via celery +gdata-python3==3.0.1 + # via -r dependencies/pip/requirements.in +geojson-rewind==0.2.0 + # via formpack +gitdb==4.0.5 + # via gitpython +gitpython==3.1.7 + # via transifex-client +idna==2.10 + # via requests +jdcal==1.4.1 + # via openpyxl +jmespath==0.10.0 + # via + # boto3 + # botocore +jsonfield==3.1.0 + # via -r dependencies/pip/requirements.in +jsonschema==3.2.0 + # via formpack +kombu==4.6.11 + # via celery +linecache2==1.0.0 + # via traceback2 +lxml==4.5.2 + # via + # -r dependencies/pip/requirements.in + # formpack + # gdata-python3 + # pyquery +markdown==3.2.2 + # via -r dependencies/pip/requirements.in +modilabs-python-utils==0.1.5 + # via -r dependencies/pip/requirements.in +numpy==1.19.0 + # via pandas +oauthlib==3.1.0 + # via django-oauth-toolkit +openpyxl==3.0.4 + # via -r dependencies/pip/requirements.in +pandas==1.0.5 + # via -r dependencies/pip/requirements.in +path.py==12.4.0 + # via formpack +path==13.1.0 + # via path.py +pillow==7.2.0 + # via + # -r dependencies/pip/requirements.in + # elaphe3 +psycopg2-binary==2.8.5 + # via -r dependencies/pip/requirements.in +pymongo==3.10.1 + # via -r dependencies/pip/requirements.in +pyquery==1.4.1 + # via formpack +pyrsistent==0.16.0 + # via jsonschema +python-crontab==2.5.1 + # via django-celery-beat +python-dateutil==2.8.1 + # via + # botocore + # pandas + # python-crontab +python-slugify==4.0.1 + # via transifex-client +pytz==2020.1 + # via + # -r dependencies/pip/requirements.in + # celery + # django + # django-timezone-field + # pandas +pyxform==1.5.1 + # via + # -r dependencies/pip/requirements.in + # formpack +redis==3.5.3 + # via + # -r dependencies/pip/requirements.in + # celery + # django-redis-sessions +requests==2.24.0 + # via + # django-oauth-toolkit + # transifex-client +s3transfer==0.3.3 + # via boto3 +https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter + # via -r dependencies/pip/requirements.in +sentry-sdk==0.16.5 + # via -r dependencies/pip/requirements.in +simplejson==3.17.2 + # via -r dependencies/pip/requirements.in +six==1.15.0 + # via + # django-extensions + # django-templated-email + # djangorestframework-csv + # ecdsa + # jsonschema + # pyrsistent + # python-dateutil + # ssrf-protect + # transifex-client + # unittest2 +smmap==3.0.4 + # via gitdb +sqlparse==0.3.1 + # via django +statistics==1.0.3.5 + # via formpack +text-unidecode==1.3 + # via python-slugify +tlslite-ng==0.7.5 + # via gdata-python3 +traceback2==1.4.0 + # via unittest2 +transifex-client==0.13.11 + # via -r dependencies/pip/requirements.in +unicodecsv==0.14.1 + # via + # djangorestframework-csv + # pyxform +unittest2==1.1.0 + # via pyxform +urllib3==1.25.9 + # via + # botocore + # requests + # sentry-sdk + # transifex-client +vine==1.3.0 + # via + # amqp + # celery +xlrd==1.2.0 + # via + # -r dependencies/pip/requirements.in + # pyxform +xlsxwriter==1.2.9 + # via formpack +xlwt==1.3.0 + # via -r dependencies/pip/requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools From 46b665049a61a6c558c267f3817f32ec6e507ad3 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 23:41:47 -0400 Subject: [PATCH 09/10] =?UTF-8?q?Fix=20XML=20attribute=20order=20and=20res?= =?UTF-8?q?ponse=20bytestring=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit issues in unit tests --- onadata/apps/api/tests/fixtures/Transportation Form.xml | 2 +- onadata/apps/api/tests/viewsets/test_xform_list_api.py | 4 ++-- .../main/tests/fixtures/transportation/transportation.xml | 2 +- onadata/apps/main/tests/test_past_bugs.py | 5 +---- .../apps/main/tests/test_user_id_string_unique_together.py | 5 +---- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/onadata/apps/api/tests/fixtures/Transportation Form.xml b/onadata/apps/api/tests/fixtures/Transportation Form.xml index 9aa6c7149..073131011 100644 --- a/onadata/apps/api/tests/fixtures/Transportation Form.xml +++ b/onadata/apps/api/tests/fixtures/Transportation Form.xml @@ -58,7 +58,7 @@ - + diff --git a/onadata/apps/api/tests/viewsets/test_xform_list_api.py b/onadata/apps/api/tests/viewsets/test_xform_list_api.py index 703af27ac..8c53d706b 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_list_api.py +++ b/onadata/apps/api/tests/viewsets/test_xform_list_api.py @@ -221,7 +221,7 @@ def test_get_xform_list_with_formid_parameter(self): with open(path) as f: form_list_xml = f.read().strip() data = {"hash": self.xform.hash, "pk": self.xform.pk} - content = response.render().content + content = response.render().content.decode('utf-8') self.assertEqual(content, form_list_xml % data) def test_retrieve_xform_xml(self): @@ -247,7 +247,7 @@ def test_retrieve_xform_xml(self): os.path.dirname(__file__), '..', 'fixtures', 'Transportation Form.xml') - with open(path, 'r') as f: + with open(path) as f: form_xml = f.read().strip() data = {"form_uuid": self.xform.uuid} content = smart_str(response.render().content).strip() diff --git a/onadata/apps/main/tests/fixtures/transportation/transportation.xml b/onadata/apps/main/tests/fixtures/transportation/transportation.xml index ed90c375e..4305eaaec 100644 --- a/onadata/apps/main/tests/fixtures/transportation/transportation.xml +++ b/onadata/apps/main/tests/fixtures/transportation/transportation.xml @@ -58,7 +58,7 @@ - + diff --git a/onadata/apps/main/tests/test_past_bugs.py b/onadata/apps/main/tests/test_past_bugs.py index 823fb2fa2..268950ec8 100644 --- a/onadata/apps/main/tests/test_past_bugs.py +++ b/onadata/apps/main/tests/test_past_bugs.py @@ -1,8 +1,6 @@ # coding: utf-8 import os -from django.utils.encoding import smart_text - from onadata.apps.logger.models import XForm, Instance from .test_base import TestBase @@ -19,8 +17,7 @@ def test_uniqueness_of_group_names_enforced(self): response = self._publish_xls_file( 'fixtures/group_names_must_be_unique.xls') message = 'The name "group_names_must_be_unique" is the same as the form name' - response_content = smart_text(response.content) - self.assertTrue(message in response_content) + self.assertTrue(message in response.json()['text']) self.assertEqual(XForm.objects.count(), pre_count) def test_mch(self): diff --git a/onadata/apps/main/tests/test_user_id_string_unique_together.py b/onadata/apps/main/tests/test_user_id_string_unique_together.py index d0980ca93..ac9279ec3 100644 --- a/onadata/apps/main/tests/test_user_id_string_unique_together.py +++ b/onadata/apps/main/tests/test_user_id_string_unique_together.py @@ -1,8 +1,6 @@ # coding: utf-8 import os -from django.utils.encoding import smart_text - from onadata.apps.logger.models import XForm from .test_base import TestBase @@ -25,12 +23,11 @@ def test_unique_together(self): # second time response = self._publish_xls_file(xls_path) - response_content = smart_text(response.content) # SQLite returns `UNIQUE constraint failed` whereas PostgreSQL # returns 'duplicate key ... violates unique constraint' self.assertIn( 'unique constraint', - response_content.lower(), + response.json()['text'].lower(), ) self.assertEqual(XForm.objects.count(), 1) self.client.logout() From 77ab828c2b4e2a9da74093280253ee5f7ce69588 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Tue, 25 May 2021 03:03:11 -0400 Subject: [PATCH 10/10] Resolve encoding-related errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and prefer explicit encoding/decoding over `smart_str()`. See #718 --- .../api/tests/viewsets/test_briefcase_api.py | 3 +-- .../api/tests/viewsets/test_data_viewset.py | 2 +- .../api/tests/viewsets/test_xform_list_api.py | 19 +++++++++---------- .../viewsets/test_xform_submission_api.py | 12 +++++++----- .../apps/api/viewsets/xform_submission_api.py | 7 ++++--- onadata/apps/logger/xform_instance_parser.py | 2 +- onadata/libs/utils/logger_tools.py | 12 ++++++++---- onadata/libs/utils/viewer_tools.py | 6 ++---- 8 files changed, 33 insertions(+), 30 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_briefcase_api.py b/onadata/apps/api/tests/viewsets/test_briefcase_api.py index b62c41186..f44704b94 100644 --- a/onadata/apps/api/tests/viewsets/test_briefcase_api.py +++ b/onadata/apps/api/tests/viewsets/test_briefcase_api.py @@ -3,7 +3,6 @@ from django.urls import reverse from django.core.files.storage import get_storage_class -from django.utils.encoding import smart_str from django_digest.test import DigestAuth from rest_framework.test import APIRequestFactory @@ -224,7 +223,7 @@ def test_view_download_submission(self): text = text.replace('{{xform_uuid}}', self.xform.uuid) self.assertContains(response, instance_id, status_code=200) - self.assertMultiLineEqual(smart_str(response.content), text) + self.assertMultiLineEqual(response.content.decode('utf-8'), text) def test_view_download_submission_other_user(self): view = BriefcaseApi.as_view({'get': 'retrieve'}) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index fb2dd09f0..cdc145726 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -21,7 +21,7 @@ def enketo_mock(url, request): response = requests.Response() response.status_code = 201 - response._content = '{"url": "https://hmh2a.enketo.formhub.org"}' + response._content = b'{"url": "https://hmh2a.enketo.formhub.org"}' return response diff --git a/onadata/apps/api/tests/viewsets/test_xform_list_api.py b/onadata/apps/api/tests/viewsets/test_xform_list_api.py index 8c53d706b..42a263950 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_list_api.py +++ b/onadata/apps/api/tests/viewsets/test_xform_list_api.py @@ -3,7 +3,6 @@ from django.conf import settings from django_digest.test import DigestAuth -from django.utils.encoding import smart_str from guardian.shortcuts import assign_perm from onadata.apps.api.tests.viewsets.test_abstract_viewset import\ @@ -40,7 +39,7 @@ def test_get_xform_list(self): form_list_xml = f.read().strip() data = {"hash": self.xform.hash, "pk": self.xform.pk} content = response.render().content - self.assertEqual(smart_str(content), form_list_xml % data) + self.assertEqual(content.decode('utf-8'), form_list_xml % data) self.assertTrue(response.has_header('X-OpenRosa-Version')) self.assertTrue( response.has_header('X-OpenRosa-Accept-Content-Length')) @@ -61,7 +60,7 @@ def test_get_xform_list_inactive_form(self): xml = '\n\n\nscreenshot.png%(hash)shttp://testserver/bob/xformsMedia/%(xform)s/%(pk)s.png""" # noqa data = {"hash": self.metadata.hash, "pk": self.metadata.pk, "xform": self.xform.pk} - content = smart_str(response.render().content).strip() + content = response.render().content.decode('utf-8').strip() self.assertEqual(content, manifest_xml % data) self.assertTrue(response.has_header('X-OpenRosa-Version')) self.assertTrue( @@ -306,7 +305,7 @@ def test_retrieve_xform_manifest_anonymous_user(self): screenshot.png%(hash)shttp://testserver/bob/xformsMedia/%(xform)s/%(pk)s.png""" # noqa data = {"hash": self.metadata.hash, "pk": self.metadata.pk, "xform": self.xform.pk} - content = smart_str(response.render().content).strip() + content = response.render().content.decode('utf-8').strip() self.assertEqual(content, manifest_xml % data) self.assertTrue(response.has_header('X-OpenRosa-Version')) self.assertTrue( diff --git a/onadata/apps/api/tests/viewsets/test_xform_submission_api.py b/onadata/apps/api/tests/viewsets/test_xform_submission_api.py index 37dca4e79..bbec28dcf 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_submission_api.py +++ b/onadata/apps/api/tests/viewsets/test_xform_submission_api.py @@ -4,7 +4,6 @@ import simplejson as json from django.contrib.auth.models import AnonymousUser from django.core.files.uploadedfile import InMemoryUploadedFile -from django.utils.encoding import smart_str from django_digest.test import DigestAuth from guardian.shortcuts import assign_perm @@ -163,8 +162,8 @@ def test_post_submission_authenticated_bad_json(self): '..', 'fixtures', 'transport_submission_bad.json') - with open(path, 'rb') as f: - data = json.loads(smart_str(f.read())) + with open(path) as f: + data = json.loads(f.read()) request = self.factory.post('/submission', data, format='json') response = self.view(request) self.assertEqual(response.status_code, 401) @@ -175,8 +174,11 @@ def test_post_submission_authenticated_bad_json(self): response = self.view(request) rendered_response = response.render() self.assertTrue('error' in rendered_response.data) - self.assertTrue(smart_str(rendered_response.data['error']). - startswith("b'Received empty submission")) + self.assertTrue( + rendered_response.data['error'].startswith( + 'Received empty submission' + ) + ) self.assertTrue(rendered_response.status_code == 400) self.assertTrue(rendered_response.has_header('X-OpenRosa-Version')) self.assertTrue( diff --git a/onadata/apps/api/viewsets/xform_submission_api.py b/onadata/apps/api/viewsets/xform_submission_api.py index 0d27ccb3b..47423a513 100644 --- a/onadata/apps/api/viewsets/xform_submission_api.py +++ b/onadata/apps/api/viewsets/xform_submission_api.py @@ -5,7 +5,6 @@ from django.conf import settings from django.contrib.auth.models import User from django.shortcuts import get_object_or_404 -from django.utils.encoding import smart_str from django.utils.translation import ugettext as _ from rest_framework import permissions @@ -211,9 +210,11 @@ def error_response(self, error, is_json_request, request): elif not is_json_request: return error else: - error_msg = xml_error_re.search(smart_str(error.content)).groups()[0] + error_msg = xml_error_re.search( + error.content.decode('utf-8') + ).groups()[0] status_code = error.status_code - return Response({'error': smart_str(error_msg)}, + return Response({'error': error_msg}, headers=self.get_openrosa_headers(request), status=status_code) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index f65f668ee..21b93c305 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -122,7 +122,7 @@ def get_deprecated_uuid_from_xml(xml): def clean_and_parse_xml(xml_string): clean_xml_str = xml_string.strip() clean_xml_str = re.sub(r">\s+<", "><", smart_str(clean_xml_str)) - xml_obj = minidom.parseString(smart_str(clean_xml_str)) + xml_obj = minidom.parseString(clean_xml_str) return xml_obj diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index cb2d1a58e..636cba311 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -558,10 +558,14 @@ class OpenRosaResponse(BaseOpenRosaResponse): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # wrap content around xml - self.content = ''' - - %s -''' % self.content + self.content = ( + b"\n" + b'\n' + b' ' + ) + self.content + ( + b'\n' + b'' + ) class OpenRosaResponseNotFound(OpenRosaResponse): diff --git a/onadata/libs/utils/viewer_tools.py b/onadata/libs/utils/viewer_tools.py index 1d2e036ae..e266d72b7 100644 --- a/onadata/libs/utils/viewer_tools.py +++ b/onadata/libs/utils/viewer_tools.py @@ -12,7 +12,6 @@ from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import InMemoryUploadedFile from django.core.mail import mail_admins -from django.utils.encoding import smart_str from django.utils.translation import ugettext as _ from onadata.libs.utils import common_tags @@ -204,8 +203,7 @@ def enketo_url(form_url, id_string, instance_xml=None, if req.status_code in [200, 201]: try: - # ToDo find out why req.json() does not work - response = json.loads(smart_str(req.content)) + response = req.json() except ValueError: pass else: @@ -217,7 +215,7 @@ def enketo_url(form_url, id_string, instance_xml=None, return response['url'] else: try: - response = json.loads(smart_str(req.content)) + response = req.json() except ValueError: pass else: