From fe03dcd6ff8a7608689c09cb8f21cfd08284d1b4 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Mon, 24 May 2021 19:34:25 -0400 Subject: [PATCH] Upgrade to pyxform 1.5.1 * Recognize select questions from the question elements themselves instead of checking the bind type; see XLSForm/pyxform#168 * Recognize new error message when a group name matches the form name See XLSForm/pyxform#510 * Set `allow_choice_duplicates` to `yes` in cascade test XLSForms See XLSForm/pyxform#23; https://github.com/XLSForm/pyxform/issues/373#issuecomment-580572668 * Update tests to use `default_name` instead of relying on the XLSForm file name See XLSForm/pyxform#130 * Update expected XML in tests to match new pyxform behavior: * `` becomes `` * See XLSForm/pyxform#393 * The bind type for `select` and `select1` becomes `string` * See XLSForm/pyxform#168 * `calculate="concat('uuid:', uuid())"` becomes `jr:preload="uid"` * See XLSForm/pyxform#94 This squashed set of changes includes work from kobotoolbox/kobocat#699. Thanks to @duvld for that contribution. --- .../tests/fixtures/Transportation Form.xml | 24 +++++++++--------- .../forms/contributions/contributions.xml | 2 +- .../bug_fixes/new_cascading_select.xls | Bin 9728 -> 9216 bytes .../new_cascading_select.xls | Bin 9728 -> 9216 bytes .../new_cascading_select.xml | 2 +- .../main/tests/fixtures/exp_line_break.xml | 2 +- .../transportation/Transportation Form.xml | 2 +- .../transportation/transportation.xml | 24 +++++++++--------- .../transportation/transportation2.xml | 2 +- onadata/apps/main/tests/test_past_bugs.py | 2 +- onadata/apps/viewer/pandas_mongo_bridge.py | 9 +++---- .../apps/viewer/tests/test_export_builder.py | 15 +++++++---- onadata/libs/utils/export_tools.py | 5 ++-- requirements/base.pip | 2 +- requirements/travis.pip | 2 +- 15 files changed, 48 insertions(+), 45 deletions(-) diff --git a/onadata/apps/api/tests/fixtures/Transportation Form.xml b/onadata/apps/api/tests/fixtures/Transportation Form.xml index 376923e80..9aa6c7149 100644 --- a/onadata/apps/api/tests/fixtures/Transportation Form.xml +++ b/onadata/apps/api/tests/fixtures/Transportation Form.xml @@ -2,7 +2,7 @@ transportation_2011_07_25 - + @@ -46,18 +46,18 @@ - + - - - - - - - - - - + + + + + + + + + + diff --git a/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml b/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml index 14f205f56..263628b88 100644 --- a/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml +++ b/onadata/apps/api/tests/fixtures/forms/contributions/contributions.xml @@ -21,7 +21,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/bug_fixes/new_cascading_select.xls b/onadata/apps/main/tests/fixtures/bug_fixes/new_cascading_select.xls index 8cfd43372ef105c2bbd53db2f79de7ef5b8e081b..3afd4f10f0e6fd2542a8515b227c2374fef277e6 100644 GIT binary patch delta 991 zcmYk5&uddb5XZm!l9!j?zhg|Ospged)Yd9eMG)*I6xxcXA{0qu5-}#&(zI$&NiIEk zd1eLw0S_L$Xvjej^-%l+^dJ#L(33Y0J?No{Gf8nD%e>{YGqc~>dAqHp)`B%FLa)vk zn%*A_2I0=U_Fg<$bG~+d(%c#VfjuF=x^`XVJLSznt@GTtp-q_vdEi2Qd-HC&v4ari zeY|3zcx$~kMx)A{KAA<{{pP^+!q%a5Y-rR|McrC8W1 z*X0ZUtj_^7%Jl)fn33Q7f0}MQf}gj?L~ZT+LRjV_a{*x*U0Xl|5%g>vMHFUSZ4g5YW{sf-Z!7mYnk$6u?)AdMO9tJ)3OrkV<3On5|Bh?OjxU zmNic+X4#yrOqR9eROtAUgG*IB)tOPrxDH zq}GH>eh(*tToIpxXHDx|N=WW4*YeIP?K-F}>%8|C4)$VY^{Ii7JZPE@i#uYM8om@_UvjJMj^`}DLL b*`|2f`Ac8%QZ^wNnUz73LQ delta 1000 zcmYjQOK1~O6g_W}$t2VKKaB~|q)IGk#VTzPL5M<0z*@zH2vKaTffiybZIlL)M6ml# zpSTiq;i@|wh#OIm;z|*8XLs&gcISsA-kVh4%y8b!x%a&}=iPUwRcg&_jEnHg6Ncss z0qFL6y@-8AdnaDGzi3`TA%QK(Z=P+>^z8KIvorP!xt&s6gNB+T?}7`~gvz_&M*FbhI&@fv6!(x_Yro z8H$@^IG<5`2()EWaX)RMom4!40BU~4g9xG+P&`Cc1LZIL+i{p`)^<#YAc7qSk0OeE zTzw#h81f0l<8)YUyAb-Sl~qZdid<4Fq84yTi($pNq)r1|(tJ$i^}fJTw4@6|zD`aB zK{e(M;Y-)$pX6@Y_x~_=hSf?|PB^Tx^3G|MXXX5H#aX%NaLUS-ET*nczU{Y)hD85% zqeG{{1`;?zpmOFAz9YBKQb%1d~uBaQm+jxRD&}@@m%aZEan=b9?!dkPqDxvXyPw&(an4ePc7X eCJM5VYl>_3FB-*1**JJ#XU}zMrO(T4_nW_+-jP%Q diff --git a/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xls b/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xls index 99bb0735156bb280060e3c7682064114fdf6ac91..2ea8cf331fb046ce16b225df18bcccb465f45727 100644 GIT binary patch delta 975 zcmYk5&ubGw6vw}_`LRhhX*NG=LT$Axt*EV0ib_xB5(HcEA{2_WF$q`_Y;9UK_#;8+ zRn|=ZgdRM2(U5~6)kEfWV=UU%dz3(!J7VzS7#$?`YGOP9Ats-QIjqs_h_* ztidZf3U}An3Z?1}Or)c7CU8kW76Z%rWo?Q4v#n2oA0j;p)|g@g26}$QLE57|syKuY%0a~= z7(p(iI81Hr8DT8)k+}eqM%xh(K?EHKM-hb;R~y6-gOyMmr@z^5oe1v>nMx<+q1j(n zt#vAMYwd_i3Lkru?3P(2eLbuS1}dQ~@jyb5zCxoq6SF0qeYC7RWgY zVgXxDr*1EHsp+8X1kTXBKOh%Tf-=}AXf=Ve^c|8QI`^MAZ{{dZz$M?MR^@jyX>dJ! z^=WR+r(cNXSbXq2^rPo-x;g&)))NZx4#I#dH2cY}l5s%`Wc(ud((~0Vl7+}N$aw2t zU0|GMy6U(%H_XFsEz_0vC*ra;VfeDo_YyyEei8DdJ1##?G@6r-=qc4E>teU{m%ie) Ttetx7;R3c~K7CytxZnQ+5aE=} delta 980 zcmYk4PiPZS5XQf^*<_PV()>?Mh$d}`g<7#{6;XsJloYH5J%|v+#u{ibw9rOr5KRPo z@Aio&K@Xn1=|WCYL5e3u@Mvz{JbDnSki?m^`j+tR=J#gyee>o`Yo;~3IVr+#jvJaU z1fbXN_apXs?Y(%bf7QIS0)f|%-#j~>>AC5v7iR5O#%*oPWpK!YwR@}gtBnnWFzn|Q z0}G2QOAFPt4TSM0B9EG z?E8Orouo=&gM_0pHt3vHaW*I%Q_Kb>M?q|0$x`~}sqMjJXh;lxI=XZTf*O{ zyl{kwa|JmilVVNgSFX`AG95lT;(|sIQZFXrGXRq)if={pbRR_Sfbr^~K6^{^DwNsZL9Sbr-!) zoZQ>Lb?(Y=cYgs_Mv#{P diff --git a/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml b/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml index 1962c50d5..bdfd13a37 100644 --- a/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml +++ b/onadata/apps/main/tests/fixtures/cascading_selects/new_cascading_select.xml @@ -150,7 +150,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/exp_line_break.xml b/onadata/apps/main/tests/fixtures/exp_line_break.xml index a817a9885..a4ed68fa1 100644 --- a/onadata/apps/main/tests/fixtures/exp_line_break.xml +++ b/onadata/apps/main/tests/fixtures/exp_line_break.xml @@ -49,7 +49,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml b/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml index 5288295ec..17b6687bc 100644 --- a/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml +++ b/onadata/apps/main/tests/fixtures/transportation/Transportation Form.xml @@ -57,7 +57,7 @@ - + diff --git a/onadata/apps/main/tests/fixtures/transportation/transportation.xml b/onadata/apps/main/tests/fixtures/transportation/transportation.xml index 05a7aae24..ed90c375e 100644 --- a/onadata/apps/main/tests/fixtures/transportation/transportation.xml +++ b/onadata/apps/main/tests/fixtures/transportation/transportation.xml @@ -2,7 +2,7 @@ transportation_2011_07_25 - + @@ -46,18 +46,18 @@ - + - - - - - - - - - - + + + + + + + + + + diff --git a/onadata/apps/main/tests/fixtures/transportation/transportation2.xml b/onadata/apps/main/tests/fixtures/transportation/transportation2.xml index aa05548fa..80ba236d0 100644 --- a/onadata/apps/main/tests/fixtures/transportation/transportation2.xml +++ b/onadata/apps/main/tests/fixtures/transportation/transportation2.xml @@ -55,7 +55,7 @@ - + diff --git a/onadata/apps/main/tests/test_past_bugs.py b/onadata/apps/main/tests/test_past_bugs.py index d4166a5c3..6ed0c19a8 100644 --- a/onadata/apps/main/tests/test_past_bugs.py +++ b/onadata/apps/main/tests/test_past_bugs.py @@ -19,7 +19,7 @@ def test_uniqueness_of_group_names_enforced(self): self._create_user_and_login() response = self._publish_xls_file( 'fixtures/group_names_must_be_unique.xls') - message = 'There are two sections with the name group_names_must_be_unique.' + message = 'The name "group_names_must_be_unique" is the same as the form name' response_content = smart_text(response.content) self.assertTrue(message in response_content) self.assertEqual(XForm.objects.count(), pre_count) diff --git a/onadata/apps/viewer/pandas_mongo_bridge.py b/onadata/apps/viewer/pandas_mongo_bridge.py index e279f86c4..1feaaffdb 100644 --- a/onadata/apps/viewer/pandas_mongo_bridge.py +++ b/onadata/apps/viewer/pandas_mongo_bridge.py @@ -13,6 +13,7 @@ except ImportError: from pandas import ExcelWriter +from pyxform.constants import SELECT_ALL_THAT_APPLY from pyxform.survey_element import SurveyElement from pyxform.section import Section, RepeatingSection from pyxform.question import Question @@ -40,8 +41,6 @@ # this is Mongo Collection where we will store the parsed submissions xform_instances = settings.MONGO_DB.instances -# the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = "select" GEOPOINT_BIND_TYPE = "geopoint" # column group delimiters @@ -123,7 +122,7 @@ def _collect_select_multiples(cls, dd): return dict([(e.get_abbreviated_xpath(), [c.get_abbreviated_xpath() for c in e.children]) for e in dd.get_survey_elements() - if e.bind.get("type") == "select"]) + if e.type == SELECT_ALL_THAT_APPLY]) @classmethod def _split_select_multiples(cls, record, select_multiples, @@ -438,9 +437,9 @@ def _build_sections_recursive(self, section_name, element, child_bind_type = child.bind.get("type") if isinstance(child, Question) and not \ question_types_to_exclude(child.type)\ - and not child_bind_type == MULTIPLE_SELECT_BIND_TYPE: + and not child.type == SELECT_ALL_THAT_APPLY: self._add_column_to_section(section_name, child) - elif child_bind_type == MULTIPLE_SELECT_BIND_TYPE: + elif child.type == SELECT_ALL_THAT_APPLY: self.select_multiples[child.get_abbreviated_xpath()] = \ [option.get_abbreviated_xpath() for option in child.children] diff --git a/onadata/apps/viewer/tests/test_export_builder.py b/onadata/apps/viewer/tests/test_export_builder.py index 5f803336e..5744c999a 100644 --- a/onadata/apps/viewer/tests/test_export_builder.py +++ b/onadata/apps/viewer/tests/test_export_builder.py @@ -197,7 +197,8 @@ class TestExportBuilder(TestBase): def _create_childrens_survey(self): return create_survey_from_xls(_logger_fixture_path( - 'childrens_survey.xls')) + 'childrens_survey.xls'), + default_name='childrens_survey') def test_build_sections_from_survey(self): survey = self._create_childrens_survey() @@ -817,7 +818,8 @@ def test_get_valid_sheet_name_catches_long_duplicate_names(self): def test_to_xls_export_generates_valid_sheet_names(self): survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xls')) + 'childrens_survey_with_a_very_long_name.xls'), + default_name='childrens_survey_with_a_very_long_name') export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xls') @@ -836,7 +838,8 @@ def test_to_xls_export_generates_valid_sheet_names(self): def test_child_record_parent_table_is_updated_when_sheet_is_renamed(self): survey = create_survey_from_xls(_logger_fixture_path( - 'childrens_survey_with_a_very_long_name.xls')) + 'childrens_survey_with_a_very_long_name.xls'), + default_name='childrens_survey_with_a_very_long_name.xls') export_builder = ExportBuilder() export_builder.set_survey(survey) xls_file = NamedTemporaryFile(suffix='.xlsx') @@ -896,7 +899,8 @@ def test_type_conversion(self): } survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xls')) + 'test_data_types/test_data_types.xls'), + default_name='test_data_types/test_data_types') export_builder = ExportBuilder() export_builder.set_survey(survey) # format submission 1 for export @@ -919,7 +923,8 @@ def test_type_conversion(self): def test_xls_convert_dates_before_1900(self): survey = create_survey_from_xls(viewer_fixture_path( - 'test_data_types/test_data_types.xls')) + 'test_data_types/test_data_types.xls'), + default_name='test_data_types/test_data_types.xls') export_builder = ExportBuilder() export_builder.set_survey(survey) data = [ diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index aafadfdcb..d8fa2cd27 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -20,6 +20,7 @@ from django.utils.text import slugify from openpyxl.date_time import SharedDate from openpyxl.workbook import Workbook +from pyxform.constants import SELECT_ALL_THAT_APPLY from pyxform.question import Question from pyxform.section import Section, RepeatingSection from savReaderWriter import SavWriter @@ -51,8 +52,6 @@ QUESTION_TYPES_TO_EXCLUDE = [ 'note', ] -# the bind type of select multiples that we use to compare -MULTIPLE_SELECT_BIND_TYPE = "select" GEOPOINT_BIND_TYPE = "geopoint" @@ -265,7 +264,7 @@ def build_sections( {child_xpath: MongoHelper.encode(child_xpath)}) # if its a select multiple, make columns out of its choices - if child.bind.get("type") == MULTIPLE_SELECT_BIND_TYPE\ + if child.type == SELECT_ALL_THAT_APPLY\ and self.SPLIT_SELECT_MULTIPLES: for c in child.children: _xpath = c.get_abbreviated_xpath() diff --git a/requirements/base.pip b/requirements/base.pip index c59f47a28..0c5d98f1b 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -13,7 +13,7 @@ poster==0.8.1 psycopg2-binary==2.7.7 pymongo==3.7.2 lxml==3.4.0 -pyxform==0.13.1 +-e git+https://github.com/jnm/pyxform@v1.5.1-py2-jnm#egg=pyxform django-reversion==2.0.8 xlrd==1.1.0 xlwt==1.3.0 diff --git a/requirements/travis.pip b/requirements/travis.pip index b3778692e..af7cb1b77 100644 --- a/requirements/travis.pip +++ b/requirements/travis.pip @@ -85,7 +85,7 @@ python-digest==1.7 -e git+https://github.com/onaio/python-json2xlsclient.git@5a39387752d819cb6387f75569dbea9a5288aa6f#egg=python_json2xlsclient python-slugify==1.2.6 pytz==2018.9 -pyxform==0.13.1 +-e git+https://github.com/jnm/pyxform@v1.5.1-py2-jnm#egg=pyxform recaptcha-client==1.0.6 redis==3.2.0 requests==2.21.0