From 4f8100831cee94455743db7479df6cc184751569 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 21 Oct 2021 23:34:50 +0000 Subject: [PATCH 01/54] extend `try_get_number()` method to handle `inf` and `nan` --- src/formpack/schema/fields.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index a0d0a8e3..24b73e69 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -1,4 +1,5 @@ # coding: utf-8 +import math from collections import defaultdict, OrderedDict from dateutil import parser from functools import partial @@ -274,16 +275,31 @@ def parse_values(self, raw_values): @staticmethod def try_get_number(val): + """ + Attempt to convert string values to integers or floats. If the value is + `inf` or `nan` or not a valid integer or float then return the string + value instead. + """ + + str_val = val + try: val = int(val) except ValueError: pass else: return val + try: val = float(val) except ValueError: pass + + # The floats `+/-inf` and `nan` cause XLS exports to fail, therefore + # return the string value instead. + if isinstance(val, float) and not math.isfinite(val): + return str_val + return val From d4e398a55c55b5b6ad8aa363b474b7999d91a27a Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 21 Oct 2021 23:52:08 +0000 Subject: [PATCH 02/54] add tests --- tests/fixtures/nested_grouped_repeatable/v1.json | 9 ++++++++- tests/test_exports.py | 16 +++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/nested_grouped_repeatable/v1.json b/tests/fixtures/nested_grouped_repeatable/v1.json index bdcf3e09..ef862af5 100644 --- a/tests/fixtures/nested_grouped_repeatable/v1.json +++ b/tests/fixtures/nested_grouped_repeatable/v1.json @@ -29,6 +29,13 @@ "label": [ "Spruce" ] + }, + { + "list_name": "tree_type", + "name": "nan", + "label": [ + "Not a tree" + ] } ], "survey": [ @@ -199,7 +206,7 @@ "group_tree/group_nest/group_egg_count": "1" } ], - "group_tree/What_kind_of_tree_is_this": "maple" + "group_tree/What_kind_of_tree_is_this": "nan" } ], "__version__": "bird_nests_v1", diff --git a/tests/test_exports.py b/tests/test_exports.py index 91931fd6..923990a2 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -789,7 +789,7 @@ def test_nested_repeats_with_copy_fields(self): 'validation_status_approved', ], [ - 'maple', + 'nan', 3, 'Bird nest survey with nested repeatable groups', 2, @@ -978,7 +978,7 @@ def test_nested_repeats(self): 1 ], [ - 'maple', + 'nan', 3, 'Bird nest survey with nested repeatable groups', 2 @@ -1118,7 +1118,7 @@ def test_nested_repeats_with_xls_types(self): 1 ], [ - 'maple', + 'nan', 3, 'Bird nest survey with nested repeatable groups', 2 @@ -1840,6 +1840,16 @@ def test_xlsx(self): fp.export(**options).to_xlsx(xls, submissions) assert xls.isfile() + def test_xlsx_with_types(self): + title, schemas, submissions = build_fixture('nested_grouped_repeatable') + fp = FormPack(schemas, title) + options = {'versions': 'bird_nests_v1', 'xls_types_as_text': False} + + with TempDir() as d: + xls = d / 'foo.xlsx' + fp.export(**options).to_xlsx(xls, submissions) + assert xls.isfile() + def test_xlsx_long_sheet_names_and_invalid_chars(self): title, schemas, submissions = build_fixture('long_names') fp = FormPack(schemas, title) From 9cee720bb402b5976de5131624814da17e8ed7e1 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 27 Oct 2021 17:36:18 +0000 Subject: [PATCH 03/54] change how fields are combined across versions for repeat groups --- src/formpack/pack.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 5f64f66e..e3c90416 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -236,11 +236,11 @@ def get_fields_for_versions(self, versions=-1, data_types=None): # # Index 0 of tmp2d will be `[field1, field2]` tmp2d = [] - # This dict is used to remember final position of each field. - # Its keys are field_names and values are tuples of coordinates in tmp2d - # Keeping example above: - # `positions[field1.name]` would be `(0, 0)` - # `positions[field2.name]` would be `(0, 1)` + # This dict is used to remember final position of each field. Its keys + # are the combination of field and section names and the values are + # tuples of coordinates in tmp2d Keeping example above: + # `positions[f'{field1.name}_{section.name}']` would be `(0, 0)` + # `positions[f'{field2.name}_{section.name}']` would be `(0, 1)` positions = {} # Create the initial field mappings from the first form version @@ -251,10 +251,11 @@ def get_fields_for_versions(self, versions=-1, data_types=None): index = 0 for section in versions_desc[0].sections.values(): for field_name, field_object in section.fields.items(): + field_section_name = f'{field_name}_{section.name}' if isinstance(field_object, CopyField): copy_fields.append(field_object) else: - positions[field_name] = (index, 0) + positions[field_section_name] = (index, 0) tmp2d.append([field_object]) index += 1 @@ -262,9 +263,10 @@ def get_fields_for_versions(self, versions=-1, data_types=None): index = 0 for section_name, section in version.sections.items(): for field_name, field_object in section.fields.items(): + field_section_name = f'{field_name}_{section_name}' if not isinstance(field_object, CopyField): - if field_name in positions: - position = positions[field_name] + if field_section_name in positions: + position = positions[field_section_name] latest_field_object = tmp2d[position[0]][position[1]] # Because versions_desc are ordered from latest to oldest, # we use current field object as the old one and the one already @@ -283,7 +285,7 @@ def get_fields_for_versions(self, versions=-1, data_types=None): # it can happen when current version has more items than newest one. index = len(tmp2d) - 1 - positions[field_name] = (index, len(tmp2d[index]) - 1) + positions[field_section_name] = (index, len(tmp2d[index]) - 1) index += 1 From 920c8fff7ba76a60238f0652f649a80c850a12a9 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 27 Oct 2021 18:05:37 +0000 Subject: [PATCH 04/54] add test for fix --- tests/test_formpack_internals.py | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/test_formpack_internals.py b/tests/test_formpack_internals.py index 66bedec7..ab3a31f4 100644 --- a/tests/test_formpack_internals.py +++ b/tests/test_formpack_internals.py @@ -301,6 +301,50 @@ def test_get_fields_for_versions_returns_unique_fields(): assert sorted(field_names) == ['hey', 'one', 'two'] +def test_get_fields_for_versions_returns_all_fields_with_repeat_groups(): + """ + Test for #275 to ensure that all repeat groups across versions are included + especially if the question names remain the same and only the group name + changes + """ + fp = FormPack( + [ + { + 'content': { + 'survey': [ + {'name': 'old_name', 'type': 'begin_repeat'}, + {'name': 'one', 'type': 'image'}, + {'name': 'two', 'type': 'image'}, + {'type': 'end_repeat'}, + ] + }, + 'version': 'vRR7hH6SxTupvtvCqu7n5d', + }, + { + 'content': { + 'survey': [ + {'name': 'new_name', 'type': 'begin_repeat'}, + {'name': 'one', 'type': 'image'}, + {'name': 'two', 'type': 'image'}, + {'type': 'end_repeat'}, + ] + }, + 'version': 'vA8xs9JVi8aiSfypLgyYW2', + }, + ] + ) + fields = fp.get_fields_for_versions(fp.versions) + field_and_section_names = [ + (field.name, field.section.name) for field in fields + ] + assert field_and_section_names == [ + ('one', 'new_name'), + ('two', 'new_name'), + ('one', 'old_name'), + ('two', 'old_name'), + ] + + def test_get_fields_for_versions_returns_newest_of_fields_with_same_name(): schemas = [ { From f1a1ad23ff597e39a5d928189be168c9463243bb Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 27 Oct 2021 19:29:32 +0000 Subject: [PATCH 05/54] chance `field_section_name` to `section_field_name` --- src/formpack/pack.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index e3c90416..0318241e 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -251,11 +251,11 @@ def get_fields_for_versions(self, versions=-1, data_types=None): index = 0 for section in versions_desc[0].sections.values(): for field_name, field_object in section.fields.items(): - field_section_name = f'{field_name}_{section.name}' + section_field_name = f'{section.name}_{field_name}' if isinstance(field_object, CopyField): copy_fields.append(field_object) else: - positions[field_section_name] = (index, 0) + positions[section_field_name] = (index, 0) tmp2d.append([field_object]) index += 1 @@ -263,10 +263,10 @@ def get_fields_for_versions(self, versions=-1, data_types=None): index = 0 for section_name, section in version.sections.items(): for field_name, field_object in section.fields.items(): - field_section_name = f'{field_name}_{section_name}' + section_field_name = f'{section_name}_{field_name}' if not isinstance(field_object, CopyField): - if field_section_name in positions: - position = positions[field_section_name] + if section_field_name in positions: + position = positions[section_field_name] latest_field_object = tmp2d[position[0]][position[1]] # Because versions_desc are ordered from latest to oldest, # we use current field object as the old one and the one already @@ -285,7 +285,7 @@ def get_fields_for_versions(self, versions=-1, data_types=None): # it can happen when current version has more items than newest one. index = len(tmp2d) - 1 - positions[field_section_name] = (index, len(tmp2d[index]) - 1) + positions[section_field_name] = (index, len(tmp2d[index]) - 1) index += 1 From 0537c6a42d4a4dfce870df4454b3a17192fb5cc6 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 27 Oct 2021 19:31:38 +0000 Subject: [PATCH 06/54] update code comment --- src/formpack/pack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 0318241e..a5dab24d 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -239,8 +239,8 @@ def get_fields_for_versions(self, versions=-1, data_types=None): # This dict is used to remember final position of each field. Its keys # are the combination of field and section names and the values are # tuples of coordinates in tmp2d Keeping example above: - # `positions[f'{field1.name}_{section.name}']` would be `(0, 0)` - # `positions[f'{field2.name}_{section.name}']` would be `(0, 1)` + # `positions[f'{section.name}_{field1.name}']` would be `(0, 0)` + # `positions[f'{section.name}_{field2.name}']` would be `(0, 1)` positions = {} # Create the initial field mappings from the first form version From 8a6300cc091a0cefecf427d1d9e78caff614c96e Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 28 Oct 2021 22:48:21 +0000 Subject: [PATCH 07/54] make requested changes --- src/formpack/pack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index a5dab24d..dc52968f 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -236,9 +236,10 @@ def get_fields_for_versions(self, versions=-1, data_types=None): # # Index 0 of tmp2d will be `[field1, field2]` tmp2d = [] - # This dict is used to remember final position of each field. Its keys + # This dict is used to remember final position of each field. Its keys # are the combination of field and section names and the values are - # tuples of coordinates in tmp2d Keeping example above: + # tuples of coordinates in tmp2d. + # Keeping example above: # `positions[f'{section.name}_{field1.name}']` would be `(0, 0)` # `positions[f'{section.name}_{field2.name}']` would be `(0, 1)` positions = {} From 66adc806be15d3a893b2564f25771467f0023c1a Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 28 Oct 2021 22:59:47 +0000 Subject: [PATCH 08/54] make requested changes --- tests/fixtures/nested_grouped_repeatable/__init__.py | 5 +++-- tests/fixtures/nested_grouped_repeatable/v1.json | 9 +-------- tests/test_exports.py | 8 ++++---- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/fixtures/nested_grouped_repeatable/__init__.py b/tests/fixtures/nested_grouped_repeatable/__init__.py index 85a7287d..2e57fb8c 100644 --- a/tests/fixtures/nested_grouped_repeatable/__init__.py +++ b/tests/fixtures/nested_grouped_repeatable/__init__.py @@ -1,8 +1,8 @@ # coding: utf-8 -''' +""" nested_grouped_repeatable -''' +""" from ..load_fixture_json import load_fixture_json @@ -11,5 +11,6 @@ 'id_string': 'nested_grouped_repeatable', 'versions': [ load_fixture_json('nested_grouped_repeatable/v1'), + load_fixture_json('nested_grouped_repeatable/v2'), ], } diff --git a/tests/fixtures/nested_grouped_repeatable/v1.json b/tests/fixtures/nested_grouped_repeatable/v1.json index ef862af5..bdcf3e09 100644 --- a/tests/fixtures/nested_grouped_repeatable/v1.json +++ b/tests/fixtures/nested_grouped_repeatable/v1.json @@ -29,13 +29,6 @@ "label": [ "Spruce" ] - }, - { - "list_name": "tree_type", - "name": "nan", - "label": [ - "Not a tree" - ] } ], "survey": [ @@ -206,7 +199,7 @@ "group_tree/group_nest/group_egg_count": "1" } ], - "group_tree/What_kind_of_tree_is_this": "nan" + "group_tree/What_kind_of_tree_is_this": "maple" } ], "__version__": "bird_nests_v1", diff --git a/tests/test_exports.py b/tests/test_exports.py index 923990a2..f538b79e 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -789,7 +789,7 @@ def test_nested_repeats_with_copy_fields(self): 'validation_status_approved', ], [ - 'nan', + 'maple', 3, 'Bird nest survey with nested repeatable groups', 2, @@ -978,7 +978,7 @@ def test_nested_repeats(self): 1 ], [ - 'nan', + 'maple', 3, 'Bird nest survey with nested repeatable groups', 2 @@ -1075,7 +1075,7 @@ def test_nested_repeats_with_xls_types(self): title, schemas, submissions = build_fixture( 'nested_grouped_repeatable') fp = FormPack(schemas, title) - options = {'versions': 'bird_nests_v1', 'xls_types_as_text': False} + options = {'versions': 'bird_nests_v2', 'xls_types_as_text': False} export_dict = fp.export(**options).to_dict(submissions) expected_dict = OrderedDict([ ('Bird nest survey with nested repeatable groups', { @@ -1843,7 +1843,7 @@ def test_xlsx(self): def test_xlsx_with_types(self): title, schemas, submissions = build_fixture('nested_grouped_repeatable') fp = FormPack(schemas, title) - options = {'versions': 'bird_nests_v1', 'xls_types_as_text': False} + options = {'versions': 'bird_nests_v2', 'xls_types_as_text': False} with TempDir() as d: xls = d / 'foo.xlsx' From 2352c61425686d934ca4b58881a857105925b5ab Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 28 Oct 2021 23:00:45 +0000 Subject: [PATCH 09/54] add v2 fixture --- .../nested_grouped_repeatable/v2.json | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 tests/fixtures/nested_grouped_repeatable/v2.json diff --git a/tests/fixtures/nested_grouped_repeatable/v2.json b/tests/fixtures/nested_grouped_repeatable/v2.json new file mode 100644 index 00000000..0b627280 --- /dev/null +++ b/tests/fixtures/nested_grouped_repeatable/v2.json @@ -0,0 +1,216 @@ +{ + "version": "bird_nests_v2", + "content": { + "choices": [ + { + "list_name": "tree_type", + "name": "pine", + "label": [ + "Pine" + ] + }, + { + "list_name": "tree_type", + "name": "oak", + "label": [ + "Oak" + ] + }, + { + "list_name": "tree_type", + "name": "maple", + "label": [ + "Maple" + ] + }, + { + "list_name": "tree_type", + "name": "spruce", + "label": [ + "Spruce" + ] + }, + { + "list_name": "tree_type", + "name": "nan", + "label": [ + "Not a tree" + ] + } + ], + "survey": [ + { + "type": "start", + "name": "start" + }, + { + "type": "end", + "name": "end" + }, + { + "type": "begin_repeat", + "name": "group_tree", + "label": [ + "For each tree" + ] + }, + { + "select_from_list_name": "tree_type", + "name": "What_kind_of_tree_is_this", + "label": [ + "What kind of tree is this?" + ], + "type": "select_one" + }, + { + "type": "begin_repeat", + "name": "group_nest", + "label": [ + "For each nest" + ] + }, + { + "type": "integer", + "name": "How_high_above_the_ground_is_the_nest", + "label": [ + "How high above the ground is the nest?" + ] + }, + { + "type": "integer", + "name": "How_many_eggs_are_in_the_nest", + "label": [ + "How many eggs are in the nest?" + ] + }, + { + "name": "group_egg", + "label": [ + "For each egg" + ], + "repeat_count": "${How_many_eggs_are_in_the_nest}", + "type": "begin_repeat" + }, + { + "type": "text", + "name": "Describe_the_egg", + "label": [ + "Describe the egg" + ] + }, + { + "type": "end_repeat" + }, + { + "type": "end_repeat" + }, + { + "type": "end_repeat" + } + ] + }, + "submissions": [ + { + "_validation_status": { + "by_whom": "user1", + "timestamp": 1531158682, + "uid": "validation_status_approved", + "color": "#00ff00", + "label":"Approved" + }, + "meta/instanceID": "uuid:f16d9a3f-0892-413e-81d4-758ab188ea0b", + "end": "2017-12-27T15:58:20.000-05:00", + "_submission_time": "2017-12-27T20:58:25", + "_id": 123, + "_uuid": "f16d9a3f-0892-413e-81d4-758ab188ea0b", + "start": "2017-12-27T15:53:26.000-05:00", + "group_tree": [ + { + "group_tree/group_nest": [ + { + "group_tree/group_nest/How_high_above_the_ground_is_the_nest": "13", + "group_tree/group_nest/group_egg": [ + { + "group_tree/group_nest/group_egg/Describe_the_egg": "brown and speckled; medium" + }, + { + "group_tree/group_nest/group_egg/Describe_the_egg": "brown and speckled; large; cracked" + }, + { + "group_tree/group_nest/group_egg/Describe_the_egg": "light tan; small" + } + ], + "group_tree/group_nest/How_many_eggs_are_in_the_nest": "3", + "group_tree/group_nest/group_egg_count": "3" + }, + { + "group_tree/group_nest/How_high_above_the_ground_is_the_nest": "15", + "group_tree/group_nest/group_egg": [ + { + "group_tree/group_nest/group_egg/Describe_the_egg": "cream-colored" + } + ], + "group_tree/group_nest/How_many_eggs_are_in_the_nest": "1", + "group_tree/group_nest/group_egg_count": "1" + } + ], + "group_tree/What_kind_of_tree_is_this": "pine" + }, + { + "group_tree/group_nest": [ + { + "group_tree/group_nest/How_high_above_the_ground_is_the_nest": "10", + "group_tree/group_nest/group_egg": [ + { + "group_tree/group_nest/group_egg/Describe_the_egg": "reddish-brown; medium" + }, + { + "group_tree/group_nest/group_egg/Describe_the_egg": "reddish-brown; small" + } + ], + "group_tree/group_nest/How_many_eggs_are_in_the_nest": "2", + "group_tree/group_nest/group_egg_count": "2" + } + ], + "group_tree/What_kind_of_tree_is_this": "spruce" + } + ], + "__version__": "bird_nests_v1", + "formhub/uuid": "ddd2f25ff0f5464a9f91ad65130f1460" + }, + { + "_validation_status": { + "by_whom": "user1", + "timestamp": 1531158682, + "uid": "validation_status_not_approved", + "color": "#00ff00", + "label":"Not approved" + }, + "meta/instanceID": "uuid:790af158-7b24-4651-b584-27bf65b9e397", + "end": "2017-12-27T15:58:50.000-05:00", + "_submission_time": "2017-12-27T20:58:55", + "_id": 124, + "_uuid": "790af158-7b24-4651-b584-27bf65b9e397", + "start": "2017-12-27T15:58:20.000-05:00", + "group_tree": [ + { + "group_tree/group_nest": [ + { + "group_tree/group_nest/How_high_above_the_ground_is_the_nest": "23", + "group_tree/group_nest/group_egg": [ + { + "group_tree/group_nest/group_egg/Describe_the_egg": "grey and speckled" + } + ], + "group_tree/group_nest/How_many_eggs_are_in_the_nest": "1", + "group_tree/group_nest/group_egg_count": "1" + } + ], + "group_tree/What_kind_of_tree_is_this": "nan" + } + ], + "__version__": "bird_nests_v1", + "formhub/uuid": "ddd2f25ff0f5464a9f91ad65130f1460" + } + ] +} From 6e238efb851a9f76ac4c8b9756d3e6b219759728 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 12 Nov 2021 18:09:11 +0000 Subject: [PATCH 10/54] clean slate with fixtures --- src/formpack/constants.py | 2 + tests/fixtures/analysis_form/__init__.py | 16 + .../fixtures/analysis_form/analysis_form.json | 46 ++ tests/fixtures/analysis_form/v1.json | 76 +++ tests/fixtures/analysis_form/v2.json | 76 +++ .../analysis_form_advanced/__init__.py | 15 + .../analysis_form_advanced/analysis_form.json | 96 ++++ tests/fixtures/analysis_form_advanced/v1.json | 96 ++++ tests/fixtures/build_fixture.py | 4 + tests/fixtures/load_fixture_json.py | 4 + tests/test_additional_field_exports.py | 431 ++++++++++++++++++ tests/test_fixtures_valid.py | 21 + 12 files changed, 883 insertions(+) create mode 100644 tests/fixtures/analysis_form/__init__.py create mode 100644 tests/fixtures/analysis_form/analysis_form.json create mode 100644 tests/fixtures/analysis_form/v1.json create mode 100644 tests/fixtures/analysis_form/v2.json create mode 100644 tests/fixtures/analysis_form_advanced/__init__.py create mode 100644 tests/fixtures/analysis_form_advanced/analysis_form.json create mode 100644 tests/fixtures/analysis_form_advanced/v1.json create mode 100644 tests/test_additional_field_exports.py diff --git a/src/formpack/constants.py b/src/formpack/constants.py index 3a038e9a..92c10354 100644 --- a/src/formpack/constants.py +++ b/src/formpack/constants.py @@ -59,6 +59,7 @@ EXPORT_SETTING_FLATTEN = 'flatten' EXPORT_SETTING_GROUP_SEP = 'group_sep' EXPORT_SETTING_HIERARCHY_IN_LABELS = 'hierarchy_in_labels' +EXPORT_SETTING_INCLUDE_ANALYSIS_FIELDS = 'include_analysis_fields' EXPORT_SETTING_INCLUDE_MEDIA_URL = 'include_media_url' EXPORT_SETTING_LANG = 'lang' EXPORT_SETTING_MULTIPLE_SELECT = 'multiple_select' @@ -71,6 +72,7 @@ OPTIONAL_EXPORT_SETTINGS = [ EXPORT_SETTING_FIELDS, EXPORT_SETTING_FLATTEN, + EXPORT_SETTING_INCLUDE_ANALYSIS_FIELDS, EXPORT_SETTING_INCLUDE_MEDIA_URL, EXPORT_SETTING_NAME, EXPORT_SETTING_QUERY, diff --git a/tests/fixtures/analysis_form/__init__.py b/tests/fixtures/analysis_form/__init__.py new file mode 100644 index 00000000..2f02da8f --- /dev/null +++ b/tests/fixtures/analysis_form/__init__.py @@ -0,0 +1,16 @@ +# coding: utf-8 +''' +analysis_form +''' + +from ..load_fixture_json import load_fixture_json, load_analysis_form_json + +DATA = { + 'title': 'Simple Clerk Interaction', + 'id_string': 'cerk_interaction', + 'versions': [ + load_fixture_json('analysis_form/v1'), + load_fixture_json('analysis_form/v2'), + ], + 'analysis_form': load_analysis_form_json('analysis_form') +} diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json new file mode 100644 index 00000000..147d7450 --- /dev/null +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -0,0 +1,46 @@ +{ + "engines": { + "acme_1_speech2text": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note_transcription_acme_1_speech2text", + "label": [ + ": ACME Transcription", + "" + ], + "source": "record_a_note", + "engine": "engines/acme_1_speech2text" + }, + { + "type": "datetime", + "name": "record_a_note_acme_timestamp", + "label": [ + "Transcription Timestamp", + "" + ], + "source": "record_a_note" + }, + { + "type": "text", + "name": "name_of_clerk_comment", + "label": [ + "Comment on the name of the clerk", + "" + ], + "source": "name_of_clerk" + }, + { + "type": "text", + "name": "name_of_shop_comment", + "label": [ + "Comment on the name of the shop", + "" + ], + "source": "name_of_shop" + } + ] +} diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json new file mode 100644 index 00000000..2e144bfd --- /dev/null +++ b/tests/fixtures/analysis_form/v1.json @@ -0,0 +1,76 @@ +{ + "version": "v1", + "content": { + "survey": [ + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "begin_group", + "name": "clerk_details", + "label": [ + "Some details of the clerk", + "Kelkaj detaloj de la oficisto" + ] + }, + { + "type": "text", + "name": "name_of_clerk", + "label": [ + "What is the clerk's name?", + "" + ] + }, + { + "type": "end_group" + } + ], + "settings": { + "version": "v1" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "record_a_note": "clerk_interaction_1.mp3", + "clerk_details/name_of_clerk": "John", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_1.mp3"} + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": "Hello how may I help you?", + "record_a_note_acme_timestamp": "2021-11-01Z", + "name_of_clerk_comment": "Sounds like an interesting person" + } + }, + { + "record_a_note": "clerk_interaction_2.mp3", + "clerk_details/name_of_clerk": "Alex", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_2.mp3"} + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": "Thank you for your business" + } + }, + { + "record_a_note": "clerk_interaction_3.mp3", + "clerk_details/name_of_clerk": "Olivier", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_3.mp3"} + ], + "_supplementalDetails": {} + } + ] +} diff --git a/tests/fixtures/analysis_form/v2.json b/tests/fixtures/analysis_form/v2.json new file mode 100644 index 00000000..15759009 --- /dev/null +++ b/tests/fixtures/analysis_form/v2.json @@ -0,0 +1,76 @@ +{ + "version": "v2", + "content": { + "survey": [ + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "begin_group", + "name": "clerk_details", + "label": [ + "Some details of the clerk", + "Kelkaj detaloj de la oficisto" + ] + }, + { + "type": "text", + "name": "name_of_shop", + "label": [ + "What is the shop's name?", + "Kio estas la nomo de la butiko?" + ] + }, + { + "type": "end_group" + } + ], + "settings": { + "version": "v2" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "record_a_note": "clerk_interaction_4.mp3", + "clerk_details/name_of_shop": "Save On", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_4.mp3"} + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": "Hello how may I help you?", + "record_a_note_acme_timestamp": "2021-11-01Z", + "name_of_shop_comment": "Pretty cliche" + } + }, + { + "record_a_note": "clerk_interaction_5.mp3", + "clerk_details/name_of_shop": "Walmart", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_5.mp3"} + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": "Thank you for your business" + } + }, + { + "record_a_note": "clerk_interaction_6.mp3", + "clerk_details/name_of_shop": "Costco", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_6.mp3"} + ], + "_supplementalDetails": {} + } + ] +} diff --git a/tests/fixtures/analysis_form_advanced/__init__.py b/tests/fixtures/analysis_form_advanced/__init__.py new file mode 100644 index 00000000..13f37942 --- /dev/null +++ b/tests/fixtures/analysis_form_advanced/__init__.py @@ -0,0 +1,15 @@ +# coding: utf-8 +''' +analysis_form_advanced +''' + +from ..load_fixture_json import load_fixture_json, load_analysis_form_json + +DATA = { + 'title': 'Advanced Clerk Interaction', + 'id_string': 'cerk_interaction_advanced', + 'versions': [ + load_fixture_json('analysis_form_advanced/v1'), + ], + 'analysis_form': load_analysis_form_json('analysis_form_advanced') +} diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json new file mode 100644 index 00000000..f1d31106 --- /dev/null +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -0,0 +1,96 @@ +{ + "engines": { + "acme_1_speech2text": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note_transcription_acme_1_speech2text", + "label": [ + ": ACME Transcription", + "" + ], + "source": "record_a_note", + "engine": "engines/acme_1_speech2text" + }, + { + "type": "select_multiple record_a_note_tones", + "name": "record_a_note_tone_of_voice", + "label": [ + "How was the tone of the clerk's voice?", + "Kiel estis la tono de la voĉo de la oficisto?" + ], + "source": "record_a_note" + }, + { + "type": "text", + "name": "goods_sold_comment", + "label": [ + "Comment on the goods sold at the store", + "Komentu la varojn venditajn en la vendejo" + ], + "source": "goods_sold" + }, + { + "type": "select_one goods_sold_ratings", + "name": "goods_sold_rating", + "label": [ + "Rate the quality of the goods sold at the store", + "Komentu la varojn vendojn en la vendejo" + ], + "source": "goods_sold" + } + ], + "additional_choices": [ + { + "list_name": "goods_sold_ratings", + "name": 1, + "label": [ + "Poor quality", + "Malbona kvalito" + ] + }, + { + "list_name": "goods_sold_ratings", + "name": 2, + "label": [ + "Average quality", + "Meza kvalito" + ] + }, + { + "list_name": "ratings", + "name": 3, + "label": [ + "High quality", + "Alta kvalito" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "anxious", + "label": [ + "Anxious", + "Maltrankvila" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "excited", + "label": [ + "Excited", + "Ekscitita" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "confused", + "label": [ + "Confused", + "Konfuzita" + ] + } + ] +} diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json new file mode 100644 index 00000000..2c87aa51 --- /dev/null +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -0,0 +1,96 @@ +{ + "version": "v1", + "content": { + "survey": [ + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "select_multiple goods", + "name": "goods_sold", + "label": [ + "What are some goods sold at the store?", + "Kio estas iuj varoj venditaj en la vendejo?" + ] + } + ], + "choices": [ + { + "list_name": "goods", + "name": "chocolate", + "label": [ + "Chocolate", + "Ĉokolado" + ] + }, + { + "list_name": "goods", + "name": "fruit", + "label": [ + "Fruit", + "Frukto" + ] + }, + { + "list_name": "goods", + "name": "pasta", + "label": [ + "Pasta", + "Pasto" + ] + } + ], + "settings": { + "version": "v1" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "record_a_note": "clerk_interaction_1.mp3", + "goods_sold": "chocolate", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_1.mp3"} + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": "Hello how may I help you?", + "record_a_note_tone_of_voice": "excited confused", + "goods_sold_comment": "Not much diversity", + "goods_sold_rating": "3" + } + }, + { + "record_a_note": "clerk_interaction_2.mp3", + "goods_sold": "chocolate fruit pasta", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_2.mp3"} + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": "Thank you for your business", + "record_a_note_tone_of_voice": "anxious excited", + "goods_sold_rating": "2" + } + }, + { + "record_a_note": "clerk_interaction_3.mp3", + "goods_sold": "pasta", + "_attachments": [ + {"mimetype": "audio/mpeg", "filename": "clerk_interaction_3.mp3"} + ], + "_supplementalDetails": { + "goods_sold_rating": "3" + } + } + ] +} diff --git a/tests/fixtures/build_fixture.py b/tests/fixtures/build_fixture.py index d2c8657d..85017917 100644 --- a/tests/fixtures/build_fixture.py +++ b/tests/fixtures/build_fixture.py @@ -16,6 +16,7 @@ def build_fixture(modulename, data_variable_name="DATA"): return fixtures title = fixtures.get('title') + analysis_form = fixtures.get('analysis_form') # separate the submissions from the schema schemas = [dict(v) for v in fixtures['versions']] @@ -26,6 +27,9 @@ def build_fixture(modulename, data_variable_name="DATA"): for submission in schema.pop('submissions'): submission.update({version_id_key: version}) submissions.append(submission) + + if analysis_form is not None: + return title, schemas, submissions, analysis_form return title, schemas, submissions diff --git a/tests/fixtures/load_fixture_json.py b/tests/fixtures/load_fixture_json.py index 83fc3023..51b8ecd5 100644 --- a/tests/fixtures/load_fixture_json.py +++ b/tests/fixtures/load_fixture_json.py @@ -8,3 +8,7 @@ def load_fixture_json(fname): with open(os.path.join(CUR_DIR, '%s.json' % fname)) as ff: content_ = ff.read() return json.loads(content_) + +def load_analysis_form_json(path): + with open(os.path.join(CUR_DIR, path, 'analysis_form.json')) as f: + return json.loads(f.read()) diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py new file mode 100644 index 00000000..64887350 --- /dev/null +++ b/tests/test_additional_field_exports.py @@ -0,0 +1,431 @@ +# coding: utf-8 +from formpack import FormPack +from .fixtures import build_fixture + +def tests_additional_field_exports(): + title, schemas, submissions, analysis_form = build_fixture('analysis_form') + pack = FormPack(schemas, title=title, analysis_form=analysis_form) + + options = {'include_analysis_fields': True, 'versions': 'v1'} + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_acme_timestamp', + 'name_of_clerk', + 'name_of_clerk_comment', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + '2021-11-01Z', + 'John', + 'Sounds like an interesting person', + ] + +def tests_additional_field_exports_advanced(): + title, schemas, submissions, analysis_form = build_fixture( + 'analysis_form_advanced' + ) + pack = FormPack(schemas, title=title, analysis_form=analysis_form) + + options = { + 'include_analysis_fields': True, + 'versions': 'v1', + 'multiple_select': 'both', + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Advanced Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_tone_of_voice', + 'record_a_note_tone_of_voice/anxious', + 'record_a_note_tone_of_voice/excited', + 'record_a_note_tone_of_voice/confused', + 'goods_sold', + 'goods_sold/chocolate', + 'goods_sold/fruit', + 'goods_sold/pasta', + 'goods_sold_comment', + 'goods_sold_rating', + ] + assert main_export_sheet['data'] == [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + 'excited confused', + '0', + '1', + '1', + 'chocolate', + '1', + '0', + '0', + 'Not much diversity', + '3', + ], + [ + 'clerk_interaction_2.mp3', + 'Thank you for your business', + 'anxious excited', + '1', + '1', + '0', + 'chocolate fruit pasta', + '1', + '1', + '1', + '', + '2', + ], + [ + 'clerk_interaction_3.mp3', + '', + '', + '', + '', + '', + 'pasta', + '0', + '0', + '1', + '', + '3', + ], + ] + + options['multiple_select'] = 'details' + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Advanced Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_tone_of_voice/anxious', + 'record_a_note_tone_of_voice/excited', + 'record_a_note_tone_of_voice/confused', + 'goods_sold/chocolate', + 'goods_sold/fruit', + 'goods_sold/pasta', + 'goods_sold_comment', + 'goods_sold_rating', + ] + assert main_export_sheet['data'] == [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + '0', + '1', + '1', + '1', + '0', + '0', + 'Not much diversity', + '3', + ], + [ + 'clerk_interaction_2.mp3', + 'Thank you for your business', + '1', + '1', + '0', + '1', + '1', + '1', + '', + '2', + ], + [ + 'clerk_interaction_3.mp3', + '', + '', + '', + '', + '0', + '0', + '1', + '', + '3', + ], + ] + + options['multiple_select'] = 'summary' + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Advanced Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_tone_of_voice', + 'goods_sold', + 'goods_sold_comment', + 'goods_sold_rating', + ] + assert main_export_sheet['data'] == [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + 'excited confused', + 'chocolate', + 'Not much diversity', + '3', + ], + [ + 'clerk_interaction_2.mp3', + 'Thank you for your business', + 'anxious excited', + 'chocolate fruit pasta', + '', + '2', + ], + [ + 'clerk_interaction_3.mp3', + '', + '', + 'pasta', + '', + '3', + ], + ] + +def tests_additional_field_exports_v2(): + title, schemas, submissions, analysis_form = build_fixture('analysis_form') + pack = FormPack(schemas, title=title, analysis_form=analysis_form) + + options = {'include_analysis_fields': True, 'versions': 'v2'} + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_acme_timestamp', + 'name_of_shop', + 'name_of_shop_comment', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_4.mp3', + 'Hello how may I help you?', + '2021-11-01Z', + 'Save On', + 'Pretty cliche', + ] + +def tests_additional_field_exports_all_versions(): + title, schemas, submissions, analysis_form = build_fixture('analysis_form') + pack = FormPack(schemas, title=title, analysis_form=analysis_form) + + options = {'include_analysis_fields': True, 'versions': pack.versions} + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 6 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_acme_timestamp', + 'name_of_shop', + 'name_of_shop_comment', + 'name_of_clerk', + 'name_of_clerk_comment', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + '2021-11-01Z', + '', + '', + 'John', + 'Sounds like an interesting person', + ] + response3 = main_export_sheet['data'][3] + assert response3 == [ + 'clerk_interaction_4.mp3', + 'Hello how may I help you?', + '2021-11-01Z', + 'Save On', + 'Pretty cliche', + '', + '', + ] + +def tests_additional_field_exports_all_versions_exclude_fields(): + title, schemas, submissions, analysis_form = build_fixture('analysis_form') + pack = FormPack(schemas, title=title, analysis_form=analysis_form) + + options = {'versions': pack.versions} + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 6 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'name_of_shop', + 'name_of_clerk', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + '', + 'John', + ] + response3 = main_export_sheet['data'][3] + assert response3 == [ + 'clerk_interaction_4.mp3', + 'Save On', + '', + ] + +def tests_additional_field_exports_all_versions_langs(): + title, schemas, submissions, analysis_form = build_fixture('analysis_form') + pack = FormPack(schemas, title=title, analysis_form=analysis_form) + + options = { + 'include_analysis_fields': True, + 'versions': pack.versions, + 'lang': 'English (en)', + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'Record a clerk saying something', + ': ACME Transcription', + 'Transcription Timestamp', + "What is the shop's name?", + 'Comment on the name of the shop', + "What is the clerk's name?", + 'Comment on the name of the clerk', + ] + + options['lang'] = 'Esperanto (es)' + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'Registri oficiston dirantan ion', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_acme_timestamp', + 'Kio estas la nomo de la butiko?', + 'name_of_shop_comment', + 'name_of_clerk', + 'name_of_clerk_comment', + ] + + options['lang'] = None + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note_acme_timestamp', + 'name_of_shop', + 'name_of_shop_comment', + 'name_of_clerk', + 'name_of_clerk_comment', + ] + +def test_simple_report_with_analysis_form(): + + title, schemas, submissions, analysis_form = build_fixture('analysis_form') + fp = FormPack(schemas, title, analysis_form=analysis_form) + + report = fp.autoreport(versions=fp.versions.keys()) + stats = report.get_stats(submissions, lang='English (en)') + + assert stats.submissions_count == 6 + + stats = [(str(repr(f)), n, d) for f, n, d in stats] + + expected = [ + ( + "", + 'record_a_note', + { + 'total_count': 6, + 'not_provided': 0, + 'provided': 6, + 'show_graph': False, + 'frequency': [ + ('clerk_interaction_1.mp3', 1), + ('clerk_interaction_2.mp3', 1), + ('clerk_interaction_3.mp3', 1), + ('clerk_interaction_4.mp3', 1), + ('clerk_interaction_5.mp3', 1), + ('clerk_interaction_6.mp3', 1), + ], + 'percentage': [ + ('clerk_interaction_1.mp3', 16.67), + ('clerk_interaction_2.mp3', 16.67), + ('clerk_interaction_3.mp3', 16.67), + ('clerk_interaction_4.mp3', 16.67), + ('clerk_interaction_5.mp3', 16.67), + ('clerk_interaction_6.mp3', 16.67), + ], + }, + ), + ( + "", + "What is the shop's name?", + { + 'total_count': 6, + 'not_provided': 3, + 'provided': 3, + 'show_graph': False, + 'frequency': [ + ('Save On', 1), + ('Walmart', 1), + ('Costco', 1), + ], + 'percentage': [ + ('Save On', 16.67), + ('Walmart', 16.67), + ('Costco', 16.67), + ], + }, + ), + ( + "", + "What is the clerk's name?", + { + 'total_count': 6, + 'not_provided': 3, + 'provided': 3, + 'show_graph': False, + 'frequency': [ + ('John', 1), + ('Alex', 1), + ('Olivier', 1), + ], + 'percentage': [ + ('John', 16.67), + ('Alex', 16.67), + ('Olivier', 16.67), + ], + }, + ), + ] + + for i, stat in enumerate(stats): + assert stat == expected[i] diff --git a/tests/test_fixtures_valid.py b/tests/test_fixtures_valid.py index 398d1ebf..5646b653 100644 --- a/tests/test_fixtures_valid.py +++ b/tests/test_fixtures_valid.py @@ -106,3 +106,24 @@ def test_xml_instances_loaded(self): fp = FormPack(**build_fixture('favcolor')) self.assertEqual(len(fp.versions), 2) + def test_analysis_form(self): + fixture = build_fixture('analysis_form') + assert 4 == len(fixture) + + title, schemas, submissions, analysis_form = fixture + fp = FormPack(schemas, title, analysis_form=analysis_form) + assert 2 == len(fp.versions) + assert 'Simple Clerk Interaction' == title + + expected_analysis_questions = sorted( + [f['name'] for f in analysis_form['additional_fields']] + ) + actual_analysis_questions = sorted( + [ + f.name + for f in fp.get_fields_for_versions(fp.versions) + if f.analysis_question + ] + ) + assert expected_analysis_questions == actual_analysis_questions + From 7273fddd246de5d81429a4095aa0dff7e204473a Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 12 Nov 2021 20:21:10 +0000 Subject: [PATCH 11/54] wip AnalysisForm class and basic passing test --- src/formpack/pack.py | 11 +++++++- src/formpack/reporting/export.py | 11 +++++++- src/formpack/schema/fields.py | 12 ++++++-- src/formpack/version.py | 38 ++++++++++++++++++++++++++ tests/test_additional_field_exports.py | 8 ++++-- 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index dc52968f..fb5e6aef 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -5,7 +5,7 @@ from copy import deepcopy from formpack.schema.fields import CopyField -from .version import FormVersion +from .version import FormVersion, AnalysisForm from .reporting import Export, AutoReport from .utils.expand_content import expand_content from .utils.replace_aliases import replace_aliases @@ -49,6 +49,8 @@ def __init__(self, versions=None, title='Submissions', id_string=None, self.asset_type = asset_type + self.analysis_form = None + self.load_all_versions(versions) # FIXME: Find a safe way to use this. Wrapping with try/except isn't enough @@ -164,6 +166,10 @@ def load_version(self, schema): self.versions[form_version.id] = form_version + def extend_survey(self, analysis_form): + af = AnalysisForm(self, analysis_form) + self.analysis_form = af + def version_diff(self, vn1, vn2): v1 = self.versions[vn1] v2 = self.versions[vn2] @@ -313,6 +319,9 @@ def get_fields_for_versions(self, versions=-1, data_types=None): else: all_fields.append(field) + if self.analysis_form: + all_fields = self.analysis_form.insert_analysis_fields(all_fields) + # Finally, add copy fields at the end all_fields += copy_fields diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 7e5f6344..f8e666f4 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -64,6 +64,7 @@ def __init__( """ self.formpack = formpack + self.analysis_form = formpack.analysis_form self.lang = lang self.group_sep = group_sep self.title = title @@ -349,6 +350,9 @@ def format_one_submission( row = self._row_cache[_section_name] _fields = tuple(current_section.fields.values()) + if self.analysis_form: + _fields = self.analysis_form.insert_analysis_fields(_fields) + def _get_attachment(val, field, attachments): """ Filter attachments for filenames that match the submission field's @@ -371,6 +375,9 @@ def _get_attachment(val, field, attachments): ] def _get_value_from_entry(entry, field): + if field.analysis_question and '_supplementalDetails' in entry: + return entry['_supplementalDetails'].get(field.name) + suffix = 'meta/' if field.data_type == 'audit' else '' return entry.get(f'{suffix}{field.path}') @@ -379,10 +386,12 @@ def _get_value_from_entry(entry, field): if self.filter_fields: _fields = tuple( field - for field in current_section.fields.values() + for field in _fields if field.path in self.filter_fields ) + + # 'rows' will contain all the formatted entries for the current # section. If you don't have repeat-group, there is only one section # with a row of size one. diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 24b73e69..d4b8da97 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -24,6 +24,12 @@ def __init__(self, name, labels, data_type, hierarchy=None, self.section = section self.can_format = can_format self.tags = kwargs.get('tags', []) + self.analysis_question = False + + source = kwargs.get('source') + if source is not None: + self.source = source + self.analysis_question = True hierarchy = list(hierarchy) if hierarchy is not None else [None] self.hierarchy = hierarchy + [self] @@ -34,7 +40,7 @@ def __init__(self, name, labels, data_type, hierarchy=None, if has_stats is not None: self.has_stats = has_stats else: - self.has_stats = data_type != "note" + self.has_stats = data_type != "note" and not self.analysis_question # do not include the root section in the path self.path = '/'.join(info.name for info in self.hierarchy[1:]) @@ -120,7 +126,7 @@ def _get_label(self, lang=UNSPECIFIED_TRANSLATION, group_sep='/', # even if `lang` can be None, we don't want the `label` to be None. label = self.labels.get(lang, self.name) # If `label` is None, no matches are found, so return `field` name. - return self.name if label is None else label + return label or self.name def __repr__(self): args = (self.__class__.__name__, self.name, self.data_type) @@ -153,6 +159,7 @@ def from_json_definition(cls, definition, hierarchy=None, labels = cls._extract_json_labels(definition, translations) appearance = definition.get('appearance') or_other = definition.get('_or_other', False) + source = definition.get('source') # normalize spaces data_type = definition['type'] @@ -227,6 +234,7 @@ def from_json_definition(cls, definition, hierarchy=None, 'section': section, 'choice': choice, 'or_other': or_other, + 'source': source, } if data_type == 'select_multiple' and appearance == 'literacy': diff --git a/src/formpack/version.py b/src/formpack/version.py index 08d0e87e..821d6cae 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -317,3 +317,41 @@ def to_xml(self, warnings=None): }) return survey._to_pretty_xml() #.encode('utf-8') + + +class AnalysisForm: + + def __init__(self, form_pack, schema): + + self.schema = schema + self.form_pack = form_pack + + survey = self.schema.get('additional_fields', []) + section = FormSection(name=form_pack.title) + + self.fields = [ + FormField.from_json_definition(dd, translations=[None], section=section) + for dd in survey + ] + + self.fields_by_source = self._get_fields_by_source() + + def __repr__(self): + return f"" + + def _get_fields_by_source(self): + fields_by_source = {} + for field in self.fields: + if field.source not in fields_by_source: + fields_by_source[field.source] = [field] + else: + fields_by_source[field.source].append(field) + return fields_by_source + + def insert_analysis_fields(self, fields): + _fields = [] + for field in fields: + _fields.append(field) + if field.name in self.fields_by_source: + _fields += self.fields_by_source[field.name] + return _fields diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 64887350..c9ff6519 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -2,11 +2,13 @@ from formpack import FormPack from .fixtures import build_fixture -def tests_additional_field_exports(): +def tests_additional_field_exports_xxx(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') - pack = FormPack(schemas, title=title, analysis_form=analysis_form) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) - options = {'include_analysis_fields': True, 'versions': 'v1'} + #options = {'include_analysis_fields': True, 'versions': 'v1'} + options = {'versions': 'v1'} export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] From 1e7233b8cf4b28e23c15200a46b1c20230f16177 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 12 Nov 2021 22:24:22 +0000 Subject: [PATCH 12/54] handle translations and choices, get all tests to pass again --- src/formpack/pack.py | 7 +++--- src/formpack/reporting/export.py | 9 +++++++- src/formpack/version.py | 18 ++++++++++++++- .../fixtures/analysis_form/analysis_form.json | 15 ++++++------ .../analysis_form_advanced/analysis_form.json | 10 ++++++-- tests/test_additional_field_exports.py | 23 +++++++++++-------- 6 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index fb5e6aef..80b474fb 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -319,9 +319,6 @@ def get_fields_for_versions(self, versions=-1, data_types=None): else: all_fields.append(field) - if self.analysis_form: - all_fields = self.analysis_form.insert_analysis_fields(all_fields) - # Finally, add copy fields at the end all_fields += copy_fields @@ -356,6 +353,7 @@ def export( filter_fields=(), xls_types_as_text=True, include_media_url=False, + include_analysis_fields=False, ): """ Create an export for given versions of the form @@ -376,7 +374,8 @@ def export( tag_cols_for_header=tag_cols_for_header, filter_fields=filter_fields, xls_types_as_text=xls_types_as_text, - include_media_url=include_media_url + include_media_url=include_media_url, + include_analysis_fields=include_analysis_fields, ) def autoreport(self, versions=-1): diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index f8e666f4..c72911dc 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -43,6 +43,7 @@ def __init__( filter_fields=(), xls_types_as_text=True, include_media_url=False, + include_analysis_fields=False, ): """ :param formpack: FormPack @@ -61,6 +62,8 @@ def __init__( :param tag_cols_for_header: list :param filter_fields: list :param xls_types_as_text: bool + :param include_media_url: bool + :param include_analysis_fields: bool """ self.formpack = formpack @@ -77,6 +80,7 @@ def __init__( self.filter_fields = filter_fields self.xls_types_as_text = xls_types_as_text self.include_media_url = include_media_url + self.include_analysis_fields = include_analysis_fields self.__r_groups_submission_mapping_values = {} if tag_cols_for_header is None: @@ -220,6 +224,9 @@ def get_fields_labels_tags_for_all_versions(self, all_fields = self.formpack.get_fields_for_versions(self.versions) + if self.analysis_form and self.include_analysis_fields: + all_fields = self.analysis_form.insert_analysis_fields(all_fields) + # Ensure that fields are filtered if they've been specified, otherwise # carry on as usual if self.filter_fields: @@ -350,7 +357,7 @@ def format_one_submission( row = self._row_cache[_section_name] _fields = tuple(current_section.fields.values()) - if self.analysis_form: + if self.analysis_form and self.include_analysis_fields: _fields = self.analysis_form.insert_analysis_fields(_fields) def _get_attachment(val, field, attachments): diff --git a/src/formpack/version.py b/src/formpack/version.py index 821d6cae..7173f433 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -329,13 +329,29 @@ def __init__(self, form_pack, schema): survey = self.schema.get('additional_fields', []) section = FormSection(name=form_pack.title) + self.translations = [ + t if t is not None else UNTRANSLATED + for t in schema.get('translations', [None]) + ] + + choices_definition = schema.get('additional_choices', ()) + field_choices = FormChoice.all_from_json_definition( + choices_definition, self.translations + ) + self.fields = [ - FormField.from_json_definition(dd, translations=[None], section=section) + FormField.from_json_definition( + dd, + translations=self.translations, + section=section, + field_choices=field_choices, + ) for dd in survey ] self.fields_by_source = self._get_fields_by_source() + def __repr__(self): return f"" diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 147d7450..6263664d 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -9,8 +9,7 @@ "type": "text", "name": "record_a_note_transcription_acme_1_speech2text", "label": [ - ": ACME Transcription", - "" + ": ACME Transcription" ], "source": "record_a_note", "engine": "engines/acme_1_speech2text" @@ -19,8 +18,7 @@ "type": "datetime", "name": "record_a_note_acme_timestamp", "label": [ - "Transcription Timestamp", - "" + "Transcription Timestamp" ], "source": "record_a_note" }, @@ -28,8 +26,7 @@ "type": "text", "name": "name_of_clerk_comment", "label": [ - "Comment on the name of the clerk", - "" + "Comment on the name of the clerk" ], "source": "name_of_clerk" }, @@ -37,10 +34,12 @@ "type": "text", "name": "name_of_shop_comment", "label": [ - "Comment on the name of the shop", - "" + "Comment on the name of the shop" ], "source": "name_of_shop" } + ], + "translations": [ + "English (en)" ] } diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index f1d31106..5e4a91c7 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -16,7 +16,8 @@ "engine": "engines/acme_1_speech2text" }, { - "type": "select_multiple record_a_note_tones", + "type": "select_multiple", + "select_from_list_name": "record_a_note_tones", "name": "record_a_note_tone_of_voice", "label": [ "How was the tone of the clerk's voice?", @@ -34,7 +35,8 @@ "source": "goods_sold" }, { - "type": "select_one goods_sold_ratings", + "type": "select_one", + "select_from_list_name": "goods_sold_ratings", "name": "goods_sold_rating", "label": [ "Rate the quality of the goods sold at the store", @@ -92,5 +94,9 @@ "Konfuzita" ] } + ], + "translations": [ + "English (en)", + "Esperanto (es)" ] } diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index c9ff6519..5ad4a7fd 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -7,8 +7,7 @@ def tests_additional_field_exports_xxx(): pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) - #options = {'include_analysis_fields': True, 'versions': 'v1'} - options = {'versions': 'v1'} + options = {'include_analysis_fields': True, 'versions': 'v1'} export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] @@ -34,7 +33,8 @@ def tests_additional_field_exports_advanced(): title, schemas, submissions, analysis_form = build_fixture( 'analysis_form_advanced' ) - pack = FormPack(schemas, title=title, analysis_form=analysis_form) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) options = { 'include_analysis_fields': True, @@ -203,7 +203,8 @@ def tests_additional_field_exports_advanced(): def tests_additional_field_exports_v2(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') - pack = FormPack(schemas, title=title, analysis_form=analysis_form) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) options = {'include_analysis_fields': True, 'versions': 'v2'} export = pack.export(**options) @@ -229,7 +230,8 @@ def tests_additional_field_exports_v2(): def tests_additional_field_exports_all_versions(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') - pack = FormPack(schemas, title=title, analysis_form=analysis_form) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) options = {'include_analysis_fields': True, 'versions': pack.versions} export = pack.export(**options) @@ -269,7 +271,8 @@ def tests_additional_field_exports_all_versions(): def tests_additional_field_exports_all_versions_exclude_fields(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') - pack = FormPack(schemas, title=title, analysis_form=analysis_form) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) options = {'versions': pack.versions} export = pack.export(**options) @@ -297,7 +300,8 @@ def tests_additional_field_exports_all_versions_exclude_fields(): def tests_additional_field_exports_all_versions_langs(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') - pack = FormPack(schemas, title=title, analysis_form=analysis_form) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) options = { 'include_analysis_fields': True, @@ -351,9 +355,10 @@ def tests_additional_field_exports_all_versions_langs(): def test_simple_report_with_analysis_form(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') - fp = FormPack(schemas, title, analysis_form=analysis_form) + pack = FormPack(schemas, title) + pack.extend_survey(analysis_form) - report = fp.autoreport(versions=fp.versions.keys()) + report = pack.autoreport(versions=pack.versions.keys()) stats = report.get_stats(submissions, lang='English (en)') assert stats.submissions_count == 6 From 7d9f1b864d2d1f92ff22789a0f1803e33d16cb08 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 12 Nov 2021 22:30:18 +0000 Subject: [PATCH 13/54] fix failing fixture test --- src/formpack/reporting/export.py | 2 -- tests/test_fixtures_valid.py | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index c72911dc..2c54359e 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -397,8 +397,6 @@ def _get_value_from_entry(entry, field): if field.path in self.filter_fields ) - - # 'rows' will contain all the formatted entries for the current # section. If you don't have repeat-group, there is only one section # with a row of size one. diff --git a/tests/test_fixtures_valid.py b/tests/test_fixtures_valid.py index 5646b653..fcc53745 100644 --- a/tests/test_fixtures_valid.py +++ b/tests/test_fixtures_valid.py @@ -111,7 +111,9 @@ def test_analysis_form(self): assert 4 == len(fixture) title, schemas, submissions, analysis_form = fixture - fp = FormPack(schemas, title, analysis_form=analysis_form) + fp = FormPack(schemas, title) + fp.extend_survey(analysis_form) + assert 2 == len(fp.versions) assert 'Simple Clerk Interaction' == title @@ -121,8 +123,7 @@ def test_analysis_form(self): actual_analysis_questions = sorted( [ f.name - for f in fp.get_fields_for_versions(fp.versions) - if f.analysis_question + for f in fp.analysis_form.fields ] ) assert expected_analysis_questions == actual_analysis_questions From f4d396fc6de396d944c80018a3993e8e9a66fb08 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Sat, 13 Nov 2021 00:03:57 +0000 Subject: [PATCH 14/54] add support for `anaylsis_type` and `settings` field attributes --- src/formpack/constants.py | 10 ++++++++++ src/formpack/schema/fields.py | 15 ++++++++++++++- src/formpack/version.py | 10 ++++------ tests/fixtures/analysis_form/analysis_form.json | 6 +++++- .../analysis_form_advanced/analysis_form.json | 6 +++++- tests/test_additional_field_exports.py | 2 +- tests/test_fixtures_valid.py | 9 +++++++++ 7 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/formpack/constants.py b/src/formpack/constants.py index 92c10354..7f495b6a 100644 --- a/src/formpack/constants.py +++ b/src/formpack/constants.py @@ -155,3 +155,13 @@ 'form_appearance', 'form_meta_edit', ] + +# Analysis types +ANALYSIS_TYPE_CODING = 'coding' +ANALYSIS_TYPE_TRANSCRIPTION = 'transcription' +ANALYSIS_TYPE_TRANSLATION = 'translation' +ANALYSIS_TYPES = [ + ANALYSIS_TYPE_CODING, + ANALYSIS_TYPE_TRANSCRIPTION, + ANALYSIS_TYPE_TRANSLATION, +] diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index d4b8da97..fc6e6f55 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -8,7 +8,11 @@ import statistics from .datadef import FormDataDef, FormChoice -from ..constants import UNSPECIFIED_TRANSLATION +from ..constants import ( + ANALYSIS_TYPES, + ANALYSIS_TYPE_CODING, + UNSPECIFIED_TRANSLATION, +) from ..utils import singlemode from ..utils.ordered_collection import OrderedDefaultdict @@ -30,6 +34,8 @@ def __init__(self, name, labels, data_type, hierarchy=None, if source is not None: self.source = source self.analysis_question = True + self.analysis_type = kwargs.get('analysis_type') + self.settings = kwargs.get('settings') hierarchy = list(hierarchy) if hierarchy is not None else [None] self.hierarchy = hierarchy + [self] @@ -160,6 +166,8 @@ def from_json_definition(cls, definition, hierarchy=None, appearance = definition.get('appearance') or_other = definition.get('_or_other', False) source = definition.get('source') + analysis_type = definition.get('analysis_type', ANALYSIS_TYPE_CODING) + settings = definition.get('settings', {}) # normalize spaces data_type = definition['type'] @@ -167,6 +175,9 @@ def from_json_definition(cls, definition, hierarchy=None, if ' ' in data_type: raise ValueError('invalid data_type: %s' % data_type) + if analysis_type not in ANALYSIS_TYPES: + raise ValueError(f'Invalid analysis data type: {analysis_type}') + if data_type in ('select_one', 'select_multiple'): choice_id = definition['select_from_list_name'] # pyxform#472 introduced dynamic list_names for select_one with the @@ -235,6 +246,8 @@ def from_json_definition(cls, definition, hierarchy=None, 'choice': choice, 'or_other': or_other, 'source': source, + 'analysis_type': analysis_type, + 'settings': settings, } if data_type == 'select_multiple' and appearance == 'literacy': diff --git a/src/formpack/version.py b/src/formpack/version.py index 7173f433..76789661 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -320,7 +320,6 @@ def to_xml(self, warnings=None): class AnalysisForm: - def __init__(self, form_pack, schema): self.schema = schema @@ -341,17 +340,16 @@ def __init__(self, form_pack, schema): self.fields = [ FormField.from_json_definition( - dd, - translations=self.translations, - section=section, + definition=data_def, field_choices=field_choices, + section=section, + translations=self.translations, ) - for dd in survey + for data_def in survey ] self.fields_by_source = self._get_fields_by_source() - def __repr__(self): return f"" diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 6263664d..7d24ab3d 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -12,7 +12,11 @@ ": ACME Transcription" ], "source": "record_a_note", - "engine": "engines/acme_1_speech2text" + "analysis_type": "transcription", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } }, { "type": "datetime", diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 5e4a91c7..18a01175 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -13,7 +13,11 @@ "" ], "source": "record_a_note", - "engine": "engines/acme_1_speech2text" + "analysis_type": "transcription", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } }, { "type": "select_multiple", diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 5ad4a7fd..f06fbf0d 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -2,7 +2,7 @@ from formpack import FormPack from .fixtures import build_fixture -def tests_additional_field_exports_xxx(): +def tests_additional_field_exports(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) diff --git a/tests/test_fixtures_valid.py b/tests/test_fixtures_valid.py index fcc53745..b202dd9b 100644 --- a/tests/test_fixtures_valid.py +++ b/tests/test_fixtures_valid.py @@ -6,6 +6,7 @@ from formpack import FormPack from .fixtures import build_fixture +from formpack.constants import ANALYSIS_TYPES class TestFormPackFixtures(unittest.TestCase): @@ -128,3 +129,11 @@ def test_analysis_form(self): ) assert expected_analysis_questions == actual_analysis_questions + f1 = fp.analysis_form.fields[0] + assert hasattr(f1, 'source') and f1.source + assert hasattr(f1, 'has_stats') and not f1.has_stats + assert ( + hasattr(f1, 'analysis_type') and f1.analysis_type in ANALYSIS_TYPES + ) + assert hasattr(f1, 'settings') + From 4ff7a3f5748e1632ec214f8faeb06d89945e1e99 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Sat, 13 Nov 2021 00:09:00 +0000 Subject: [PATCH 15/54] minor edit --- src/formpack/pack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 80b474fb..444c158e 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -167,8 +167,7 @@ def load_version(self, schema): self.versions[form_version.id] = form_version def extend_survey(self, analysis_form): - af = AnalysisForm(self, analysis_form) - self.analysis_form = af + self.analysis_form = AnalysisForm(self, analysis_form) def version_diff(self, vn1, vn2): v1 = self.versions[vn1] From 9a661718c7478bf363f019808fa89b7eb56d8a8e Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 19 Nov 2021 22:43:44 +0000 Subject: [PATCH 16/54] wip support for repeat groups --- src/formpack/reporting/export.py | 28 ++- src/formpack/version.py | 9 +- tests/fixtures/analysis_form_advanced/v1.json | 23 ++- .../analysis_form_repeat_groups/__init__.py | 15 ++ .../analysis_form.json | 37 ++++ .../analysis_form_repeat_groups/v1.json | 166 ++++++++++++++++++ tests/test_additional_field_exports.py | 70 ++++++++ 7 files changed, 335 insertions(+), 13 deletions(-) create mode 100644 tests/fixtures/analysis_form_repeat_groups/__init__.py create mode 100644 tests/fixtures/analysis_form_repeat_groups/analysis_form.json create mode 100644 tests/fixtures/analysis_form_repeat_groups/v1.json diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 2c54359e..ce98dbbb 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -316,6 +316,8 @@ def format_one_submission( submission, current_section, attachments=None, + supplemental_details=None, + repeat_index=0, ): # 'current_section' is the name of what will become sheets in xls. @@ -381,9 +383,14 @@ def _get_attachment(val, field, attachments): if re.match(fr'^.*/{_val}$', f['filename']) is not None ] - def _get_value_from_entry(entry, field): - if field.analysis_question and '_supplementalDetails' in entry: - return entry['_supplementalDetails'].get(field.name) + def _get_value_from_entry(entry, field, supplemental_details): + if field.analysis_question and supplemental_details: + sd = supplemental_details.get(field.name) + if isinstance(sd, str): + return sd + if isinstance(sd, list): + _v = [v['value'] for v in sd if v['_index'] == repeat_index] + return _v[0] if _v else '' suffix = 'meta/' if field.data_type == 'audit' else '' return entry.get(f'{suffix}{field.path}') @@ -425,13 +432,15 @@ def _get_value_from_entry(entry, field): row.update(_empty_row) attachments = entry.get('_attachments') or attachments + supplemental_details = ( + entry.get('_supplementalDetails') or supplemental_details + ) for field in _fields: # TODO: pass a context to fields so they can all format ? if field.can_format: - # get submission value for this field - val = _get_value_from_entry(entry, field) + val = _get_value_from_entry(entry, field, supplemental_details) # get the attachment for this field attachment = _get_attachment(val, field, attachments) # get a mapping of {"col_name": "val", ...} @@ -486,7 +495,9 @@ def _get_value_from_entry(entry, field): chunk = self.format_one_submission( entry[child_section.path], child_section, - attachments, + attachemnts=attachments, + supplemental_details=supplemental_details, + repeat_index=repeat_index, ) for key, value in iter(chunk.items()): if key in chunks: @@ -494,7 +505,12 @@ def _get_value_from_entry(entry, field): else: chunks[key] = value + # Reset the repeat index once we're done with this current + # repeat group + repeat_index = 0 + _indexes[_section_name] += 1 + repeat_index += 1 return chunks diff --git a/src/formpack/version.py b/src/formpack/version.py index 76789661..6a1ce492 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -362,10 +362,17 @@ def _get_fields_by_source(self): fields_by_source[field.source].append(field) return fields_by_source + def _map_sections_to_analysis_fields(self, survey_field): + _fields = [] + for analysis_field in self.fields_by_source[survey_field.name]: + analysis_field.section = survey_field.section + _fields.append(analysis_field) + return _fields + def insert_analysis_fields(self, fields): _fields = [] for field in fields: _fields.append(field) if field.name in self.fields_by_source: - _fields += self.fields_by_source[field.name] + _fields += self._map_sections_to_analysis_fields(field) return _fields diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index 2c87aa51..44731270 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -2,6 +2,14 @@ "version": "v1", "content": { "survey": [ + { + "type": "begin_group", + "name": "clerk_interactions", + "label": [ + "Clerk interactions", + "Komizo-interagoj" + ] + }, { "type": "audio", "name": "record_a_note", @@ -17,6 +25,9 @@ "What are some goods sold at the store?", "Kio estas iuj varoj venditaj en la vendejo?" ] + }, + { + "type": "end_group" } ], "choices": [ @@ -58,8 +69,8 @@ }, "submissions": [ { - "record_a_note": "clerk_interaction_1.mp3", - "goods_sold": "chocolate", + "clerk_interactions/record_a_note": "clerk_interaction_1.mp3", + "clerk_interactions/goods_sold": "chocolate", "_attachments": [ {"mimetype": "audio/mpeg", "filename": "clerk_interaction_1.mp3"} ], @@ -71,8 +82,8 @@ } }, { - "record_a_note": "clerk_interaction_2.mp3", - "goods_sold": "chocolate fruit pasta", + "clerk_interactions/record_a_note": "clerk_interaction_2.mp3", + "clerk_interactions/goods_sold": "chocolate fruit pasta", "_attachments": [ {"mimetype": "audio/mpeg", "filename": "clerk_interaction_2.mp3"} ], @@ -83,8 +94,8 @@ } }, { - "record_a_note": "clerk_interaction_3.mp3", - "goods_sold": "pasta", + "clerk_interactions/record_a_note": "clerk_interaction_3.mp3", + "clerk_interactions/goods_sold": "pasta", "_attachments": [ {"mimetype": "audio/mpeg", "filename": "clerk_interaction_3.mp3"} ], diff --git a/tests/fixtures/analysis_form_repeat_groups/__init__.py b/tests/fixtures/analysis_form_repeat_groups/__init__.py new file mode 100644 index 00000000..47f3e500 --- /dev/null +++ b/tests/fixtures/analysis_form_repeat_groups/__init__.py @@ -0,0 +1,15 @@ +# coding: utf-8 +''' +analysis_form_repeat_groups +''' + +from ..load_fixture_json import load_fixture_json, load_analysis_form_json + +DATA = { + 'title': 'Clerk Interaction Repeat Groups', + 'id_string': 'cerk_interaction_repeat_groups', + 'versions': [ + load_fixture_json('analysis_form_repeat_groups/v1'), + ], + 'analysis_form': load_analysis_form_json('analysis_form_repeat_groups') +} diff --git a/tests/fixtures/analysis_form_repeat_groups/analysis_form.json b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json new file mode 100644 index 00000000..b8528916 --- /dev/null +++ b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json @@ -0,0 +1,37 @@ +{ + "engines": { + "acme_1_speech2text": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note_transcription_acme_1_speech2text", + "label": [ + ": ACME Transcription", + "" + ], + "source": "record_a_note", + "analysis_type": "transcription", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } + }, + { + "type": "text", + "name": "record_a_noise_comment_on_noise_level", + "label": [ + "Comment on noise level", + "Komentu pri brunivelo" + ], + "source": "record_a_noise", + "analysis_type": "coding" + } + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] +} diff --git a/tests/fixtures/analysis_form_repeat_groups/v1.json b/tests/fixtures/analysis_form_repeat_groups/v1.json new file mode 100644 index 00000000..a8b1bce1 --- /dev/null +++ b/tests/fixtures/analysis_form_repeat_groups/v1.json @@ -0,0 +1,166 @@ +{ + "version": "v1", + "content": { + "survey": [ + { + "type": "text", + "name": "enumerator_name", + "label": [ + "What is your name?", + "Kio estas via nomo?" + ] + }, + { + "type": "begin_repeat", + "name": "stores", + "label": [ + "Stores", + "Vendejoj" + ] + }, + { + "type": "text", + "name": "store_name", + "label": [ + "What is the store name?", + "Kio estas la nomo de la vendejo?" + ] + }, + { + "type": "begin_group", + "name": "recordings", + "label": [ + "Recordings", + "Registradoj" + ] + }, + { + "type": "begin_repeat", + "name": "record_interactions", + "label": [ + "Record interactions", + "Registri interagojn" + ] + }, + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "end_repeat" + }, + { + "type": "begin_repeat", + "name": "record_ambient_noises", + "label": [ + "Record ambient noises", + "Registri ĉirkaŭajn bruojn" + ] + }, + { + "type": "audio", + "name": "record_a_noise", + "label": [ + "Record some abient noise", + "Registru iun ĉirkaŭan bruon" + ] + }, + { + "type": "end_repeat" + }, + { + "type": "end_group" + }, + { + "type": "end_repeat" + } + ], + "settings": { + "version": "v1" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "enumerator_name": "John Doe", + "stores": [ + { + "stores/store_name": "Costco", + "stores/recordings/record_interactions": [ + { + "stores/recordings/record_interactions/record_a_note": "clerk_interaction_1.mp3" + }, + { + "stores/recordings/record_interactions/record_a_note": "clerk_interaction_2.mp3" + }, + { + "stores/recordings/record_interactions/record_a_note": "clerk_interaction_3.mp3" + } + ], + "stores/recordings/record_ambient_noises": [ + { + "stores/recordings/record_ambient_noises/record_a_noise": "noise_1.mp3" + }, + { + "stores/recordings/record_ambient_noises/record_a_noise": "noise_2.mp3" + } + ] + } + ], + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_1.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_2.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_3.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "noise_1.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "noise_2.mp3" + } + ], + "_supplementalDetails": { + "record_a_note_transcription_acme_1_speech2text": [ + { + "_index": 0, + "value": "Hello how may I help you?" + }, + { + "_index": 2, + "value": "Thank you for your business" + } + ], + "record_a_noise_comment_on_noise_level": [ + { + "_index": 0, + "value": "Lot's of noise" + }, + { + "_index": 1, + "value": "Quiet" + } + ] + } + } + ] +} diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index f06fbf0d..b7e8112b 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -29,6 +29,76 @@ def tests_additional_field_exports(): 'Sounds like an interesting person', ] +def tests_additional_field_exports_repeat_groups(): + title, schemas, submissions, analysis_form = build_fixture( + 'analysis_form_repeat_groups' + ) + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'include_analysis_fields': True, + 'versions': 'v1', + } + export = pack.export(**options) + values = export.to_dict(submissions) + assert [ + 'Clerk Interaction Repeat Groups', + 'stores', + 'record_interactions', + 'record_ambient_noises', + ] == list(values.keys()) + + main_export_sheet = values['Clerk Interaction Repeat Groups'] + assert ['enumerator_name', '_index'] == main_export_sheet['fields'] + main_response0 = main_export_sheet['data'][0] + assert 'John Doe' == main_response0[0] + + repeat_sheet_0 = values['stores'] + assert 'Costco' == repeat_sheet_0['data'][0][0] + + repeat_sheet_1 = values['record_interactions'] + assert [ + 'record_a_note', + 'record_a_note_transcription_acme_1_speech2text', + ] == repeat_sheet_1['fields'][:2] + assert 3 == len(repeat_sheet_1['data']) + repeat_data_response_1 = [res[:2] for res in repeat_sheet_1['data']] + repeat_data_expected_1 = [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + ], + [ + 'clerk_interaction_2.mp3', + '', + ], + [ + 'clerk_interaction_3.mp3', + 'Thank you for your business', + ], + ] + assert repeat_data_expected_1 == repeat_data_response_1 + + repeat_sheet_2 = values['record_ambient_noises'] + assert [ + 'record_a_noise', + 'record_a_noise_comment_on_noise_level', + ] == repeat_sheet_2['fields'][:2] + assert 2 == len(repeat_sheet_2['data']) + repeat_data_response_2 = [res[:2] for res in repeat_sheet_2['data']] + repeat_data_expected_2 = [ + [ + 'noise_1.mp3', + "Lot's of noise", + ], + [ + 'noise_2.mp3', + 'Quiet', + ], + ] + assert repeat_data_expected_2 == repeat_data_response_2 + def tests_additional_field_exports_advanced(): title, schemas, submissions, analysis_form = build_fixture( 'analysis_form_advanced' From 978c9f3cc8e22514a05f190fd919d6d699228bd5 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 19 Nov 2021 22:45:39 +0000 Subject: [PATCH 17/54] fix typo --- src/formpack/reporting/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index ce98dbbb..71583bf5 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -495,7 +495,7 @@ def _get_value_from_entry(entry, field, supplemental_details): chunk = self.format_one_submission( entry[child_section.path], child_section, - attachemnts=attachments, + attachments=attachments, supplemental_details=supplemental_details, repeat_index=repeat_index, ) From 98535f447eaa675cca8fab324b9e2b43c1c526ca Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 00:58:49 +0000 Subject: [PATCH 18/54] restructure fixtures with kpi model changes --- .../fixtures/analysis_form/analysis_form.json | 97 ++++---- tests/fixtures/analysis_form/v1.json | 91 ++++---- tests/fixtures/analysis_form/v2.json | 91 ++++---- .../analysis_form_advanced/analysis_form.json | 210 +++++++++--------- tests/fixtures/analysis_form_advanced/v1.json | 151 +++++++------ .../analysis_form.json | 70 +++--- .../analysis_form_repeat_groups/v1.json | 56 ++--- 7 files changed, 415 insertions(+), 351 deletions(-) diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 7d24ab3d..1b30316d 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -1,49 +1,54 @@ { - "engines": { - "acme_1_speech2text": { - "details": "an external service provided by ACME, Inc." - } + "engines": { + "acme_1_speech2text": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note/transcript_acme_1_speech2text", + "path": [ + "record_a_note", + "transcript_acme_1_speech2text" + ], + "label": ": ACME Transcription", + "source": "record_a_note", + "analysis_type": "transcript", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } }, - "additional_fields": [ - { - "type": "text", - "name": "record_a_note_transcription_acme_1_speech2text", - "label": [ - ": ACME Transcription" - ], - "source": "record_a_note", - "analysis_type": "transcription", - "settings": { - "mode": "auto", - "engine": "engines/acme_1_speech2text" - } - }, - { - "type": "datetime", - "name": "record_a_note_acme_timestamp", - "label": [ - "Transcription Timestamp" - ], - "source": "record_a_note" - }, - { - "type": "text", - "name": "name_of_clerk_comment", - "label": [ - "Comment on the name of the clerk" - ], - "source": "name_of_clerk" - }, - { - "type": "text", - "name": "name_of_shop_comment", - "label": [ - "Comment on the name of the shop" - ], - "source": "name_of_shop" - } - ], - "translations": [ - "English (en)" - ] + { + "type": "datetime", + "name": "record_a_note/acme_timestamp", + "path": [ + "record_a_note", + "acme_timestamp" + ], + "label": "Transcription Timestamp", + "source": "record_a_note" + }, + { + "type": "text", + "name": "name_of_clerk/comment", + "path": [ + "name_of_clerk", + "comment" + ], + "label": "Comment on the name of the clerk", + "source": "name_of_clerk" + }, + { + "type": "text", + "name": "name_of_shop/comment", + "path": [ + "name_of_shop", + "comment" + ], + "label": "Comment on the name of the shop", + "source": "name_of_shop" + } + ] } diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index 2e144bfd..0bcd0139 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -2,43 +2,43 @@ "version": "v1", "content": { "survey": [ - { - "type": "audio", - "name": "record_a_note", - "label": [ - "Record a clerk saying something", - "Registri oficiston dirantan ion" - ] - }, - { - "type": "begin_group", - "name": "clerk_details", - "label": [ - "Some details of the clerk", - "Kelkaj detaloj de la oficisto" - ] - }, - { - "type": "text", - "name": "name_of_clerk", - "label": [ - "What is the clerk's name?", - "" - ] - }, - { - "type": "end_group" - } + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "begin_group", + "name": "clerk_details", + "label": [ + "Some details of the clerk", + "Kelkaj detaloj de la oficisto" + ] + }, + { + "type": "text", + "name": "name_of_clerk", + "label": [ + "What is the clerk's name?", + "" + ] + }, + { + "type": "end_group" + } ], "settings": { - "version": "v1" + "version": "v1" }, "translated": [ - "label" + "label" ], "translations": [ - "English (en)", - "Esperanto (es)" + "English (en)", + "Esperanto (es)" ] }, "submissions": [ @@ -46,29 +46,44 @@ "record_a_note": "clerk_interaction_1.mp3", "clerk_details/name_of_clerk": "John", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_1.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_1.mp3" + } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": "Hello how may I help you?", - "record_a_note_acme_timestamp": "2021-11-01Z", - "name_of_clerk_comment": "Sounds like an interesting person" + "record_a_note": { + "transcript_acme_1_speech2text": "Hello how may I help you?", + "acme_timestamp": "2021-11-01Z" + }, + "name_of_clerk": { + "comment": "Sounds like an interesting person" + } } }, { "record_a_note": "clerk_interaction_2.mp3", "clerk_details/name_of_clerk": "Alex", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_2.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_2.mp3" + } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": "Thank you for your business" + "record_a_note": { + "transcript_acme_1_speech2text": "Thank you for your business" + } } }, { "record_a_note": "clerk_interaction_3.mp3", "clerk_details/name_of_clerk": "Olivier", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_3.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_3.mp3" + } ], "_supplementalDetails": {} } diff --git a/tests/fixtures/analysis_form/v2.json b/tests/fixtures/analysis_form/v2.json index 15759009..5ac1bdc4 100644 --- a/tests/fixtures/analysis_form/v2.json +++ b/tests/fixtures/analysis_form/v2.json @@ -2,43 +2,43 @@ "version": "v2", "content": { "survey": [ - { - "type": "audio", - "name": "record_a_note", - "label": [ - "Record a clerk saying something", - "Registri oficiston dirantan ion" - ] - }, - { - "type": "begin_group", - "name": "clerk_details", - "label": [ - "Some details of the clerk", - "Kelkaj detaloj de la oficisto" - ] - }, - { - "type": "text", - "name": "name_of_shop", - "label": [ - "What is the shop's name?", - "Kio estas la nomo de la butiko?" - ] - }, - { - "type": "end_group" - } + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "begin_group", + "name": "clerk_details", + "label": [ + "Some details of the clerk", + "Kelkaj detaloj de la oficisto" + ] + }, + { + "type": "text", + "name": "name_of_shop", + "label": [ + "What is the shop's name?", + "Kio estas la nomo de la butiko?" + ] + }, + { + "type": "end_group" + } ], "settings": { - "version": "v2" + "version": "v2" }, "translated": [ - "label" + "label" ], "translations": [ - "English (en)", - "Esperanto (es)" + "English (en)", + "Esperanto (es)" ] }, "submissions": [ @@ -46,29 +46,44 @@ "record_a_note": "clerk_interaction_4.mp3", "clerk_details/name_of_shop": "Save On", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_4.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_4.mp3" + } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": "Hello how may I help you?", - "record_a_note_acme_timestamp": "2021-11-01Z", - "name_of_shop_comment": "Pretty cliche" + "record_a_note": { + "transcript_acme_1_speech2text": "Hello how may I help you?", + "acme_timestamp": "2021-11-01Z" + }, + "name_of_shop": { + "comment": "Pretty cliche" + } } }, { "record_a_note": "clerk_interaction_5.mp3", "clerk_details/name_of_shop": "Walmart", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_5.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_5.mp3" + } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": "Thank you for your business" + "record_a_note": { + "transcript_acme_1_speech2text": "Thank you for your business" + } } }, { "record_a_note": "clerk_interaction_6.mp3", "clerk_details/name_of_shop": "Costco", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_6.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_6.mp3" + } ], "_supplementalDetails": {} } diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 18a01175..d4e1e266 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -1,106 +1,110 @@ { - "engines": { - "acme_1_speech2text": { - "details": "an external service provided by ACME, Inc." - } + "engines": { + "acme_1_speech2text": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note/transcript_acme_1_speech2text", + "path": ["record_a_note", "transcript_acme_1_speech2text"], + "label": [ + ": ACME Transcription", + "" + ], + "source": "record_a_note", + "analysis_type": "transcript", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } }, - "additional_fields": [ - { - "type": "text", - "name": "record_a_note_transcription_acme_1_speech2text", - "label": [ - ": ACME Transcription", - "" - ], - "source": "record_a_note", - "analysis_type": "transcription", - "settings": { - "mode": "auto", - "engine": "engines/acme_1_speech2text" - } - }, - { - "type": "select_multiple", - "select_from_list_name": "record_a_note_tones", - "name": "record_a_note_tone_of_voice", - "label": [ - "How was the tone of the clerk's voice?", - "Kiel estis la tono de la voĉo de la oficisto?" - ], - "source": "record_a_note" - }, - { - "type": "text", - "name": "goods_sold_comment", - "label": [ - "Comment on the goods sold at the store", - "Komentu la varojn venditajn en la vendejo" - ], - "source": "goods_sold" - }, - { - "type": "select_one", - "select_from_list_name": "goods_sold_ratings", - "name": "goods_sold_rating", - "label": [ - "Rate the quality of the goods sold at the store", - "Komentu la varojn vendojn en la vendejo" - ], - "source": "goods_sold" - } - ], - "additional_choices": [ - { - "list_name": "goods_sold_ratings", - "name": 1, - "label": [ - "Poor quality", - "Malbona kvalito" - ] - }, - { - "list_name": "goods_sold_ratings", - "name": 2, - "label": [ - "Average quality", - "Meza kvalito" - ] - }, - { - "list_name": "ratings", - "name": 3, - "label": [ - "High quality", - "Alta kvalito" - ] - }, - { - "list_name": "record_a_note_tones", - "name": "anxious", - "label": [ - "Anxious", - "Maltrankvila" - ] - }, - { - "list_name": "record_a_note_tones", - "name": "excited", - "label": [ - "Excited", - "Ekscitita" - ] - }, - { - "list_name": "record_a_note_tones", - "name": "confused", - "label": [ - "Confused", - "Konfuzita" - ] - } - ], - "translations": [ - "English (en)", - "Esperanto (es)" - ] + { + "type": "select_multiple", + "select_from_list_name": "record_a_note_tones", + "name": "record_a_note/tone_of_voice", + "path": ["record_a_note", "tone_of_voice"], + "label": [ + "How was the tone of the clerk's voice?", + "Kiel estis la tono de la voĉo de la oficisto?" + ], + "source": "record_a_note" + }, + { + "type": "text", + "name": "goods_sold/comment", + "path": ["goods_sold", "comment"], + "label": [ + "Comment on the goods sold at the store", + "Komentu la varojn venditajn en la vendejo" + ], + "source": "goods_sold" + }, + { + "type": "select_one", + "select_from_list_name": "goods_sold_ratings", + "name": "goods_sold/rating", + "path": ["goods_sold", "rating"], + "label": [ + "Rate the quality of the goods sold at the store", + "Komentu la varojn vendojn en la vendejo" + ], + "source": "goods_sold" + } + ], + "additional_choices": [ + { + "list_name": "goods_sold_ratings", + "name": 1, + "label": [ + "Poor quality", + "Malbona kvalito" + ] + }, + { + "list_name": "goods_sold_ratings", + "name": 2, + "label": [ + "Average quality", + "Meza kvalito" + ] + }, + { + "list_name": "ratings", + "name": 3, + "label": [ + "High quality", + "Alta kvalito" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "anxious", + "label": [ + "Anxious", + "Maltrankvila" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "excited", + "label": [ + "Excited", + "Ekscitita" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "confused", + "label": [ + "Confused", + "Konfuzita" + ] + } + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] } diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index 44731270..dc97cc7e 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -2,69 +2,69 @@ "version": "v1", "content": { "survey": [ - { - "type": "begin_group", - "name": "clerk_interactions", - "label": [ - "Clerk interactions", - "Komizo-interagoj" - ] - }, - { - "type": "audio", - "name": "record_a_note", - "label": [ - "Record a clerk saying something", - "Registri oficiston dirantan ion" - ] - }, - { - "type": "select_multiple goods", - "name": "goods_sold", - "label": [ - "What are some goods sold at the store?", - "Kio estas iuj varoj venditaj en la vendejo?" - ] - }, - { - "type": "end_group" - } + { + "type": "begin_group", + "name": "clerk_interactions", + "label": [ + "Clerk interactions", + "Komizo-interagoj" + ] + }, + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "select_multiple goods", + "name": "goods_sold", + "label": [ + "What are some goods sold at the store?", + "Kio estas iuj varoj venditaj en la vendejo?" + ] + }, + { + "type": "end_group" + } ], "choices": [ - { - "list_name": "goods", - "name": "chocolate", - "label": [ - "Chocolate", - "Ĉokolado" - ] - }, - { - "list_name": "goods", - "name": "fruit", - "label": [ - "Fruit", - "Frukto" - ] - }, - { - "list_name": "goods", - "name": "pasta", - "label": [ - "Pasta", - "Pasto" - ] - } + { + "list_name": "goods", + "name": "chocolate", + "label": [ + "Chocolate", + "Ĉokolado" + ] + }, + { + "list_name": "goods", + "name": "fruit", + "label": [ + "Fruit", + "Frukto" + ] + }, + { + "list_name": "goods", + "name": "pasta", + "label": [ + "Pasta", + "Pasto" + ] + } ], "settings": { - "version": "v1" + "version": "v1" }, "translated": [ - "label" + "label" ], "translations": [ - "English (en)", - "Esperanto (es)" + "English (en)", + "Esperanto (es)" ] }, "submissions": [ @@ -72,35 +72,54 @@ "clerk_interactions/record_a_note": "clerk_interaction_1.mp3", "clerk_interactions/goods_sold": "chocolate", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_1.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_1.mp3" + } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": "Hello how may I help you?", - "record_a_note_tone_of_voice": "excited confused", - "goods_sold_comment": "Not much diversity", - "goods_sold_rating": "3" + "record_a_note": { + "transcript_acme_1_speech2text": "Hello how may I help you?", + "tone_of_voice": "excited confused" + }, + "goods_sold": { + "comment": "Not much diversity", + "rating": "3" + } } }, { "clerk_interactions/record_a_note": "clerk_interaction_2.mp3", "clerk_interactions/goods_sold": "chocolate fruit pasta", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_2.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_2.mp3" + } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": "Thank you for your business", - "record_a_note_tone_of_voice": "anxious excited", - "goods_sold_rating": "2" + "record_a_note": { + "transcription_acme_1_speech2text": "Thank you for your business", + "tone_of_voice": "anxious excited" + }, + "goods_sold": { + "rating": "2" + } } }, { "clerk_interactions/record_a_note": "clerk_interaction_3.mp3", "clerk_interactions/goods_sold": "pasta", "_attachments": [ - {"mimetype": "audio/mpeg", "filename": "clerk_interaction_3.mp3"} + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_3.mp3" + } ], "_supplementalDetails": { - "goods_sold_rating": "3" + "goods_sold": { + "rating": "3" + } } } ] diff --git a/tests/fixtures/analysis_form_repeat_groups/analysis_form.json b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json index b8528916..1b715b72 100644 --- a/tests/fixtures/analysis_form_repeat_groups/analysis_form.json +++ b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json @@ -1,37 +1,39 @@ { - "engines": { - "acme_1_speech2text": { - "details": "an external service provided by ACME, Inc." - } + "engines": { + "acme_1_speech2text": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note/transcript_acme_1_speech2text", + "path": ["record_a_note", "transcript_acme_1_speech2text"], + "label": [ + ": ACME Transcription", + "" + ], + "source": "record_a_note", + "analysis_type": "transcript", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } }, - "additional_fields": [ - { - "type": "text", - "name": "record_a_note_transcription_acme_1_speech2text", - "label": [ - ": ACME Transcription", - "" - ], - "source": "record_a_note", - "analysis_type": "transcription", - "settings": { - "mode": "auto", - "engine": "engines/acme_1_speech2text" - } - }, - { - "type": "text", - "name": "record_a_noise_comment_on_noise_level", - "label": [ - "Comment on noise level", - "Komentu pri brunivelo" - ], - "source": "record_a_noise", - "analysis_type": "coding" - } - ], - "translations": [ - "English (en)", - "Esperanto (es)" - ] + { + "type": "text", + "name": "record_a_noise/comment_on_noise_level", + "path": ["record_a_noise", "comment_on_noise_level"], + "label": [ + "Comment on noise level", + "Komentu pri brunivelo" + ], + "source": "record_a_noise", + "analysis_type": "coding" + } + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] } diff --git a/tests/fixtures/analysis_form_repeat_groups/v1.json b/tests/fixtures/analysis_form_repeat_groups/v1.json index a8b1bce1..eb121796 100644 --- a/tests/fixtures/analysis_form_repeat_groups/v1.json +++ b/tests/fixtures/analysis_form_repeat_groups/v1.json @@ -27,12 +27,12 @@ ] }, { - "type": "begin_group", - "name": "recordings", - "label": [ - "Recordings", - "Registradoj" - ] + "type": "begin_group", + "name": "recordings", + "label": [ + "Recordings", + "Registradoj" + ] }, { "type": "begin_repeat", @@ -140,26 +140,30 @@ } ], "_supplementalDetails": { - "record_a_note_transcription_acme_1_speech2text": [ - { - "_index": 0, - "value": "Hello how may I help you?" - }, - { - "_index": 2, - "value": "Thank you for your business" - } - ], - "record_a_noise_comment_on_noise_level": [ - { - "_index": 0, - "value": "Lot's of noise" - }, - { - "_index": 1, - "value": "Quiet" - } - ] + "record_a_note": { + "transcript_acme_1_speech2text": [ + { + "_index": 0, + "value": "Hello how may I help you?" + }, + { + "_index": 2, + "value": "Thank you for your business" + } + ] + }, + "record_a_noise": { + "comment_on_noise_level": [ + { + "_index": 0, + "value": "Lot's of noise" + }, + { + "_index": 1, + "value": "Quiet" + } + ] + } } } ] From 18837749f141add97c37b3777a7b1cabd4729a84 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 01:28:39 +0000 Subject: [PATCH 19/54] update tests with new syntax --- src/formpack/constants.py | 4 +- tests/fixtures/analysis_form_advanced/v1.json | 2 +- tests/test_additional_field_exports.py | 76 +++++++++---------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/formpack/constants.py b/src/formpack/constants.py index 7f495b6a..ad2dc5a3 100644 --- a/src/formpack/constants.py +++ b/src/formpack/constants.py @@ -158,10 +158,10 @@ # Analysis types ANALYSIS_TYPE_CODING = 'coding' -ANALYSIS_TYPE_TRANSCRIPTION = 'transcription' +ANALYSIS_TYPE_TRANSCRIPT = 'transcript' ANALYSIS_TYPE_TRANSLATION = 'translation' ANALYSIS_TYPES = [ ANALYSIS_TYPE_CODING, - ANALYSIS_TYPE_TRANSCRIPTION, + ANALYSIS_TYPE_TRANSCRIPT, ANALYSIS_TYPE_TRANSLATION, ] diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index dc97cc7e..b482c0cf 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -99,7 +99,7 @@ ], "_supplementalDetails": { "record_a_note": { - "transcription_acme_1_speech2text": "Thank you for your business", + "transcript_acme_1_speech2text": "Thank you for your business", "tone_of_voice": "anxious excited" }, "goods_sold": { diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index b7e8112b..bd42e4b7 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -2,7 +2,7 @@ from formpack import FormPack from .fixtures import build_fixture -def tests_additional_field_exports(): +def tests_additional_field_exports_x(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -15,10 +15,10 @@ def tests_additional_field_exports(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_acme_timestamp', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/acme_timestamp', 'name_of_clerk', - 'name_of_clerk_comment', + 'name_of_clerk/comment', ] response0 = main_export_sheet['data'][0] assert response0 == [ @@ -60,7 +60,7 @@ def tests_additional_field_exports_repeat_groups(): repeat_sheet_1 = values['record_interactions'] assert [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', + 'record_a_note/transcript_acme_1_speech2text', ] == repeat_sheet_1['fields'][:2] assert 3 == len(repeat_sheet_1['data']) repeat_data_response_1 = [res[:2] for res in repeat_sheet_1['data']] @@ -83,7 +83,7 @@ def tests_additional_field_exports_repeat_groups(): repeat_sheet_2 = values['record_ambient_noises'] assert [ 'record_a_noise', - 'record_a_noise_comment_on_noise_level', + 'record_a_noise/comment_on_noise_level', ] == repeat_sheet_2['fields'][:2] assert 2 == len(repeat_sheet_2['data']) repeat_data_response_2 = [res[:2] for res in repeat_sheet_2['data']] @@ -118,17 +118,17 @@ def tests_additional_field_exports_advanced(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_tone_of_voice', - 'record_a_note_tone_of_voice/anxious', - 'record_a_note_tone_of_voice/excited', - 'record_a_note_tone_of_voice/confused', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/tone_of_voice', + 'record_a_note/tone_of_voice/anxious', + 'record_a_note/tone_of_voice/excited', + 'record_a_note/tone_of_voice/confused', 'goods_sold', 'goods_sold/chocolate', 'goods_sold/fruit', 'goods_sold/pasta', - 'goods_sold_comment', - 'goods_sold_rating', + 'goods_sold/comment', + 'goods_sold/rating', ] assert main_export_sheet['data'] == [ [ @@ -182,15 +182,15 @@ def tests_additional_field_exports_advanced(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_tone_of_voice/anxious', - 'record_a_note_tone_of_voice/excited', - 'record_a_note_tone_of_voice/confused', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/tone_of_voice/anxious', + 'record_a_note/tone_of_voice/excited', + 'record_a_note/tone_of_voice/confused', 'goods_sold/chocolate', 'goods_sold/fruit', 'goods_sold/pasta', - 'goods_sold_comment', - 'goods_sold_rating', + 'goods_sold/comment', + 'goods_sold/rating', ] assert main_export_sheet['data'] == [ [ @@ -238,11 +238,11 @@ def tests_additional_field_exports_advanced(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_tone_of_voice', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/tone_of_voice', 'goods_sold', - 'goods_sold_comment', - 'goods_sold_rating', + 'goods_sold/comment', + 'goods_sold/rating', ] assert main_export_sheet['data'] == [ [ @@ -284,10 +284,10 @@ def tests_additional_field_exports_v2(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_acme_timestamp', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/acme_timestamp', 'name_of_shop', - 'name_of_shop_comment', + 'name_of_shop/comment', ] response0 = main_export_sheet['data'][0] assert response0 == [ @@ -311,12 +311,12 @@ def tests_additional_field_exports_all_versions(): assert 6 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_acme_timestamp', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/acme_timestamp', 'name_of_shop', - 'name_of_shop_comment', + 'name_of_shop/comment', 'name_of_clerk', - 'name_of_clerk_comment', + 'name_of_clerk/comment', ] response0 = main_export_sheet['data'][0] assert response0 == [ @@ -399,12 +399,12 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Registri oficiston dirantan ion', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_acme_timestamp', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/acme_timestamp', 'Kio estas la nomo de la butiko?', - 'name_of_shop_comment', + 'name_of_shop/comment', 'name_of_clerk', - 'name_of_clerk_comment', + 'name_of_clerk/comment', ] options['lang'] = None @@ -414,12 +414,12 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note_transcription_acme_1_speech2text', - 'record_a_note_acme_timestamp', + 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/acme_timestamp', 'name_of_shop', - 'name_of_shop_comment', + 'name_of_shop/comment', 'name_of_clerk', - 'name_of_clerk_comment', + 'name_of_clerk/comment', ] def test_simple_report_with_analysis_form(): From 07a3cbfd7624aaee7f60090e9a713751b19430ce Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 01:35:58 +0000 Subject: [PATCH 20/54] return analysis form field labels to lists --- tests/fixtures/analysis_form/analysis_form.json | 16 ++++++++++++---- tests/test_additional_field_exports.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 1b30316d..edc4768f 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -12,7 +12,9 @@ "record_a_note", "transcript_acme_1_speech2text" ], - "label": ": ACME Transcription", + "label": [ + ": ACME Transcription" + ], "source": "record_a_note", "analysis_type": "transcript", "settings": { @@ -27,7 +29,9 @@ "record_a_note", "acme_timestamp" ], - "label": "Transcription Timestamp", + "label": [ + "Transcription Timestamp" + ], "source": "record_a_note" }, { @@ -37,7 +41,9 @@ "name_of_clerk", "comment" ], - "label": "Comment on the name of the clerk", + "label": [ + "Comment on the name of the clerk" + ], "source": "name_of_clerk" }, { @@ -47,7 +53,9 @@ "name_of_shop", "comment" ], - "label": "Comment on the name of the shop", + "label": [ + "Comment on the name of the shop" + ], "source": "name_of_shop" } ] diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index bd42e4b7..716efc49 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -2,7 +2,7 @@ from formpack import FormPack from .fixtures import build_fixture -def tests_additional_field_exports_x(): +def tests_additional_field_exports(): title, schemas, submissions, analysis_form = build_fixture('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) From af46ed561069ffcbcc808385906a1ed10859c51e Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 01:59:20 +0000 Subject: [PATCH 21/54] wip get tests passing again --- src/formpack/reporting/export.py | 13 ++++++++----- src/formpack/schema/fields.py | 3 +++ tests/fixtures/analysis_form/analysis_form.json | 3 +++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 71583bf5..a0bcd48c 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -385,11 +385,14 @@ def _get_attachment(val, field, attachments): def _get_value_from_entry(entry, field, supplemental_details): if field.analysis_question and supplemental_details: - sd = supplemental_details.get(field.name) - if isinstance(sd, str): - return sd - if isinstance(sd, list): - _v = [v['value'] for v in sd if v['_index'] == repeat_index] + sd = supplemental_details.get(field.source, {}) + if not sd: + return + val = sd.get(field.analysis_path[-1], '') + if isinstance(val, str): + return val + if isinstance(val, list): + _v = [v['value'] for v in val if v['_index'] == repeat_index] return _v[0] if _v else '' suffix = 'meta/' if field.data_type == 'audit' else '' diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index fc6e6f55..ac29cc54 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -35,6 +35,7 @@ def __init__(self, name, labels, data_type, hierarchy=None, self.source = source self.analysis_question = True self.analysis_type = kwargs.get('analysis_type') + self.analysis_path = kwargs.get('analysis_path') self.settings = kwargs.get('settings') hierarchy = list(hierarchy) if hierarchy is not None else [None] @@ -168,6 +169,7 @@ def from_json_definition(cls, definition, hierarchy=None, source = definition.get('source') analysis_type = definition.get('analysis_type', ANALYSIS_TYPE_CODING) settings = definition.get('settings', {}) + analysis_path = definition.get('path') # normalize spaces data_type = definition['type'] @@ -248,6 +250,7 @@ def from_json_definition(cls, definition, hierarchy=None, 'source': source, 'analysis_type': analysis_type, 'settings': settings, + 'analysis_path': analysis_path, } if data_type == 'select_multiple' and appearance == 'literacy': diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index edc4768f..5b80e37d 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -58,5 +58,8 @@ ], "source": "name_of_shop" } + ], + "translations": [ + "English (en)" ] } From 6ca0127a494291c45cec8734e90c8a3e9ae22774 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 02:05:45 +0000 Subject: [PATCH 22/54] use defaultdict to refactor --- src/formpack/version.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/formpack/version.py b/src/formpack/version.py index 6a1ce492..23032d97 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -1,5 +1,5 @@ # coding: utf-8 -from collections import OrderedDict +from collections import OrderedDict, defaultdict from pyxform import aliases as pyxform_aliases @@ -354,12 +354,9 @@ def __repr__(self): return f"" def _get_fields_by_source(self): - fields_by_source = {} + fields_by_source = defaultdict(list) for field in self.fields: - if field.source not in fields_by_source: - fields_by_source[field.source] = [field] - else: - fields_by_source[field.source].append(field) + fields_by_source[field.source].append(field) return fields_by_source def _map_sections_to_analysis_fields(self, survey_field): From b22bf79c784ef7cbd1183356434c345b9a9ea68f Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 18:56:54 +0000 Subject: [PATCH 23/54] refactor extracting supplemental value for export --- src/formpack/reporting/export.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index a0bcd48c..81f81e08 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -383,17 +383,27 @@ def _get_attachment(val, field, attachments): if re.match(fr'^.*/{_val}$', f['filename']) is not None ] + def _get_value_from_supplemental_details(field, supplemental_details): + source, name = field.analysis_path + _sup_details = supplemental_details.get(source, {}) + + if not _sup_details: + return + + val = _sup_details.get(name, '') + if isinstance(val, str): + return val + + # TODO: improve MVP handling of repeat groups for future + if isinstance(val, list): + _v = [v['value'] for v in val if v['_index'] == repeat_index] + return _v[0] if _v else '' + def _get_value_from_entry(entry, field, supplemental_details): if field.analysis_question and supplemental_details: - sd = supplemental_details.get(field.source, {}) - if not sd: - return - val = sd.get(field.analysis_path[-1], '') - if isinstance(val, str): - return val - if isinstance(val, list): - _v = [v['value'] for v in val if v['_index'] == repeat_index] - return _v[0] if _v else '' + return _get_value_from_supplemental_details( + field, supplemental_details + ) suffix = 'meta/' if field.data_type == 'audit' else '' return entry.get(f'{suffix}{field.path}') From 3a7429f744cad1f732649e6067fb8ddbe3b11454 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 30 Nov 2021 19:58:30 +0000 Subject: [PATCH 24/54] handle labels better and refactor AnalysisForm class --- src/formpack/version.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/formpack/version.py b/src/formpack/version.py index 23032d97..6d6f003b 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -326,6 +326,7 @@ def __init__(self, form_pack, schema): self.form_pack = form_pack survey = self.schema.get('additional_fields', []) + fields_by_name = {row['name']:row for row in survey} section = FormSection(name=form_pack.title) self.translations = [ @@ -338,16 +339,26 @@ def __init__(self, form_pack, schema): choices_definition, self.translations ) - self.fields = [ - FormField.from_json_definition( + for data_def in survey: + field = FormField.from_json_definition( definition=data_def, field_choices=field_choices, section=section, translations=self.translations, ) - for data_def in survey - ] + _f = fields_by_name[field.name] + _labels = LabelStruct() + if 'label' in _f: + if not isinstance(_f['label'], list): + _f['label'] = [_f['label']] + _labels = LabelStruct( + labels=_f['label'], translations=self.translations + ) + field.labels = _labels + section.fields[field.name] = field + + self.fields = list(section.fields.values()) self.fields_by_source = self._get_fields_by_source() def __repr__(self): From 5624352d1158060486bd36e31ec291f127a63afb Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Fri, 3 Dec 2021 21:58:53 +0000 Subject: [PATCH 25/54] refactor and type annotate AnalysisForm class methods --- src/formpack/version.py | 95 +++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/src/formpack/version.py b/src/formpack/version.py index 6d6f003b..c531f6e7 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -1,5 +1,11 @@ # coding: utf-8 from collections import OrderedDict, defaultdict +from typing import ( + Any, + Dict, + List, + Union, +) from pyxform import aliases as pyxform_aliases @@ -37,7 +43,33 @@ def get(self, key, default=None): return self._vals.get(key, default) -class FormVersion: +class BaseForm: + @staticmethod + def _get_translations(content: Dict[str, List[Any]]) -> List[str]: + return [ + t if t is not None else UNTRANSLATED + for t in content.get('translations', [None]) + ] + + @staticmethod + def _get_fields_by_name( + survey: Dict[str, Union[str, List[Any]]] + ) -> Dict[str, Dict[str, Union[str, List[Any]]]]: + return {row['name']: row for row in survey if 'name' in row} + + @staticmethod + def _get_field_labels( + field: FormField, + translations: List[str], + ) -> LabelStruct: + if 'label' in field: + if not isinstance(field['label'], list): + field['label'] = [field['label']] + return LabelStruct(labels=field['label'], translations=translations) + return LabelStruct() + + +class FormVersion(BaseForm): @classmethod def verify_schema_structure(cls, struct): if 'content' not in struct: @@ -89,15 +121,14 @@ def __init__(self, form_pack, schema): content = self.schema['content'] - self.translations = [t if t is not None else UNTRANSLATED - for t in content.get('translations', [None])] + self.translations = self._get_translations(content) # TODO: put those parts in a separate method and unit test it survey = content.get('survey', []) survey = self._append_pseudo_questions(survey) - fields_by_name = dict([(row.get('name'), row) for row in survey]) + fields_by_name = self._get_fields_by_name(survey) # Analyze the survey schema and extract the informations we need # to build the export: the sections, the choices, the fields @@ -197,15 +228,7 @@ def __init__(self, form_pack, schema): section.fields[field.name] = field _f = fields_by_name[field.name] - _labels = LabelStruct() - - if 'label' in _f: - if not isinstance(_f['label'], list): - _f['label'] = [_f['label']] - _labels = LabelStruct(labels=_f['label'], - translations=self.translations) - - field.labels = _labels + field.labels = self._get_field_labels(_f, self.translations) assert 'labels' not in _f # FIXME: Find a safe way to use this. Wrapping with try/except isn't enough @@ -319,20 +342,21 @@ def to_xml(self, warnings=None): return survey._to_pretty_xml() #.encode('utf-8') -class AnalysisForm: - def __init__(self, form_pack, schema): +class AnalysisForm(BaseForm): + def __init__( + self, + formpack: 'FormPack', + schema: Dict[str, Union[str, List[Any]]], + ) -> None: self.schema = schema - self.form_pack = form_pack + self.formpack = formpack survey = self.schema.get('additional_fields', []) - fields_by_name = {row['name']:row for row in survey} - section = FormSection(name=form_pack.title) + fields_by_name = self._get_fields_by_name(survey) + section = FormSection(name=formpack.title) - self.translations = [ - t if t is not None else UNTRANSLATED - for t in schema.get('translations', [None]) - ] + self.translations = self._get_translations(schema) choices_definition = schema.get('additional_choices', ()) field_choices = FormChoice.all_from_json_definition( @@ -347,37 +371,36 @@ def __init__(self, form_pack, schema): translations=self.translations, ) - _f = fields_by_name[field.name] - _labels = LabelStruct() - if 'label' in _f: - if not isinstance(_f['label'], list): - _f['label'] = [_f['label']] - _labels = LabelStruct( - labels=_f['label'], translations=self.translations - ) - field.labels = _labels + field.labels = self._get_field_labels( + field=fields_by_name[field.name], + translations=self.translations, + ) section.fields[field.name] = field self.fields = list(section.fields.values()) self.fields_by_source = self._get_fields_by_source() - def __repr__(self): - return f"" + def __repr__(self) -> str: + return f"" - def _get_fields_by_source(self): + def _get_fields_by_source(self) -> Dict[str, List[FormField]]: fields_by_source = defaultdict(list) for field in self.fields: fields_by_source[field.source].append(field) return fields_by_source - def _map_sections_to_analysis_fields(self, survey_field): + def _map_sections_to_analysis_fields( + self, survey_field: FormField + ) -> List[FormField]: _fields = [] for analysis_field in self.fields_by_source[survey_field.name]: analysis_field.section = survey_field.section _fields.append(analysis_field) return _fields - def insert_analysis_fields(self, fields): + def insert_analysis_fields( + self, fields: List[FormField] + ) -> List[FormField]: _fields = [] for field in fields: _fields.append(field) From a8f60dae68116d4d2f2e5adc5ea9e942964eb836 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 7 Dec 2021 17:55:14 +0000 Subject: [PATCH 26/54] remove `[Any]` from annotations --- src/formpack/version.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/formpack/version.py b/src/formpack/version.py index c531f6e7..e8da4003 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -1,7 +1,6 @@ # coding: utf-8 from collections import OrderedDict, defaultdict from typing import ( - Any, Dict, List, Union, @@ -45,7 +44,7 @@ def get(self, key, default=None): class BaseForm: @staticmethod - def _get_translations(content: Dict[str, List[Any]]) -> List[str]: + def _get_translations(content: Dict[str, List]) -> List[str]: return [ t if t is not None else UNTRANSLATED for t in content.get('translations', [None]) @@ -53,8 +52,8 @@ def _get_translations(content: Dict[str, List[Any]]) -> List[str]: @staticmethod def _get_fields_by_name( - survey: Dict[str, Union[str, List[Any]]] - ) -> Dict[str, Dict[str, Union[str, List[Any]]]]: + survey: Dict[str, Union[str, List]] + ) -> Dict[str, Dict[str, Union[str, List]]]: return {row['name']: row for row in survey if 'name' in row} @staticmethod @@ -346,7 +345,7 @@ class AnalysisForm(BaseForm): def __init__( self, formpack: 'FormPack', - schema: Dict[str, Union[str, List[Any]]], + schema: Dict[str, Union[str, List]], ) -> None: self.schema = schema From 33bfed560f6868fef991c251a33290b7794bb9a3 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 7 Dec 2021 18:03:41 +0000 Subject: [PATCH 27/54] add more type annotations --- src/formpack/pack.py | 2 +- src/formpack/reporting/export.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 444c158e..72bcb6e4 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -166,7 +166,7 @@ def load_version(self, schema): self.versions[form_version.id] = form_version - def extend_survey(self, analysis_form): + def extend_survey(self, analysis_form: AnalysisForm) -> None: self.analysis_form = AnalysisForm(self, analysis_form) def version_diff(self, vn1, vn2): diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 81f81e08..49d610e5 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -4,7 +4,12 @@ import zipfile from collections import defaultdict, OrderedDict from inspect import isclass -from typing import Iterator, Generator, Optional +from typing import ( + Dict, + Generator, + Iterator, + Optional, +) import xlsxwriter @@ -383,7 +388,9 @@ def _get_attachment(val, field, attachments): if re.match(fr'^.*/{_val}$', f['filename']) is not None ] - def _get_value_from_supplemental_details(field, supplemental_details): + def _get_value_from_supplemental_details( + field, supplemental_details: Dict + ) -> Optional[str]: source, name = field.analysis_path _sup_details = supplemental_details.get(source, {}) @@ -399,7 +406,9 @@ def _get_value_from_supplemental_details(field, supplemental_details): _v = [v['value'] for v in val if v['_index'] == repeat_index] return _v[0] if _v else '' - def _get_value_from_entry(entry, field, supplemental_details): + def _get_value_from_entry( + entry: Dict, field: FormField, supplemental_details: Dict + ) -> Optional[str]: if field.analysis_question and supplemental_details: return _get_value_from_supplemental_details( field, supplemental_details From 2cefe710e13c3f23b3ba95cf1a1fbed35bec86bc Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 7 Dec 2021 18:05:27 +0000 Subject: [PATCH 28/54] fix failing tests with missing import --- src/formpack/reporting/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 49d610e5..5a3bfe3e 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -18,7 +18,7 @@ TAG_COLUMNS_AND_SEPARATORS, UNSPECIFIED_TRANSLATION, ) -from ..schema import CopyField +from ..schema import CopyField, FormField from ..submission import FormSubmission from ..utils.exceptions import FormPackGeoJsonError from ..utils.flatten_content import flatten_tag_list From 431ee61c832b077cc39ccdfeee968d52ee581bd1 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 7 Dec 2021 18:25:48 +0000 Subject: [PATCH 29/54] clean up tests, remove `load_analysis_form_json()` from `build_fixture()` --- tests/fixtures/analysis_form/__init__.py | 3 +- .../analysis_form_advanced/__init__.py | 3 +- .../analysis_form_repeat_groups/__init__.py | 3 +- tests/fixtures/build_fixture.py | 3 - tests/test_additional_field_exports.py | 112 ++++-------------- tests/test_fixtures_valid.py | 6 +- 6 files changed, 32 insertions(+), 98 deletions(-) diff --git a/tests/fixtures/analysis_form/__init__.py b/tests/fixtures/analysis_form/__init__.py index 2f02da8f..c2e63cb6 100644 --- a/tests/fixtures/analysis_form/__init__.py +++ b/tests/fixtures/analysis_form/__init__.py @@ -3,7 +3,7 @@ analysis_form ''' -from ..load_fixture_json import load_fixture_json, load_analysis_form_json +from ..load_fixture_json import load_fixture_json DATA = { 'title': 'Simple Clerk Interaction', @@ -12,5 +12,4 @@ load_fixture_json('analysis_form/v1'), load_fixture_json('analysis_form/v2'), ], - 'analysis_form': load_analysis_form_json('analysis_form') } diff --git a/tests/fixtures/analysis_form_advanced/__init__.py b/tests/fixtures/analysis_form_advanced/__init__.py index 13f37942..cfa87315 100644 --- a/tests/fixtures/analysis_form_advanced/__init__.py +++ b/tests/fixtures/analysis_form_advanced/__init__.py @@ -3,7 +3,7 @@ analysis_form_advanced ''' -from ..load_fixture_json import load_fixture_json, load_analysis_form_json +from ..load_fixture_json import load_fixture_json DATA = { 'title': 'Advanced Clerk Interaction', @@ -11,5 +11,4 @@ 'versions': [ load_fixture_json('analysis_form_advanced/v1'), ], - 'analysis_form': load_analysis_form_json('analysis_form_advanced') } diff --git a/tests/fixtures/analysis_form_repeat_groups/__init__.py b/tests/fixtures/analysis_form_repeat_groups/__init__.py index 47f3e500..e484b9b2 100644 --- a/tests/fixtures/analysis_form_repeat_groups/__init__.py +++ b/tests/fixtures/analysis_form_repeat_groups/__init__.py @@ -3,7 +3,7 @@ analysis_form_repeat_groups ''' -from ..load_fixture_json import load_fixture_json, load_analysis_form_json +from ..load_fixture_json import load_fixture_json DATA = { 'title': 'Clerk Interaction Repeat Groups', @@ -11,5 +11,4 @@ 'versions': [ load_fixture_json('analysis_form_repeat_groups/v1'), ], - 'analysis_form': load_analysis_form_json('analysis_form_repeat_groups') } diff --git a/tests/fixtures/build_fixture.py b/tests/fixtures/build_fixture.py index 85017917..b6927cd9 100644 --- a/tests/fixtures/build_fixture.py +++ b/tests/fixtures/build_fixture.py @@ -16,7 +16,6 @@ def build_fixture(modulename, data_variable_name="DATA"): return fixtures title = fixtures.get('title') - analysis_form = fixtures.get('analysis_form') # separate the submissions from the schema schemas = [dict(v) for v in fixtures['versions']] @@ -28,8 +27,6 @@ def build_fixture(modulename, data_variable_name="DATA"): submission.update({version_id_key: version}) submissions.append(submission) - if analysis_form is not None: - return title, schemas, submissions, analysis_form return title, schemas, submissions diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 716efc49..c80af0f1 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -1,9 +1,11 @@ # coding: utf-8 from formpack import FormPack from .fixtures import build_fixture +from .fixtures.load_fixture_json import load_analysis_form_json def tests_additional_field_exports(): - title, schemas, submissions, analysis_form = build_fixture('analysis_form') + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -30,9 +32,8 @@ def tests_additional_field_exports(): ] def tests_additional_field_exports_repeat_groups(): - title, schemas, submissions, analysis_form = build_fixture( - 'analysis_form_repeat_groups' - ) + title, schemas, submissions = build_fixture('analysis_form_repeat_groups') + analysis_form = load_analysis_form_json('analysis_form_repeat_groups') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -100,9 +101,8 @@ def tests_additional_field_exports_repeat_groups(): assert repeat_data_expected_2 == repeat_data_response_2 def tests_additional_field_exports_advanced(): - title, schemas, submissions, analysis_form = build_fixture( - 'analysis_form_advanced' - ) + title, schemas, submissions = build_fixture('analysis_form_advanced') + analysis_form = load_analysis_form_json('analysis_form_advanced') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -272,7 +272,8 @@ def tests_additional_field_exports_advanced(): ] def tests_additional_field_exports_v2(): - title, schemas, submissions, analysis_form = build_fixture('analysis_form') + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -299,7 +300,8 @@ def tests_additional_field_exports_v2(): ] def tests_additional_field_exports_all_versions(): - title, schemas, submissions, analysis_form = build_fixture('analysis_form') + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -340,7 +342,8 @@ def tests_additional_field_exports_all_versions(): ] def tests_additional_field_exports_all_versions_exclude_fields(): - title, schemas, submissions, analysis_form = build_fixture('analysis_form') + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -369,7 +372,8 @@ def tests_additional_field_exports_all_versions_exclude_fields(): ] def tests_additional_field_exports_all_versions_langs(): - title, schemas, submissions, analysis_form = build_fixture('analysis_form') + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) @@ -423,86 +427,20 @@ def tests_additional_field_exports_all_versions_langs(): ] def test_simple_report_with_analysis_form(): - - title, schemas, submissions, analysis_form = build_fixture('analysis_form') + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title) pack.extend_survey(analysis_form) + lang = 'English (en)' report = pack.autoreport(versions=pack.versions.keys()) - stats = report.get_stats(submissions, lang='English (en)') + stats = report.get_stats(submissions, lang=lang) assert stats.submissions_count == 6 - stats = [(str(repr(f)), n, d) for f, n, d in stats] - - expected = [ - ( - "", - 'record_a_note', - { - 'total_count': 6, - 'not_provided': 0, - 'provided': 6, - 'show_graph': False, - 'frequency': [ - ('clerk_interaction_1.mp3', 1), - ('clerk_interaction_2.mp3', 1), - ('clerk_interaction_3.mp3', 1), - ('clerk_interaction_4.mp3', 1), - ('clerk_interaction_5.mp3', 1), - ('clerk_interaction_6.mp3', 1), - ], - 'percentage': [ - ('clerk_interaction_1.mp3', 16.67), - ('clerk_interaction_2.mp3', 16.67), - ('clerk_interaction_3.mp3', 16.67), - ('clerk_interaction_4.mp3', 16.67), - ('clerk_interaction_5.mp3', 16.67), - ('clerk_interaction_6.mp3', 16.67), - ], - }, - ), - ( - "", - "What is the shop's name?", - { - 'total_count': 6, - 'not_provided': 3, - 'provided': 3, - 'show_graph': False, - 'frequency': [ - ('Save On', 1), - ('Walmart', 1), - ('Costco', 1), - ], - 'percentage': [ - ('Save On', 16.67), - ('Walmart', 16.67), - ('Costco', 16.67), - ], - }, - ), - ( - "", - "What is the clerk's name?", - { - 'total_count': 6, - 'not_provided': 3, - 'provided': 3, - 'show_graph': False, - 'frequency': [ - ('John', 1), - ('Alex', 1), - ('Olivier', 1), - ], - 'percentage': [ - ('John', 16.67), - ('Alex', 16.67), - ('Olivier', 16.67), - ], - }, - ), - ] - - for i, stat in enumerate(stats): - assert stat == expected[i] + stats = set([n for f, n, d in stats]) + analysis_fields = set( + [f._get_label(lang=lang) for f in pack.analysis_form.fields] + ) + # Ensure analysis fields aren't making it into the report + assert stats - analysis_fields == stats diff --git a/tests/test_fixtures_valid.py b/tests/test_fixtures_valid.py index b202dd9b..552f9081 100644 --- a/tests/test_fixtures_valid.py +++ b/tests/test_fixtures_valid.py @@ -6,6 +6,7 @@ from formpack import FormPack from .fixtures import build_fixture +from .fixtures.load_fixture_json import load_analysis_form_json from formpack.constants import ANALYSIS_TYPES @@ -109,9 +110,10 @@ def test_xml_instances_loaded(self): def test_analysis_form(self): fixture = build_fixture('analysis_form') - assert 4 == len(fixture) + assert 3 == len(fixture) - title, schemas, submissions, analysis_form = fixture + title, schemas, submissions = fixture + analysis_form = load_analysis_form_json('analysis_form') fp = FormPack(schemas, title) fp.extend_survey(analysis_form) From 71d4c64bc36808bf41e8a79614f6dfb8383be09d Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 7 Dec 2021 18:30:31 +0000 Subject: [PATCH 30/54] update autoreport test --- tests/test_additional_field_exports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index c80af0f1..b79d7a8e 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -443,4 +443,4 @@ def test_simple_report_with_analysis_form(): [f._get_label(lang=lang) for f in pack.analysis_form.fields] ) # Ensure analysis fields aren't making it into the report - assert stats - analysis_fields == stats + assert not stats.intersection(analysis_fields) From eb483c04f75000ab6832f51eb1a83d7394cf32de Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Mon, 24 Jan 2022 20:42:53 +0000 Subject: [PATCH 31/54] support updated transcript structure, ammend tests, remove support for repeat groups --- src/formpack/pack.py | 3 ++- src/formpack/reporting/export.py | 16 +++------------ .../fixtures/analysis_form/analysis_form.json | 9 +++------ tests/fixtures/analysis_form/v1.json | 8 ++++++-- tests/fixtures/analysis_form/v2.json | 8 ++++++-- .../analysis_form_advanced/analysis_form.json | 10 +++------- tests/fixtures/analysis_form_advanced/v1.json | 8 ++++++-- .../analysis_form.json | 10 +++------- .../analysis_form_repeat_groups/v1.json | 2 +- tests/test_additional_field_exports.py | 20 ++++++++++--------- 10 files changed, 44 insertions(+), 50 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 72bcb6e4..cb9b4418 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -3,6 +3,7 @@ import json from collections import OrderedDict from copy import deepcopy +from typing import Dict from formpack.schema.fields import CopyField from .version import FormVersion, AnalysisForm @@ -166,7 +167,7 @@ def load_version(self, schema): self.versions[form_version.id] = form_version - def extend_survey(self, analysis_form: AnalysisForm) -> None: + def extend_survey(self, analysis_form: Dict) -> None: self.analysis_form = AnalysisForm(self, analysis_form) def version_diff(self, vn1, vn2): diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 5a3bfe3e..dfe5bbf6 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -322,7 +322,6 @@ def format_one_submission( current_section, attachments=None, supplemental_details=None, - repeat_index=0, ): # 'current_section' is the name of what will become sheets in xls. @@ -398,13 +397,10 @@ def _get_value_from_supplemental_details( return val = _sup_details.get(name, '') - if isinstance(val, str): - return val + if field.analysis_type == 'transcript': + return val['value'] - # TODO: improve MVP handling of repeat groups for future - if isinstance(val, list): - _v = [v['value'] for v in val if v['_index'] == repeat_index] - return _v[0] if _v else '' + return val def _get_value_from_entry( entry: Dict, field: FormField, supplemental_details: Dict @@ -519,7 +515,6 @@ def _get_value_from_entry( child_section, attachments=attachments, supplemental_details=supplemental_details, - repeat_index=repeat_index, ) for key, value in iter(chunk.items()): if key in chunks: @@ -527,12 +522,7 @@ def _get_value_from_entry( else: chunks[key] = value - # Reset the repeat index once we're done with this current - # repeat group - repeat_index = 0 - _indexes[_section_name] += 1 - repeat_index += 1 return chunks diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 5b80e37d..3e8e64b2 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -1,19 +1,16 @@ { "engines": { - "acme_1_speech2text": { + "transcript": { "details": "an external service provided by ACME, Inc." } }, "additional_fields": [ { "type": "text", - "name": "record_a_note/transcript_acme_1_speech2text", + "name": "record_a_note/transcript", "path": [ "record_a_note", - "transcript_acme_1_speech2text" - ], - "label": [ - ": ACME Transcription" + "transcript" ], "source": "record_a_note", "analysis_type": "transcript", diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index 0bcd0139..b4736ad2 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -53,7 +53,9 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": "Hello how may I help you?", + "transcript": { + "value": "Hello how may I help you?" + }, "acme_timestamp": "2021-11-01Z" }, "name_of_clerk": { @@ -72,7 +74,9 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": "Thank you for your business" + "transcript": { + "value": "Thank you for your business" + } } } }, diff --git a/tests/fixtures/analysis_form/v2.json b/tests/fixtures/analysis_form/v2.json index 5ac1bdc4..691447e4 100644 --- a/tests/fixtures/analysis_form/v2.json +++ b/tests/fixtures/analysis_form/v2.json @@ -53,7 +53,9 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": "Hello how may I help you?", + "transcript": { + "value": "Hello how may I help you?" + }, "acme_timestamp": "2021-11-01Z" }, "name_of_shop": { @@ -72,7 +74,9 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": "Thank you for your business" + "transcript": { + "value": "Thank you for your business" + } } } }, diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index d4e1e266..5a9e2be6 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -1,18 +1,14 @@ { "engines": { - "acme_1_speech2text": { + "transcript": { "details": "an external service provided by ACME, Inc." } }, "additional_fields": [ { "type": "text", - "name": "record_a_note/transcript_acme_1_speech2text", - "path": ["record_a_note", "transcript_acme_1_speech2text"], - "label": [ - ": ACME Transcription", - "" - ], + "name": "record_a_note/transcript", + "path": ["record_a_note", "transcript"], "source": "record_a_note", "analysis_type": "transcript", "settings": { diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index b482c0cf..a521af0d 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -79,7 +79,9 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": "Hello how may I help you?", + "transcript": { + "value": "Hello how may I help you?" + }, "tone_of_voice": "excited confused" }, "goods_sold": { @@ -99,7 +101,9 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": "Thank you for your business", + "transcript": { + "value": "Thank you for your business" + }, "tone_of_voice": "anxious excited" }, "goods_sold": { diff --git a/tests/fixtures/analysis_form_repeat_groups/analysis_form.json b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json index 1b715b72..8ccbe323 100644 --- a/tests/fixtures/analysis_form_repeat_groups/analysis_form.json +++ b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json @@ -1,18 +1,14 @@ { "engines": { - "acme_1_speech2text": { + "transcript": { "details": "an external service provided by ACME, Inc." } }, "additional_fields": [ { "type": "text", - "name": "record_a_note/transcript_acme_1_speech2text", - "path": ["record_a_note", "transcript_acme_1_speech2text"], - "label": [ - ": ACME Transcription", - "" - ], + "name": "record_a_note/transcript", + "path": ["record_a_note", "transcript"], "source": "record_a_note", "analysis_type": "transcript", "settings": { diff --git a/tests/fixtures/analysis_form_repeat_groups/v1.json b/tests/fixtures/analysis_form_repeat_groups/v1.json index eb121796..94b65319 100644 --- a/tests/fixtures/analysis_form_repeat_groups/v1.json +++ b/tests/fixtures/analysis_form_repeat_groups/v1.json @@ -141,7 +141,7 @@ ], "_supplementalDetails": { "record_a_note": { - "transcript_acme_1_speech2text": [ + "transcript": [ { "_index": 0, "value": "Hello how may I help you?" diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index b79d7a8e..588fe8aa 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -1,4 +1,5 @@ # coding: utf-8 +import unittest from formpack import FormPack from .fixtures import build_fixture from .fixtures.load_fixture_json import load_analysis_form_json @@ -17,7 +18,7 @@ def tests_additional_field_exports(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/acme_timestamp', 'name_of_clerk', 'name_of_clerk/comment', @@ -31,6 +32,7 @@ def tests_additional_field_exports(): 'Sounds like an interesting person', ] +@unittest.skip('Currently not supporting repeat groups') def tests_additional_field_exports_repeat_groups(): title, schemas, submissions = build_fixture('analysis_form_repeat_groups') analysis_form = load_analysis_form_json('analysis_form_repeat_groups') @@ -118,7 +120,7 @@ def tests_additional_field_exports_advanced(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/tone_of_voice', 'record_a_note/tone_of_voice/anxious', 'record_a_note/tone_of_voice/excited', @@ -182,7 +184,7 @@ def tests_additional_field_exports_advanced(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/tone_of_voice/anxious', 'record_a_note/tone_of_voice/excited', 'record_a_note/tone_of_voice/confused', @@ -238,7 +240,7 @@ def tests_additional_field_exports_advanced(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/tone_of_voice', 'goods_sold', 'goods_sold/comment', @@ -285,7 +287,7 @@ def tests_additional_field_exports_v2(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', @@ -313,7 +315,7 @@ def tests_additional_field_exports_all_versions(): assert 6 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', @@ -388,7 +390,7 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Record a clerk saying something', - ': ACME Transcription', + 'record_a_note/transcript', 'Transcription Timestamp', "What is the shop's name?", 'Comment on the name of the shop', @@ -403,7 +405,7 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Registri oficiston dirantan ion', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/acme_timestamp', 'Kio estas la nomo de la butiko?', 'name_of_shop/comment', @@ -418,7 +420,7 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript_acme_1_speech2text', + 'record_a_note/transcript', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', From 1cbd7a41b1b6df84a77de18743ea480bd406ce78 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Mon, 24 Jan 2022 23:26:21 +0000 Subject: [PATCH 32/54] handle translations --- src/formpack/reporting/export.py | 8 ++- src/formpack/schema/fields.py | 71 ++++++++++++++++--- .../fixtures/analysis_form/analysis_form.json | 18 +++++ tests/fixtures/analysis_form/v1.json | 5 ++ tests/test_additional_field_exports.py | 20 ++++++ 5 files changed, 112 insertions(+), 10 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index dfe5bbf6..8555ca2b 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -14,6 +14,7 @@ import xlsxwriter from ..constants import ( + ANALYSIS_TYPE_TRANSCRIPT, GEO_QUESTION_TYPES, TAG_COLUMNS_AND_SEPARATORS, UNSPECIFIED_TRANSLATION, @@ -396,8 +397,11 @@ def _get_value_from_supplemental_details( if not _sup_details: return - val = _sup_details.get(name, '') - if field.analysis_type == 'transcript': + val = _sup_details.get(name) + if val is None: + return '' + + if field.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: return val['value'] return val diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index ac29cc54..2682faea 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -11,6 +11,7 @@ from ..constants import ( ANALYSIS_TYPES, ANALYSIS_TYPE_CODING, + ANALYSIS_TYPE_TRANSLATION, UNSPECIFIED_TRANSLATION, ) from ..utils import singlemode @@ -424,6 +425,56 @@ def get_substats(self, stats, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATI class TextField(ExtendedFormField): + @property + def _is_translation(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + + def get_labels( + self, + lang=UNSPECIFIED_TRANSLATION, + group_sep="/", + hierarchy_in_labels=False, + multiple_select="both", + *args, + **kwargs, + ): + if self._is_translation: + return self.get_value_names() + args = lang, group_sep, hierarchy_in_labels, multiple_select + return [self._get_label(*args)] + + def get_value_names(self, multiple_select="both", *args, **kwargs): + if self._is_translation: + return [ + f'{self.name}_{code}' for code in self.settings['translations'] + ] + return super().get_value_names() + + def format( + self, + val, + lang=UNSPECIFIED_TRANSLATION, + group_sep="/", + hierarchy_in_labels=False, + multiple_select="both", + xls_types_as_text=True, + *args, + **kwargs, + ): + if val is None: + val = '' + + if self._is_translation: + cells = dict.fromkeys(self.get_value_names(), '') + for code in self.settings['translations']: + try: + val = val[code]['value'] + except (TypeError, KeyError): + val = '' + cells[f'{self.name}_{code}'] = val + return cells + + return {self.name: val} def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): @@ -436,19 +487,23 @@ def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): for key, val in top: percentage.append((key, self._get_percentage(val, total))) - stats.update({ - 'frequency': top, - 'percentage': percentage, - }) + stats.update( + { + 'frequency': top, + 'percentage': percentage, + } + ) return stats - def get_disaggregated_stats(self, metrics, top_splitters, - lang=UNSPECIFIED_TRANSLATION, limit=100): + def get_disaggregated_stats( + self, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATION, limit=100 + ): parent = super() - stats = parent.get_disaggregated_stats(metrics, top_splitters, lang, - limit) + stats = parent.get_disaggregated_stats( + metrics, top_splitters, lang, limit + ) substats = self.get_substats(stats, metrics, top_splitters, lang) # sort values by total frequency diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 3e8e64b2..04807432 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -19,6 +19,24 @@ "engine": "engines/acme_1_speech2text" } }, + { + "type": "text", + "name": "record_a_note/translated", + "path": [ + "record_a_note", + "translated" + ], + "source": "record_a_note", + "analysis_type": "translation", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_translate", + "translations": [ + "es", + "af" + ] + } + }, { "type": "datetime", "name": "record_a_note/acme_timestamp", diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index b4736ad2..eab08268 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -56,6 +56,11 @@ "transcript": { "value": "Hello how may I help you?" }, + "translated": { + "es": { + "value": "Saluton, kiel mi povas helpi vin?" + } + }, "acme_timestamp": "2021-11-01Z" }, "name_of_clerk": { diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 588fe8aa..f73b2a4a 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -19,6 +19,8 @@ def tests_additional_field_exports(): assert main_export_sheet['fields'] == [ 'record_a_note', 'record_a_note/transcript', + 'record_a_note/translated_es', + 'record_a_note/translated_af', 'record_a_note/acme_timestamp', 'name_of_clerk', 'name_of_clerk/comment', @@ -27,6 +29,8 @@ def tests_additional_field_exports(): assert response0 == [ 'clerk_interaction_1.mp3', 'Hello how may I help you?', + 'Saluton, kiel mi povas helpi vin?', + '', '2021-11-01Z', 'John', 'Sounds like an interesting person', @@ -288,6 +292,8 @@ def tests_additional_field_exports_v2(): assert main_export_sheet['fields'] == [ 'record_a_note', 'record_a_note/transcript', + 'record_a_note/translated_es', + 'record_a_note/translated_af', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', @@ -296,6 +302,8 @@ def tests_additional_field_exports_v2(): assert response0 == [ 'clerk_interaction_4.mp3', 'Hello how may I help you?', + '', + '', '2021-11-01Z', 'Save On', 'Pretty cliche', @@ -316,6 +324,8 @@ def tests_additional_field_exports_all_versions(): assert main_export_sheet['fields'] == [ 'record_a_note', 'record_a_note/transcript', + 'record_a_note/translated_es', + 'record_a_note/translated_af', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', @@ -326,6 +336,8 @@ def tests_additional_field_exports_all_versions(): assert response0 == [ 'clerk_interaction_1.mp3', 'Hello how may I help you?', + 'Saluton, kiel mi povas helpi vin?', + '', '2021-11-01Z', '', '', @@ -336,6 +348,8 @@ def tests_additional_field_exports_all_versions(): assert response3 == [ 'clerk_interaction_4.mp3', 'Hello how may I help you?', + '', + '', '2021-11-01Z', 'Save On', 'Pretty cliche', @@ -391,6 +405,8 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Record a clerk saying something', 'record_a_note/transcript', + 'record_a_note/translated_es', + 'record_a_note/translated_af', 'Transcription Timestamp', "What is the shop's name?", 'Comment on the name of the shop', @@ -406,6 +422,8 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Registri oficiston dirantan ion', 'record_a_note/transcript', + 'record_a_note/translated_es', + 'record_a_note/translated_af', 'record_a_note/acme_timestamp', 'Kio estas la nomo de la butiko?', 'name_of_shop/comment', @@ -421,6 +439,8 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'record_a_note', 'record_a_note/transcript', + 'record_a_note/translated_es', + 'record_a_note/translated_af', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', From bec0601b677a324d144b38194213e3967a39134e Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 8 Feb 2022 23:08:17 +0000 Subject: [PATCH 33/54] wip integrating changes from kpi --- src/formpack/reporting/export.py | 7 ++ src/formpack/schema/fields.py | 109 ++++++++++-------- src/formpack/version.py | 9 ++ .../fixtures/analysis_form/analysis_form.json | 30 +++-- tests/test_additional_field_exports.py | 4 +- 5 files changed, 90 insertions(+), 69 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 8555ca2b..e81f86ef 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -15,6 +15,7 @@ from ..constants import ( ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, GEO_QUESTION_TYPES, TAG_COLUMNS_AND_SEPARATORS, UNSPECIFIED_TRANSLATION, @@ -397,6 +398,9 @@ def _get_value_from_supplemental_details( if not _sup_details: return + if 'translated_' in name: + name = 'translated' + val = _sup_details.get(name) if val is None: return '' @@ -404,6 +408,9 @@ def _get_value_from_supplemental_details( if field.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: return val['value'] + if field.analysis_type == ANALYSIS_TYPE_TRANSLATION: + return val[field.language]['value'] + return val def _get_value_from_entry( diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 2682faea..37139b5c 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -11,6 +11,7 @@ from ..constants import ( ANALYSIS_TYPES, ANALYSIS_TYPE_CODING, + ANALYSIS_TYPE_TRANSCRIPT, ANALYSIS_TYPE_TRANSLATION, UNSPECIFIED_TRANSLATION, ) @@ -38,6 +39,10 @@ def __init__(self, name, labels, data_type, hierarchy=None, self.analysis_type = kwargs.get('analysis_type') self.analysis_path = kwargs.get('analysis_path') self.settings = kwargs.get('settings') + if self.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: + self.languages = kwargs.get('languages') + if self.analysis_type == ANALYSIS_TYPE_TRANSLATION: + self.language = kwargs.get('language') hierarchy = list(hierarchy) if hierarchy is not None else [None] self.hierarchy = hierarchy + [self] @@ -171,6 +176,8 @@ def from_json_definition(cls, definition, hierarchy=None, analysis_type = definition.get('analysis_type', ANALYSIS_TYPE_CODING) settings = definition.get('settings', {}) analysis_path = definition.get('path') + languages = definition.get('languages') + language = definition.get('language') # normalize spaces data_type = definition['type'] @@ -252,6 +259,8 @@ def from_json_definition(cls, definition, hierarchy=None, 'analysis_type': analysis_type, 'settings': settings, 'analysis_path': analysis_path, + 'language': language, + 'languages': languages, } if data_type == 'select_multiple' and appearance == 'literacy': @@ -425,56 +434,56 @@ def get_substats(self, stats, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATI class TextField(ExtendedFormField): - @property - def _is_translation(self): - return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION - - def get_labels( - self, - lang=UNSPECIFIED_TRANSLATION, - group_sep="/", - hierarchy_in_labels=False, - multiple_select="both", - *args, - **kwargs, - ): - if self._is_translation: - return self.get_value_names() - args = lang, group_sep, hierarchy_in_labels, multiple_select - return [self._get_label(*args)] - - def get_value_names(self, multiple_select="both", *args, **kwargs): - if self._is_translation: - return [ - f'{self.name}_{code}' for code in self.settings['translations'] - ] - return super().get_value_names() - - def format( - self, - val, - lang=UNSPECIFIED_TRANSLATION, - group_sep="/", - hierarchy_in_labels=False, - multiple_select="both", - xls_types_as_text=True, - *args, - **kwargs, - ): - if val is None: - val = '' - - if self._is_translation: - cells = dict.fromkeys(self.get_value_names(), '') - for code in self.settings['translations']: - try: - val = val[code]['value'] - except (TypeError, KeyError): - val = '' - cells[f'{self.name}_{code}'] = val - return cells - - return {self.name: val} + #@property + #def _is_translation(self): + # return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + + #def get_labels( + # self, + # lang=UNSPECIFIED_TRANSLATION, + # group_sep="/", + # hierarchy_in_labels=False, + # multiple_select="both", + # *args, + # **kwargs, + #): + # if self._is_translation: + # return self.get_value_names() + # args = lang, group_sep, hierarchy_in_labels, multiple_select + # return [self._get_label(*args)] + + #def get_value_names(self, multiple_select="both", *args, **kwargs): + # if self._is_translation: + # return [ + # f'{self.name}_{code}' for code in self.settings['translations'] + # ] + # return super().get_value_names() + + #def format( + # self, + # val, + # lang=UNSPECIFIED_TRANSLATION, + # group_sep="/", + # hierarchy_in_labels=False, + # multiple_select="both", + # xls_types_as_text=True, + # *args, + # **kwargs, + #): + # if val is None: + # val = '' + + # if self._is_translation: + # cells = dict.fromkeys(self.get_value_names(), '') + # for code in self.settings['translations']: + # try: + # val = val[code]['value'] + # except (TypeError, KeyError): + # val = '' + # cells[f'{self.name}_{code}'] = val + # return cells + + # return {self.name: val} def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): diff --git a/src/formpack/version.py b/src/formpack/version.py index e8da4003..554684b9 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -363,6 +363,15 @@ def __init__( ) for data_def in survey: + data_type = data_def['type'] + if data_type in ['translation', 'transcript']: + data_def.update( + { + 'type': 'text', + 'analysis_type': data_type, + } + ) + field = FormField.from_json_definition( definition=data_def, field_choices=field_choices, diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 04807432..a76391f4 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -1,40 +1,38 @@ { "engines": { - "transcript": { - "details": "an external service provided by ACME, Inc." + "engines/transcript_manual": { + "details": "A human provided transcription" } }, "additional_fields": [ { - "type": "text", + "type": "transcript", "name": "record_a_note/transcript", + "label": "record_a_note Transcript", "path": [ "record_a_note", "transcript" ], + "languages": ["en"], "source": "record_a_note", - "analysis_type": "transcript", "settings": { - "mode": "auto", - "engine": "engines/acme_1_speech2text" + "mode": "manual", + "engine": "engines/transcript_manual" } }, { - "type": "text", - "name": "record_a_note/translated", + "type": "translation", + "name": "record_a_note/translated_es", + "label": "record_a_note Translated (es)", + "language": "es", "path": [ "record_a_note", - "translated" + "translated_es" ], "source": "record_a_note", - "analysis_type": "translation", "settings": { - "mode": "auto", - "engine": "engines/acme_1_translate", - "translations": [ - "es", - "af" - ] + "mode": "manual", + "engine": "engines/translation_manual" } }, { diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index f73b2a4a..5bb21d7a 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -4,7 +4,7 @@ from .fixtures import build_fixture from .fixtures.load_fixture_json import load_analysis_form_json -def tests_additional_field_exports(): +def tests_additional_field_exports_xxx(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) @@ -20,7 +20,6 @@ def tests_additional_field_exports(): 'record_a_note', 'record_a_note/transcript', 'record_a_note/translated_es', - 'record_a_note/translated_af', 'record_a_note/acme_timestamp', 'name_of_clerk', 'name_of_clerk/comment', @@ -30,7 +29,6 @@ def tests_additional_field_exports(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Saluton, kiel mi povas helpi vin?', - '', '2021-11-01Z', 'John', 'Sounds like an interesting person', From 1e24c633cce7cbce55a62fb5e4a4a3ea16d72aed Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 9 Feb 2022 19:01:19 +0000 Subject: [PATCH 34/54] handle filtering exports and different language transcriptions for the same field --- src/formpack/reporting/export.py | 24 ++++++++++++------- .../fixtures/analysis_form/analysis_form.json | 4 ++-- tests/fixtures/analysis_form/v1.json | 6 +++-- tests/test_additional_field_exports.py | 14 +++++------ 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index e81f86ef..5e16d4e5 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -231,9 +231,6 @@ def get_fields_labels_tags_for_all_versions(self, all_fields = self.formpack.get_fields_for_versions(self.versions) - if self.analysis_form and self.include_analysis_fields: - all_fields = self.analysis_form.insert_analysis_fields(all_fields) - # Ensure that fields are filtered if they've been specified, otherwise # carry on as usual if self.filter_fields: @@ -243,6 +240,10 @@ def get_fields_labels_tags_for_all_versions(self, if field.path in self.filter_fields ] + # TODO: For MVP, just reattach additional fields to their source + if self.analysis_form and self.include_analysis_fields: + all_fields = self.analysis_form.insert_analysis_fields(all_fields) + all_sections = {} # List of fields we generate ourselves to add at the very end @@ -398,7 +399,7 @@ def _get_value_from_supplemental_details( if not _sup_details: return - if 'translated_' in name: + if 'translation' in name: name = 'translated' val = _sup_details.get(name) @@ -406,10 +407,12 @@ def _get_value_from_supplemental_details( return '' if field.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: - return val['value'] - - if field.analysis_type == ANALYSIS_TYPE_TRANSLATION: - return val[field.language]['value'] + val = f'[{val["languageCode"]}] {val["value"]}' + elif field.analysis_type == ANALYSIS_TYPE_TRANSLATION: + try: + val = val[field.language]['value'] + except KeyError: + val = '' return val @@ -433,6 +436,11 @@ def _get_value_from_entry( if field.path in self.filter_fields ) + # TODO: For MVP, just reattach additional fields to their source + if self.analysis_form and self.include_analysis_fields: + _fields = self.analysis_form.insert_analysis_fields(_fields) + + # 'rows' will contain all the formatted entries for the current # section. If you don't have repeat-group, there is only one section # with a row of size one. diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index a76391f4..926aef82 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -22,12 +22,12 @@ }, { "type": "translation", - "name": "record_a_note/translated_es", + "name": "record_a_note/translation_es", "label": "record_a_note Translated (es)", "language": "es", "path": [ "record_a_note", - "translated_es" + "translation_es" ], "source": "record_a_note", "settings": { diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index eab08268..0fd3e461 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -54,7 +54,8 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Hello how may I help you?" + "value": "Hello how may I help you?", + "languageCode": "en" }, "translated": { "es": { @@ -80,7 +81,8 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Thank you for your business" + "value": "Thank you for your business", + "languageCode": "en" } } } diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 5bb21d7a..6722fb44 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -10,7 +10,11 @@ def tests_additional_field_exports_xxx(): pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) - options = {'include_analysis_fields': True, 'versions': 'v1'} + options = { + 'include_analysis_fields': True, + 'versions': 'v1', + 'filter_fields': ['record_a_note'], + } export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] @@ -19,19 +23,15 @@ def tests_additional_field_exports_xxx(): assert main_export_sheet['fields'] == [ 'record_a_note', 'record_a_note/transcript', - 'record_a_note/translated_es', + 'record_a_note/translation_es', 'record_a_note/acme_timestamp', - 'name_of_clerk', - 'name_of_clerk/comment', ] response0 = main_export_sheet['data'][0] assert response0 == [ 'clerk_interaction_1.mp3', - 'Hello how may I help you?', + '[en] Hello how may I help you?', 'Saluton, kiel mi povas helpi vin?', '2021-11-01Z', - 'John', - 'Sounds like an interesting person', ] @unittest.skip('Currently not supporting repeat groups') From 15dbb28b162a7fc0d38e5fc589f4f00a622407d5 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 15 Feb 2022 21:26:27 +0000 Subject: [PATCH 35/54] use the source label for prefix of transcript and translation field label --- src/formpack/schema/fields.py | 40 +++++++++++++++----------- src/formpack/version.py | 1 + tests/test_additional_field_exports.py | 15 +++++++--- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 37139b5c..eae4b7f3 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -434,23 +434,31 @@ def get_substats(self, stats, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATI class TextField(ExtendedFormField): - #@property - #def _is_translation(self): - # return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + @property + def _is_transcript(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSCRIPT - #def get_labels( - # self, - # lang=UNSPECIFIED_TRANSLATION, - # group_sep="/", - # hierarchy_in_labels=False, - # multiple_select="both", - # *args, - # **kwargs, - #): - # if self._is_translation: - # return self.get_value_names() - # args = lang, group_sep, hierarchy_in_labels, multiple_select - # return [self._get_label(*args)] + @property + def _is_translation(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + + def get_labels( + self, + lang=UNSPECIFIED_TRANSLATION, + group_sep='/', + hierarchy_in_labels=False, + multiple_select='both', + *args, + **kwargs, + ): + args = lang, group_sep, hierarchy_in_labels, multiple_select + if hasattr(self, 'source') and lang != UNSPECIFIED_TRANSLATION: + source_label = self.source_field._get_label(*args) + if self._is_translation: + return [f'{source_label} - translation ({self.language})'] + elif self._is_transcript: + return [f'{source_label} - transcript'] + return [self._get_label(*args)] #def get_value_names(self, multiple_select="both", *args, **kwargs): # if self._is_translation: diff --git a/src/formpack/version.py b/src/formpack/version.py index 554684b9..731788cb 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -403,6 +403,7 @@ def _map_sections_to_analysis_fields( _fields = [] for analysis_field in self.fields_by_source[survey_field.name]: analysis_field.section = survey_field.section + analysis_field.source_field = survey_field _fields.append(analysis_field) return _fields diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 6722fb44..ea3b1dc5 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -14,17 +14,24 @@ def tests_additional_field_exports_xxx(): 'include_analysis_fields': True, 'versions': 'v1', 'filter_fields': ['record_a_note'], + 'lang': 'English (en)' } export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] assert 3 == len(main_export_sheet['data']) + #assert main_export_sheet['fields'] == [ + # 'record_a_note', + # 'record_a_note/transcript', + # 'record_a_note/translation_es', + # 'record_a_note/acme_timestamp', + #] assert main_export_sheet['fields'] == [ - 'record_a_note', - 'record_a_note/transcript', - 'record_a_note/translation_es', - 'record_a_note/acme_timestamp', + 'Record a clerk saying something', + 'Record a clerk saying something - transcript', + 'Record a clerk saying something - translation (es)', + 'Transcription Timestamp', ] response0 = main_export_sheet['data'][0] assert response0 == [ From aee1e0dacb4e9a30ed8e1aa6b4b025eec39ac7a6 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 16 Feb 2022 20:11:03 +0000 Subject: [PATCH 36/54] apply requested changes to transcript export format --- src/formpack/reporting/export.py | 2 +- src/formpack/schema/fields.py | 65 +++++++++---------- .../fixtures/analysis_form/analysis_form.json | 17 ++++- tests/fixtures/analysis_form/v1.json | 8 +-- tests/test_additional_field_exports.py | 12 +++- 5 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 5e16d4e5..1075d248 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -407,7 +407,7 @@ def _get_value_from_supplemental_details( return '' if field.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: - val = f'[{val["languageCode"]}] {val["value"]}' + return val elif field.analysis_type == ANALYSIS_TYPE_TRANSLATION: try: val = val[field.language]['value'] diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index eae4b7f3..88f8e1fd 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -457,41 +457,40 @@ def get_labels( if self._is_translation: return [f'{source_label} - translation ({self.language})'] elif self._is_transcript: - return [f'{source_label} - transcript'] + return [f'{source_label} - transcript ({code})' for code in self.languages] + if self._is_transcript: + return self.get_value_names() return [self._get_label(*args)] - #def get_value_names(self, multiple_select="both", *args, **kwargs): - # if self._is_translation: - # return [ - # f'{self.name}_{code}' for code in self.settings['translations'] - # ] - # return super().get_value_names() - - #def format( - # self, - # val, - # lang=UNSPECIFIED_TRANSLATION, - # group_sep="/", - # hierarchy_in_labels=False, - # multiple_select="both", - # xls_types_as_text=True, - # *args, - # **kwargs, - #): - # if val is None: - # val = '' - - # if self._is_translation: - # cells = dict.fromkeys(self.get_value_names(), '') - # for code in self.settings['translations']: - # try: - # val = val[code]['value'] - # except (TypeError, KeyError): - # val = '' - # cells[f'{self.name}_{code}'] = val - # return cells - - # return {self.name: val} + def get_value_names(self, multiple_select='both', *args, **kwargs): + if self._is_transcript: + return [ + f'{self.name}_{code}' for code in self.languages + ] + return super().get_value_names() + + def format( + self, + val, + lang=UNSPECIFIED_TRANSLATION, + group_sep='/', + hierarchy_in_labels=False, + multiple_select='both', + xls_types_as_text=True, + *args, + **kwargs, + ): + if val is None: + val = '' + + if self._is_transcript: + cells = dict.fromkeys(self.get_value_names(), '') + # TODO why is this necessary?? + if isinstance(val, dict): + cells[f'{self.name}_{val["languageCode"]}'] = val['value'] + return cells + + return {self.name: val} def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 926aef82..56c46a8a 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -13,13 +13,28 @@ "record_a_note", "transcript" ], - "languages": ["en"], + "languages": ["en", "es"], "source": "record_a_note", "settings": { "mode": "manual", "engine": "engines/transcript_manual" } }, + { + "type": "translation", + "name": "record_a_note/translation_en", + "label": "record_a_note Translated (en)", + "language": "en", + "path": [ + "record_a_note", + "translation_en" + ], + "source": "record_a_note", + "settings": { + "mode": "manual", + "engine": "engines/translation_manual" + } + }, { "type": "translation", "name": "record_a_note/translation_es", diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index 0fd3e461..7aac0100 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -54,12 +54,12 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Hello how may I help you?", - "languageCode": "en" + "value": "Saluton, kiel mi povas helpi vin?", + "languageCode": "es" }, "translated": { - "es": { - "value": "Saluton, kiel mi povas helpi vin?" + "en": { + "value": "Hello how may I help you?" } }, "acme_timestamp": "2021-11-01Z" diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index ea3b1dc5..736c9861 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -23,21 +23,27 @@ def tests_additional_field_exports_xxx(): assert 3 == len(main_export_sheet['data']) #assert main_export_sheet['fields'] == [ # 'record_a_note', - # 'record_a_note/transcript', + # 'record_a_note/transcript_en', + # 'record_a_note/transcript_es', + # 'record_a_note/translation_en', # 'record_a_note/translation_es', # 'record_a_note/acme_timestamp', #] assert main_export_sheet['fields'] == [ 'Record a clerk saying something', - 'Record a clerk saying something - transcript', + 'Record a clerk saying something - transcript (en)', + 'Record a clerk saying something - transcript (es)', + 'Record a clerk saying something - translation (en)', 'Record a clerk saying something - translation (es)', 'Transcription Timestamp', ] response0 = main_export_sheet['data'][0] assert response0 == [ 'clerk_interaction_1.mp3', - '[en] Hello how may I help you?', + '', 'Saluton, kiel mi povas helpi vin?', + 'Hello how may I help you?', + '', '2021-11-01Z', ] From 2f290b11152d605e17dfec67000e9ae8247c3b24 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 16 Feb 2022 21:34:48 +0000 Subject: [PATCH 37/54] update handling of transcript export formatting --- src/formpack/schema/fields.py | 10 +++------ tests/test_additional_field_exports.py | 30 +++++++++++++------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 88f8e1fd..658781d7 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -452,21 +452,17 @@ def get_labels( **kwargs, ): args = lang, group_sep, hierarchy_in_labels, multiple_select - if hasattr(self, 'source') and lang != UNSPECIFIED_TRANSLATION: + if hasattr(self, 'source'): source_label = self.source_field._get_label(*args) if self._is_translation: return [f'{source_label} - translation ({self.language})'] elif self._is_transcript: return [f'{source_label} - transcript ({code})' for code in self.languages] - if self._is_transcript: - return self.get_value_names() return [self._get_label(*args)] def get_value_names(self, multiple_select='both', *args, **kwargs): if self._is_transcript: - return [ - f'{self.name}_{code}' for code in self.languages - ] + return [f'{self.source_field.name} - transcript ({code})' for code in self.languages] return super().get_value_names() def format( @@ -487,7 +483,7 @@ def format( cells = dict.fromkeys(self.get_value_names(), '') # TODO why is this necessary?? if isinstance(val, dict): - cells[f'{self.name}_{val["languageCode"]}'] = val['value'] + cells[f'{self.source_field.name} - transcript ({val["languageCode"]})'] = val['value'] return cells return {self.name: val} diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 736c9861..98d88c23 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -14,29 +14,29 @@ def tests_additional_field_exports_xxx(): 'include_analysis_fields': True, 'versions': 'v1', 'filter_fields': ['record_a_note'], - 'lang': 'English (en)' + #'lang': 'English (en)' } export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] assert 3 == len(main_export_sheet['data']) - #assert main_export_sheet['fields'] == [ - # 'record_a_note', - # 'record_a_note/transcript_en', - # 'record_a_note/transcript_es', - # 'record_a_note/translation_en', - # 'record_a_note/translation_es', - # 'record_a_note/acme_timestamp', - #] assert main_export_sheet['fields'] == [ - 'Record a clerk saying something', - 'Record a clerk saying something - transcript (en)', - 'Record a clerk saying something - transcript (es)', - 'Record a clerk saying something - translation (en)', - 'Record a clerk saying something - translation (es)', - 'Transcription Timestamp', + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', + 'record_a_note/acme_timestamp', ] + #assert main_export_sheet['fields'] == [ + # 'Record a clerk saying something', + # 'Record a clerk saying something - transcript (en)', + # 'Record a clerk saying something - transcript (es)', + # 'Record a clerk saying something - translation (en)', + # 'Record a clerk saying something - translation (es)', + # 'Transcription Timestamp', + #] response0 = main_export_sheet['data'][0] assert response0 == [ 'clerk_interaction_1.mp3', From 79c65fb11c345b91a1d470bf7754d6734f82e6da Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 17 Feb 2022 19:27:26 +0000 Subject: [PATCH 38/54] minor cleanup --- src/formpack/reporting/export.py | 4 +--- src/formpack/schema/fields.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 1075d248..6b0e13b1 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -406,9 +406,7 @@ def _get_value_from_supplemental_details( if val is None: return '' - if field.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: - return val - elif field.analysis_type == ANALYSIS_TYPE_TRANSLATION: + if field.analysis_type == ANALYSIS_TYPE_TRANSLATION: try: val = val[field.language]['value'] except KeyError: diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 658781d7..189f1911 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -481,9 +481,10 @@ def format( if self._is_transcript: cells = dict.fromkeys(self.get_value_names(), '') - # TODO why is this necessary?? if isinstance(val, dict): - cells[f'{self.source_field.name} - transcript ({val["languageCode"]})'] = val['value'] + cells[ + f'{self.source_field.name} - transcript ({val["languageCode"]})' + ] = val['value'] return cells return {self.name: val} From 484a0ebc5e6059c5111f3b16d2ae503de061e425 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 22 Mar 2022 19:31:51 +0000 Subject: [PATCH 39/54] black formatting --- src/formpack/reporting/export.py | 9 +- src/formpack/schema/fields.py | 10 +- src/formpack/version.py | 2 +- tests/fixtures/load_fixture_json.py | 1 + tests/test_additional_field_exports.py | 12 +- tests/test_exports.py | 230 +++++++++++-------------- tests/test_fixtures_valid.py | 5 +- tests/test_utils_flatten_content.py | 17 +- 8 files changed, 129 insertions(+), 157 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 07c55b71..be74af8d 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -441,16 +441,13 @@ def _get_value_from_entry( # carry on as usual if self.filter_fields: _fields = tuple( - field - for field in _fields - if field.path in self.filter_fields + field for field in _fields if field.path in self.filter_fields ) # TODO: For MVP, just reattach additional fields to their source if self.analysis_form and self.include_analysis_fields: _fields = self.analysis_form.insert_analysis_fields(_fields) - # 'rows' will contain all the formatted entries for the current # section. If you don't have repeat-group, there is only one section # with a row of size one. @@ -487,7 +484,9 @@ def _get_value_from_entry( # TODO: pass a context to fields so they can all format ? if field.can_format: # get submission value for this field - val = _get_value_from_entry(entry, field, supplemental_details) + val = _get_value_from_entry( + entry, field, supplemental_details + ) # get the attachment for this field attachment = _get_attachment(val, field, attachments) # get a mapping of {"col_name": "val", ...} diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 343f3abc..c580c53e 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -481,12 +481,18 @@ def get_labels( if self._is_translation: return [f'{source_label} - translation ({self.language})'] elif self._is_transcript: - return [f'{source_label} - transcript ({code})' for code in self.languages] + return [ + f'{source_label} - transcript ({code})' + for code in self.languages + ] return [self._get_label(*args)] def get_value_names(self, multiple_select='both', *args, **kwargs): if self._is_transcript: - return [f'{self.source_field.name} - transcript ({code})' for code in self.languages] + return [ + f'{self.source_field.name} - transcript ({code})' + for code in self.languages + ] return super().get_value_names() def format( diff --git a/src/formpack/version.py b/src/formpack/version.py index 4fbf7b40..40b361ce 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -363,7 +363,7 @@ def to_xml(self, warnings=None): } ) - return survey._to_pretty_xml() #.encode('utf-8') + return survey._to_pretty_xml() # .encode('utf-8') class AnalysisForm(BaseForm): diff --git a/tests/fixtures/load_fixture_json.py b/tests/fixtures/load_fixture_json.py index 4ebabbfc..482974c9 100644 --- a/tests/fixtures/load_fixture_json.py +++ b/tests/fixtures/load_fixture_json.py @@ -10,6 +10,7 @@ def load_fixture_json(fname): content_ = ff.read() return json.loads(content_) + def load_analysis_form_json(path): with open(os.path.join(CUR_DIR, path, 'analysis_form.json')) as f: return json.loads(f.read()) diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 98d88c23..2859a02d 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -4,6 +4,7 @@ from .fixtures import build_fixture from .fixtures.load_fixture_json import load_analysis_form_json + def tests_additional_field_exports_xxx(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') @@ -29,14 +30,14 @@ def tests_additional_field_exports_xxx(): 'record_a_note - translation (es)', 'record_a_note/acme_timestamp', ] - #assert main_export_sheet['fields'] == [ + # assert main_export_sheet['fields'] == [ # 'Record a clerk saying something', # 'Record a clerk saying something - transcript (en)', # 'Record a clerk saying something - transcript (es)', # 'Record a clerk saying something - translation (en)', # 'Record a clerk saying something - translation (es)', # 'Transcription Timestamp', - #] + # ] response0 = main_export_sheet['data'][0] assert response0 == [ 'clerk_interaction_1.mp3', @@ -47,6 +48,7 @@ def tests_additional_field_exports_xxx(): '2021-11-01Z', ] + @unittest.skip('Currently not supporting repeat groups') def tests_additional_field_exports_repeat_groups(): title, schemas, submissions = build_fixture('analysis_form_repeat_groups') @@ -117,6 +119,7 @@ def tests_additional_field_exports_repeat_groups(): ] assert repeat_data_expected_2 == repeat_data_response_2 + def tests_additional_field_exports_advanced(): title, schemas, submissions = build_fixture('analysis_form_advanced') analysis_form = load_analysis_form_json('analysis_form_advanced') @@ -288,6 +291,7 @@ def tests_additional_field_exports_advanced(): ], ] + def tests_additional_field_exports_v2(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') @@ -320,6 +324,7 @@ def tests_additional_field_exports_v2(): 'Pretty cliche', ] + def tests_additional_field_exports_all_versions(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') @@ -368,6 +373,7 @@ def tests_additional_field_exports_all_versions(): '', ] + def tests_additional_field_exports_all_versions_exclude_fields(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') @@ -398,6 +404,7 @@ def tests_additional_field_exports_all_versions_exclude_fields(): '', ] + def tests_additional_field_exports_all_versions_langs(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') @@ -459,6 +466,7 @@ def tests_additional_field_exports_all_versions_langs(): 'name_of_clerk/comment', ] + def test_simple_report_with_analysis_form(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') diff --git a/tests/test_exports.py b/tests/test_exports.py index e05d0170..ab61093d 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -372,7 +372,9 @@ def test_submissions_of_group_exports(self): ) def test_translations_labels_mismatch(self): - title, schemas, submissions = build_fixture('translations_labels_mismatch') + title, schemas, submissions = build_fixture( + 'translations_labels_mismatch' + ) with self.assertRaises(TranslationError) as e: fp = FormPack(schemas, title) @@ -1194,138 +1196,100 @@ def test_nested_repeats_with_xls_types(self): fp = FormPack(schemas, title) options = {'versions': 'bird_nests_v2', 'xls_types_as_text': False} export_dict = fp.export(**options).to_dict(submissions) - expected_dict = OrderedDict([ - ('Bird nest survey with nested repeatable groups', { - 'fields': [ - 'start', - 'end', - '_index' - ], - 'data': [ - [ - parser.parse('2017-12-27T15:53:26.000-05:00'), - parser.parse('2017-12-27T15:58:20.000-05:00'), - 1 - ], - [ - parser.parse('2017-12-27T15:58:20.000-05:00'), - parser.parse('2017-12-27T15:58:50.000-05:00'), - 2 - ] - ] - }), - ('group_tree', { - 'fields': [ - 'What_kind_of_tree_is_this', - '_index', - '_parent_table_name', - '_parent_index' - ], - 'data': [ - [ - 'pine', - 1, - 'Bird nest survey with nested repeatable groups', - 1 - ], - [ - 'spruce', - 2, - 'Bird nest survey with nested repeatable groups', - 1 - ], - [ - 'nan', - 3, - 'Bird nest survey with nested repeatable groups', - 2 - ] - ] - }), - ('group_nest', { - 'fields': [ - 'How_high_above_the_ground_is_the_nest', - 'How_many_eggs_are_in_the_nest', - '_index', - '_parent_table_name', - '_parent_index' - ], - 'data': [ - [ - 13, - 3, - 1, - 'group_tree', - 1 - ], - [ - 15, - 1, - 2, - 'group_tree', - 1 - ], - [ - 10, - 2, - 3, - 'group_tree', - 2 - ], - [ - 23, - 1, - 4, - 'group_tree', - 3 - ] - ] - }), - ('group_egg', { - 'fields': [ - 'Describe_the_egg', - '_parent_table_name', - '_parent_index' - ], - 'data': [ - [ - 'brown and speckled; medium', - 'group_nest', - 1 - ], - [ - 'brown and speckled; large; cracked', - 'group_nest', - 1 - ], - [ - 'light tan; small', - 'group_nest', - 1 - ], - [ - 'cream-colored', - 'group_nest', - 2 - ], - [ - 'reddish-brown; medium', - 'group_nest', - 3 - ], - [ - 'reddish-brown; small', - 'group_nest', - 3 - ], - [ - 'grey and speckled', - 'group_nest', - 4 - ] - ] - }) - ]) + expected_dict = OrderedDict( + [ + ( + 'Bird nest survey with nested repeatable groups', + { + 'fields': ['start', 'end', '_index'], + 'data': [ + [ + parser.parse('2017-12-27T15:53:26.000-05:00'), + parser.parse('2017-12-27T15:58:20.000-05:00'), + 1, + ], + [ + parser.parse('2017-12-27T15:58:20.000-05:00'), + parser.parse('2017-12-27T15:58:50.000-05:00'), + 2, + ], + ], + }, + ), + ( + 'group_tree', + { + 'fields': [ + 'What_kind_of_tree_is_this', + '_index', + '_parent_table_name', + '_parent_index', + ], + 'data': [ + [ + 'pine', + 1, + 'Bird nest survey with nested repeatable groups', + 1, + ], + [ + 'spruce', + 2, + 'Bird nest survey with nested repeatable groups', + 1, + ], + [ + 'nan', + 3, + 'Bird nest survey with nested repeatable groups', + 2, + ], + ], + }, + ), + ( + 'group_nest', + { + 'fields': [ + 'How_high_above_the_ground_is_the_nest', + 'How_many_eggs_are_in_the_nest', + '_index', + '_parent_table_name', + '_parent_index', + ], + 'data': [ + [13, 3, 1, 'group_tree', 1], + [15, 1, 2, 'group_tree', 1], + [10, 2, 3, 'group_tree', 2], + [23, 1, 4, 'group_tree', 3], + ], + }, + ), + ( + 'group_egg', + { + 'fields': [ + 'Describe_the_egg', + '_parent_table_name', + '_parent_index', + ], + 'data': [ + ['brown and speckled; medium', 'group_nest', 1], + [ + 'brown and speckled; large; cracked', + 'group_nest', + 1, + ], + ['light tan; small', 'group_nest', 1], + ['cream-colored', 'group_nest', 2], + ['reddish-brown; medium', 'group_nest', 3], + ['reddish-brown; small', 'group_nest', 3], + ['grey and speckled', 'group_nest', 4], + ], + }, + ), + ] + ) self.assertEqual(export_dict, expected_dict) def test_repeats_alias(self): diff --git a/tests/test_fixtures_valid.py b/tests/test_fixtures_valid.py index f7e466df..0d647b26 100644 --- a/tests/test_fixtures_valid.py +++ b/tests/test_fixtures_valid.py @@ -136,10 +136,7 @@ def test_analysis_form(self): [f['name'] for f in analysis_form['additional_fields']] ) actual_analysis_questions = sorted( - [ - f.name - for f in fp.analysis_form.fields - ] + [f.name for f in fp.analysis_form.fields] ) assert expected_analysis_questions == actual_analysis_questions diff --git a/tests/test_utils_flatten_content.py b/tests/test_utils_flatten_content.py index e236f4b8..bdc6d2e4 100644 --- a/tests/test_utils_flatten_content.py +++ b/tests/test_utils_flatten_content.py @@ -495,16 +495,13 @@ def test_flatten_tags(): def test_col_order(): - assert ( - _order_cols( - [ - 'label', - 'type', - 'name', - ] - ) - == ['type', 'name', 'label'] - ) + assert _order_cols( + [ + 'label', + 'type', + 'name', + ] + ) == ['type', 'name', 'label'] def test_flatten_translated_label_with_xpath(): From e6c4abedcc66d99e1862a336bae0be59a49de7f0 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 22 Mar 2022 20:11:23 +0000 Subject: [PATCH 40/54] fix breaking additional fields tests --- tests/fixtures/analysis_form/v2.json | 6 +- .../analysis_form_advanced/analysis_form.json | 4 +- tests/fixtures/analysis_form_advanced/v1.json | 6 +- tests/test_additional_field_exports.py | 105 ++++++++++++------ 4 files changed, 80 insertions(+), 41 deletions(-) diff --git a/tests/fixtures/analysis_form/v2.json b/tests/fixtures/analysis_form/v2.json index 691447e4..f4bb2961 100644 --- a/tests/fixtures/analysis_form/v2.json +++ b/tests/fixtures/analysis_form/v2.json @@ -54,7 +54,8 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Hello how may I help you?" + "value": "Hello how may I help you?", + "languageCode": "en" }, "acme_timestamp": "2021-11-01Z" }, @@ -75,7 +76,8 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Thank you for your business" + "value": "Thank you for your business", + "languageCode": "en" } } } diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 5a9e2be6..00904534 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -6,11 +6,11 @@ }, "additional_fields": [ { - "type": "text", + "type": "transcript", "name": "record_a_note/transcript", "path": ["record_a_note", "transcript"], "source": "record_a_note", - "analysis_type": "transcript", + "languages": ["en"], "settings": { "mode": "auto", "engine": "engines/acme_1_speech2text" diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index a521af0d..16ec1297 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -80,7 +80,8 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Hello how may I help you?" + "value": "Hello how may I help you?", + "languageCode": "en" }, "tone_of_voice": "excited confused" }, @@ -102,7 +103,8 @@ "_supplementalDetails": { "record_a_note": { "transcript": { - "value": "Thank you for your business" + "value": "Thank you for your business", + "languageCode": "en" }, "tone_of_voice": "anxious excited" }, diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 2859a02d..140813ad 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -5,7 +5,7 @@ from .fixtures.load_fixture_json import load_analysis_form_json -def tests_additional_field_exports_xxx(): +def test_additional_field_exports_without_labels(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) @@ -15,7 +15,6 @@ def tests_additional_field_exports_xxx(): 'include_analysis_fields': True, 'versions': 'v1', 'filter_fields': ['record_a_note'], - #'lang': 'English (en)' } export = pack.export(**options) values = export.to_dict(submissions) @@ -30,14 +29,42 @@ def tests_additional_field_exports_xxx(): 'record_a_note - translation (es)', 'record_a_note/acme_timestamp', ] - # assert main_export_sheet['fields'] == [ - # 'Record a clerk saying something', - # 'Record a clerk saying something - transcript (en)', - # 'Record a clerk saying something - transcript (es)', - # 'Record a clerk saying something - translation (en)', - # 'Record a clerk saying something - translation (es)', - # 'Transcription Timestamp', - # ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + '', + 'Saluton, kiel mi povas helpi vin?', + 'Hello how may I help you?', + '', + '2021-11-01Z', + ] + + +def test_additional_field_exports_with_labels(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'include_analysis_fields': True, + 'versions': 'v1', + 'filter_fields': ['record_a_note'], + 'lang': 'English (en)' + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'Record a clerk saying something', + 'Record a clerk saying something - transcript (en)', + 'Record a clerk saying something - transcript (es)', + 'Record a clerk saying something - translation (en)', + 'Record a clerk saying something - translation (es)', + 'Transcription Timestamp', + ] response0 = main_export_sheet['data'][0] assert response0 == [ 'clerk_interaction_1.mp3', @@ -50,7 +77,7 @@ def tests_additional_field_exports_xxx(): @unittest.skip('Currently not supporting repeat groups') -def tests_additional_field_exports_repeat_groups(): +def test_additional_field_exports_repeat_groups(): title, schemas, submissions = build_fixture('analysis_form_repeat_groups') analysis_form = load_analysis_form_json('analysis_form_repeat_groups') pack = FormPack(schemas, title=title) @@ -120,7 +147,7 @@ def tests_additional_field_exports_repeat_groups(): assert repeat_data_expected_2 == repeat_data_response_2 -def tests_additional_field_exports_advanced(): +def test_additional_field_exports_advanced(): title, schemas, submissions = build_fixture('analysis_form_advanced') analysis_form = load_analysis_form_json('analysis_form_advanced') pack = FormPack(schemas, title=title) @@ -138,7 +165,7 @@ def tests_additional_field_exports_advanced(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript', + 'record_a_note - transcript (en)', 'record_a_note/tone_of_voice', 'record_a_note/tone_of_voice/anxious', 'record_a_note/tone_of_voice/excited', @@ -202,7 +229,7 @@ def tests_additional_field_exports_advanced(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript', + 'record_a_note - transcript (en)', 'record_a_note/tone_of_voice/anxious', 'record_a_note/tone_of_voice/excited', 'record_a_note/tone_of_voice/confused', @@ -258,7 +285,7 @@ def tests_additional_field_exports_advanced(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript', + 'record_a_note - transcript (en)', 'record_a_note/tone_of_voice', 'goods_sold', 'goods_sold/comment', @@ -292,7 +319,7 @@ def tests_additional_field_exports_advanced(): ] -def tests_additional_field_exports_v2(): +def test_additional_field_exports_v2(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) @@ -306,9 +333,10 @@ def tests_additional_field_exports_v2(): assert 3 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript', - 'record_a_note/translated_es', - 'record_a_note/translated_af', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', @@ -319,13 +347,14 @@ def tests_additional_field_exports_v2(): 'Hello how may I help you?', '', '', + '', '2021-11-01Z', 'Save On', 'Pretty cliche', ] -def tests_additional_field_exports_all_versions(): +def test_additional_field_exports_all_versions(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) @@ -339,9 +368,10 @@ def tests_additional_field_exports_all_versions(): assert 6 == len(main_export_sheet['data']) assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript', - 'record_a_note/translated_es', - 'record_a_note/translated_af', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', @@ -351,8 +381,9 @@ def tests_additional_field_exports_all_versions(): response0 = main_export_sheet['data'][0] assert response0 == [ 'clerk_interaction_1.mp3', - 'Hello how may I help you?', + '', 'Saluton, kiel mi povas helpi vin?', + 'Hello how may I help you?', '', '2021-11-01Z', '', @@ -366,6 +397,7 @@ def tests_additional_field_exports_all_versions(): 'Hello how may I help you?', '', '', + '', '2021-11-01Z', 'Save On', 'Pretty cliche', @@ -374,7 +406,7 @@ def tests_additional_field_exports_all_versions(): ] -def tests_additional_field_exports_all_versions_exclude_fields(): +def test_additional_field_exports_all_versions_exclude_fields(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) @@ -405,7 +437,7 @@ def tests_additional_field_exports_all_versions_exclude_fields(): ] -def tests_additional_field_exports_all_versions_langs(): +def test_additional_field_exports_all_versions_langs(): title, schemas, submissions = build_fixture('analysis_form') analysis_form = load_analysis_form_json('analysis_form') pack = FormPack(schemas, title=title) @@ -422,9 +454,10 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Record a clerk saying something', - 'record_a_note/transcript', - 'record_a_note/translated_es', - 'record_a_note/translated_af', + 'Record a clerk saying something - transcript (en)', + 'Record a clerk saying something - transcript (es)', + 'Record a clerk saying something - translation (en)', + 'Record a clerk saying something - translation (es)', 'Transcription Timestamp', "What is the shop's name?", 'Comment on the name of the shop', @@ -439,9 +472,10 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'Registri oficiston dirantan ion', - 'record_a_note/transcript', - 'record_a_note/translated_es', - 'record_a_note/translated_af', + 'Registri oficiston dirantan ion - transcript (en)', + 'Registri oficiston dirantan ion - transcript (es)', + 'Registri oficiston dirantan ion - translation (en)', + 'Registri oficiston dirantan ion - translation (es)', 'record_a_note/acme_timestamp', 'Kio estas la nomo de la butiko?', 'name_of_shop/comment', @@ -456,9 +490,10 @@ def tests_additional_field_exports_all_versions_langs(): assert main_export_sheet['fields'] == [ 'record_a_note', - 'record_a_note/transcript', - 'record_a_note/translated_es', - 'record_a_note/translated_af', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', 'record_a_note/acme_timestamp', 'name_of_shop', 'name_of_shop/comment', From 0bcbdb3e488411321b76356e1a2464582eeb744c Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 22 Mar 2022 20:30:16 +0000 Subject: [PATCH 41/54] pin pyxform version to 1.7.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cf08a406..c99efd48 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ 'lxml', 'path.py', 'pyquery', - 'pyxform', + 'pyxform==1.7.0', 'statistics', 'XlsxWriter', 'geojson-rewind', From a41861a9107e0b1805592555bdf3c06d6ed2b64d Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 22 Mar 2022 21:05:02 +0000 Subject: [PATCH 42/54] clean up, use constants --- src/formpack/reporting/export.py | 11 +++-------- src/formpack/schema/fields.py | 7 ++++++- src/formpack/version.py | 11 +++++++++-- tests/test_additional_field_exports.py | 2 +- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index be74af8d..a155a705 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -403,7 +403,7 @@ def _get_attachment(val, field, attachments): ] def _get_value_from_supplemental_details( - field, supplemental_details: Dict + field: FormField, supplemental_details: Dict ) -> Optional[str]: source, name = field.analysis_path _sup_details = supplemental_details.get(source, {}) @@ -411,19 +411,14 @@ def _get_value_from_supplemental_details( if not _sup_details: return - if 'translation' in name: + # TODO: Fix this on KPI side so that names are consistent + if ANALYSIS_TYPE_TRANSLATION in name: name = 'translated' val = _sup_details.get(name) if val is None: return '' - if field.analysis_type == ANALYSIS_TYPE_TRANSLATION: - try: - val = val[field.language]['value'] - except KeyError: - val = '' - return val def _get_value_from_entry( diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index c580c53e..47feed4b 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -509,7 +509,12 @@ def format( if val is None: val = '' - if self._is_transcript: + if self._is_translation and isinstance(val, dict): + try: + val = val[self.language]['value'] + except KeyError: + val = '' + elif self._is_transcript: cells = dict.fromkeys(self.get_value_names(), '') if isinstance(val, dict): cells[ diff --git a/src/formpack/version.py b/src/formpack/version.py index 40b361ce..42fd6507 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -8,7 +8,11 @@ from pyxform import aliases as pyxform_aliases -from .constants import UNTRANSLATED +from .constants import ( + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + UNTRANSLATED, +) from .errors import SchemaError from .errors import TranslationError from .schema import FormField, FormGroup, FormSection, FormChoice @@ -389,7 +393,10 @@ def __init__( for data_def in survey: data_type = data_def['type'] - if data_type in ['translation', 'transcript']: + if data_type in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: data_def.update( { 'type': 'text', diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 140813ad..f080e2e3 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -50,7 +50,7 @@ def test_additional_field_exports_with_labels(): 'include_analysis_fields': True, 'versions': 'v1', 'filter_fields': ['record_a_note'], - 'lang': 'English (en)' + 'lang': 'English (en)', } export = pack.export(**options) values = export.to_dict(submissions) From 4f772e7ef14341856f62dbca1dc854e67b555cde Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 13 Apr 2022 23:11:11 +0000 Subject: [PATCH 43/54] remove `include_analysis_fields` flag --- src/formpack/constants.py | 2 -- src/formpack/pack.py | 2 -- src/formpack/reporting/export.py | 9 +++------ tests/test_additional_field_exports.py | 14 ++++++-------- tests/test_kobo_locking.py | 2 +- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/formpack/constants.py b/src/formpack/constants.py index 14c7b779..c8112962 100644 --- a/src/formpack/constants.py +++ b/src/formpack/constants.py @@ -60,7 +60,6 @@ EXPORT_SETTING_FLATTEN = 'flatten' EXPORT_SETTING_GROUP_SEP = 'group_sep' EXPORT_SETTING_HIERARCHY_IN_LABELS = 'hierarchy_in_labels' -EXPORT_SETTING_INCLUDE_ANALYSIS_FIELDS = 'include_analysis_fields' EXPORT_SETTING_INCLUDE_MEDIA_URL = 'include_media_url' EXPORT_SETTING_LANG = 'lang' EXPORT_SETTING_MULTIPLE_SELECT = 'multiple_select' @@ -73,7 +72,6 @@ OPTIONAL_EXPORT_SETTINGS = [ EXPORT_SETTING_FIELDS, EXPORT_SETTING_FLATTEN, - EXPORT_SETTING_INCLUDE_ANALYSIS_FIELDS, EXPORT_SETTING_INCLUDE_MEDIA_URL, EXPORT_SETTING_NAME, EXPORT_SETTING_QUERY, diff --git a/src/formpack/pack.py b/src/formpack/pack.py index fd736454..7cf6e1f4 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -370,7 +370,6 @@ def export( filter_fields=(), xls_types_as_text=True, include_media_url=False, - include_analysis_fields=False, ): """ Create an export for given versions of the form. @@ -392,7 +391,6 @@ def export( filter_fields=filter_fields, xls_types_as_text=xls_types_as_text, include_media_url=include_media_url, - include_analysis_fields=include_analysis_fields, ) def autoreport(self, versions=-1): diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 8c49128e..e4a4bb4b 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -49,7 +49,6 @@ def __init__( filter_fields=(), xls_types_as_text=True, include_media_url=False, - include_analysis_fields=False, ): """ :param formpack: FormPack @@ -69,7 +68,6 @@ def __init__( :param filter_fields: list :param xls_types_as_text: bool :param include_media_url: bool - :param include_analysis_fields: bool """ self.formpack = formpack @@ -86,7 +84,6 @@ def __init__( self.filter_fields = filter_fields self.xls_types_as_text = xls_types_as_text self.include_media_url = include_media_url - self.include_analysis_fields = include_analysis_fields self.__r_groups_submission_mapping_values = {} if tag_cols_for_header is None: @@ -244,7 +241,7 @@ def get_fields_labels_tags_for_all_versions( ] # TODO: For MVP, just reattach additional fields to their source - if self.analysis_form and self.include_analysis_fields: + if self.analysis_form: all_fields = self.analysis_form.insert_analysis_fields(all_fields) # Collect all the sections regardless if they contain any fields @@ -378,7 +375,7 @@ def format_one_submission( row = self._row_cache[_section_name] _fields = tuple(current_section.fields.values()) - if self.analysis_form and self.include_analysis_fields: + if self.analysis_form: _fields = self.analysis_form.insert_analysis_fields(_fields) def _get_attachment(val, field, attachments): @@ -440,7 +437,7 @@ def _get_value_from_entry( ) # TODO: For MVP, just reattach additional fields to their source - if self.analysis_form and self.include_analysis_fields: + if self.analysis_form: _fields = self.analysis_form.insert_analysis_fields(_fields) # 'rows' will contain all the formatted entries for the current diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index f080e2e3..52c7175e 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -12,7 +12,6 @@ def test_additional_field_exports_without_labels(): pack.extend_survey(analysis_form) options = { - 'include_analysis_fields': True, 'versions': 'v1', 'filter_fields': ['record_a_note'], } @@ -47,7 +46,6 @@ def test_additional_field_exports_with_labels(): pack.extend_survey(analysis_form) options = { - 'include_analysis_fields': True, 'versions': 'v1', 'filter_fields': ['record_a_note'], 'lang': 'English (en)', @@ -84,7 +82,6 @@ def test_additional_field_exports_repeat_groups(): pack.extend_survey(analysis_form) options = { - 'include_analysis_fields': True, 'versions': 'v1', } export = pack.export(**options) @@ -154,7 +151,6 @@ def test_additional_field_exports_advanced(): pack.extend_survey(analysis_form) options = { - 'include_analysis_fields': True, 'versions': 'v1', 'multiple_select': 'both', } @@ -325,7 +321,7 @@ def test_additional_field_exports_v2(): pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) - options = {'include_analysis_fields': True, 'versions': 'v2'} + options = {'versions': 'v2'} export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] @@ -360,7 +356,7 @@ def test_additional_field_exports_all_versions(): pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) - options = {'include_analysis_fields': True, 'versions': pack.versions} + options = {'versions': pack.versions} export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] @@ -412,7 +408,10 @@ def test_additional_field_exports_all_versions_exclude_fields(): pack = FormPack(schemas, title=title) pack.extend_survey(analysis_form) - options = {'versions': pack.versions} + options = { + 'versions': pack.versions, + 'filter_fields': ['record_a_note', 'name_of_shop', 'name_of_clerk'], + } export = pack.export(**options) values = export.to_dict(submissions) main_export_sheet = values['Simple Clerk Interaction'] @@ -444,7 +443,6 @@ def test_additional_field_exports_all_versions_langs(): pack.extend_survey(analysis_form) options = { - 'include_analysis_fields': True, 'versions': pack.versions, 'lang': 'English (en)', } diff --git a/tests/test_kobo_locking.py b/tests/test_kobo_locking.py index 705c6d25..ef15500b 100644 --- a/tests/test_kobo_locking.py +++ b/tests/test_kobo_locking.py @@ -50,7 +50,7 @@ def _construct_xlsx_for_import(self, sheet_name, sheet_content): for row_num, row_list in enumerate(sheet_content): for col_num, cell_value in enumerate(row_list): if cell_value and cell_value is not None: - worksheet.cell(row_num+1, col_num+1).value = cell_value + worksheet.cell(row_num + 1, col_num + 1).value = cell_value xlsx_import_io = BytesIO() workbook_to_import.save(xlsx_import_io) xlsx_import_io.seek(0) From a48975c3a0ccb295c305ce315a388282476b3b6e Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 13 Apr 2022 23:12:39 +0000 Subject: [PATCH 44/54] remove pyxform version pin from setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c99efd48..cf08a406 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ 'lxml', 'path.py', 'pyquery', - 'pyxform==1.7.0', + 'pyxform', 'statistics', 'XlsxWriter', 'geojson-rewind', From 53afa089a2db10f29ee7209c829ada68bad73596 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 13 Apr 2022 23:21:49 +0000 Subject: [PATCH 45/54] make requested changes --- src/formpack/schema/fields.py | 84 +++++++-------- src/formpack/version.py | 186 +++++++++++++++++----------------- 2 files changed, 135 insertions(+), 135 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 47feed4b..5a614bbc 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -64,7 +64,7 @@ def __init__( if has_stats is not None: self.has_stats = has_stats else: - self.has_stats = data_type != "note" and not self.analysis_question + self.has_stats = data_type != 'note' and not self.analysis_question # do not include the root section in the path self.path = '/'.join(info.name for info in self.hierarchy[1:]) @@ -458,13 +458,25 @@ def get_substats( class TextField(ExtendedFormField): - @property - def _is_transcript(self): - return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSCRIPT + def get_disaggregated_stats( + self, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATION, limit=100 + ): - @property - def _is_translation(self): - return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + parent = super() + stats = parent.get_disaggregated_stats( + metrics, top_splitters, lang, limit + ) + substats = self.get_substats(stats, metrics, top_splitters, lang) + + # sort values by total frequency + def sum_frequencies(element): + return sum(v for k, v in element[1]['frequency']) + + values = sorted(substats.items(), key=sum_frequencies, reverse=True) + + stats.update({'values': values[:limit]}) + + return stats def get_labels( self, @@ -487,6 +499,21 @@ def get_labels( ] return [self._get_label(*args)] + def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): + + stats = super().get_stats(metrics, lang, limit) + + top = metrics.most_common(limit) + total = stats['total_count'] + + percentage = [] + for key, val in top: + percentage.append((key, self._get_percentage(val, total))) + + stats.update({'frequency': top, 'percentage': percentage}) + + return stats + def get_value_names(self, multiple_select='both', *args, **kwargs): if self._is_transcript: return [ @@ -495,6 +522,14 @@ def get_value_names(self, multiple_select='both', *args, **kwargs): ] return super().get_value_names() + @property + def _is_transcript(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSCRIPT + + @property + def _is_translation(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + def format( self, val, @@ -524,41 +559,6 @@ def format( return {self.name: val} - def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): - - stats = super().get_stats(metrics, lang, limit) - - top = metrics.most_common(limit) - total = stats['total_count'] - - percentage = [] - for key, val in top: - percentage.append((key, self._get_percentage(val, total))) - - stats.update({'frequency': top, 'percentage': percentage}) - - return stats - - def get_disaggregated_stats( - self, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATION, limit=100 - ): - - parent = super() - stats = parent.get_disaggregated_stats( - metrics, top_splitters, lang, limit - ) - substats = self.get_substats(stats, metrics, top_splitters, lang) - - # sort values by total frequency - def sum_frequencies(element): - return sum(v for k, v in element[1]['frequency']) - - values = sorted(substats.items(), key=sum_frequencies, reverse=True) - - stats.update({'values': values[:limit]}) - - return stats - class MediaField(TextField): def get_labels(self, include_media_url=False, *args, **kwargs): diff --git a/src/formpack/version.py b/src/formpack/version.py index 42fd6507..4873b689 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -52,19 +52,6 @@ def get(self, key, default=None): class BaseForm: - @staticmethod - def _get_translations(content: Dict[str, List]) -> List[str]: - return [ - t if t is not None else UNTRANSLATED - for t in content.get('translations', [None]) - ] - - @staticmethod - def _get_fields_by_name( - survey: Dict[str, Union[str, List]] - ) -> Dict[str, Dict[str, Union[str, List]]]: - return {row['name']: row for row in survey if 'name' in row} - @staticmethod def _get_field_labels( field: FormField, @@ -76,6 +63,99 @@ def _get_field_labels( return LabelStruct(labels=field['label'], translations=translations) return LabelStruct() + @staticmethod + def _get_fields_by_name( + survey: Dict[str, Union[str, List]] + ) -> Dict[str, Dict[str, Union[str, List]]]: + return {row['name']: row for row in survey if 'name' in row} + + @staticmethod + def _get_translations(content: Dict[str, List]) -> List[str]: + return [ + t if t is not None else UNTRANSLATED + for t in content.get('translations', [None]) + ] + + +class AnalysisForm(BaseForm): + def __init__( + self, + formpack: 'FormPack', + schema: Dict[str, Union[str, List]], + ) -> None: + + self.schema = schema + self.formpack = formpack + + survey = self.schema.get('additional_fields', []) + fields_by_name = self._get_fields_by_name(survey) + section = FormSection(name=formpack.title) + + self.translations = self._get_translations(schema) + + choices_definition = schema.get('additional_choices', ()) + field_choices = FormChoice.all_from_json_definition( + choices_definition, self.translations + ) + + for data_def in survey: + data_type = data_def['type'] + if data_type in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: + data_def.update( + { + 'type': 'text', + 'analysis_type': data_type, + } + ) + + field = FormField.from_json_definition( + definition=data_def, + field_choices=field_choices, + section=section, + translations=self.translations, + ) + + field.labels = self._get_field_labels( + field=fields_by_name[field.name], + translations=self.translations, + ) + section.fields[field.name] = field + + self.fields = list(section.fields.values()) + self.fields_by_source = self._get_fields_by_source() + + def __repr__(self) -> str: + return f"" + + def _get_fields_by_source(self) -> Dict[str, List[FormField]]: + fields_by_source = defaultdict(list) + for field in self.fields: + fields_by_source[field.source].append(field) + return fields_by_source + + def _map_sections_to_analysis_fields( + self, survey_field: FormField + ) -> List[FormField]: + _fields = [] + for analysis_field in self.fields_by_source[survey_field.name]: + analysis_field.section = survey_field.section + analysis_field.source_field = survey_field + _fields.append(analysis_field) + return _fields + + def insert_analysis_fields( + self, fields: List[FormField] + ) -> List[FormField]: + _fields = [] + for field in fields: + _fields.append(field) + if field.name in self.fields_by_source: + _fields += self._map_sections_to_analysis_fields(field) + return _fields + class FormVersion(BaseForm): @classmethod @@ -368,83 +448,3 @@ def to_xml(self, warnings=None): ) return survey._to_pretty_xml() # .encode('utf-8') - - -class AnalysisForm(BaseForm): - def __init__( - self, - formpack: 'FormPack', - schema: Dict[str, Union[str, List]], - ) -> None: - - self.schema = schema - self.formpack = formpack - - survey = self.schema.get('additional_fields', []) - fields_by_name = self._get_fields_by_name(survey) - section = FormSection(name=formpack.title) - - self.translations = self._get_translations(schema) - - choices_definition = schema.get('additional_choices', ()) - field_choices = FormChoice.all_from_json_definition( - choices_definition, self.translations - ) - - for data_def in survey: - data_type = data_def['type'] - if data_type in [ - ANALYSIS_TYPE_TRANSCRIPT, - ANALYSIS_TYPE_TRANSLATION, - ]: - data_def.update( - { - 'type': 'text', - 'analysis_type': data_type, - } - ) - - field = FormField.from_json_definition( - definition=data_def, - field_choices=field_choices, - section=section, - translations=self.translations, - ) - - field.labels = self._get_field_labels( - field=fields_by_name[field.name], - translations=self.translations, - ) - section.fields[field.name] = field - - self.fields = list(section.fields.values()) - self.fields_by_source = self._get_fields_by_source() - - def __repr__(self) -> str: - return f"" - - def _get_fields_by_source(self) -> Dict[str, List[FormField]]: - fields_by_source = defaultdict(list) - for field in self.fields: - fields_by_source[field.source].append(field) - return fields_by_source - - def _map_sections_to_analysis_fields( - self, survey_field: FormField - ) -> List[FormField]: - _fields = [] - for analysis_field in self.fields_by_source[survey_field.name]: - analysis_field.section = survey_field.section - analysis_field.source_field = survey_field - _fields.append(analysis_field) - return _fields - - def insert_analysis_fields( - self, fields: List[FormField] - ) -> List[FormField]: - _fields = [] - for field in fields: - _fields.append(field) - if field.name in self.fields_by_source: - _fields += self._map_sections_to_analysis_fields(field) - return _fields From 289b52398dc55a5d74e2505d7989a0edd18c0157 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 13 Apr 2022 23:37:23 +0000 Subject: [PATCH 46/54] use `filter_fields` for controlling export of additional analysis fields --- src/formpack/reporting/export.py | 14 ++++++-------- tests/test_additional_field_exports.py | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index e4a4bb4b..e72670d4 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -233,6 +233,9 @@ def get_fields_labels_tags_for_all_versions( # Ensure that fields are filtered if they've been specified, otherwise # carry on as usual + if self.analysis_form: + all_fields = self.analysis_form.insert_analysis_fields(all_fields) + if self.filter_fields: all_fields = [ field @@ -240,10 +243,6 @@ def get_fields_labels_tags_for_all_versions( if field.path in self.filter_fields ] - # TODO: For MVP, just reattach additional fields to their source - if self.analysis_form: - all_fields = self.analysis_form.insert_analysis_fields(all_fields) - # Collect all the sections regardless if they contain any fields all_sections = {} for version in self.versions.values(): @@ -429,6 +428,9 @@ def _get_value_from_entry( suffix = 'meta/' if field.data_type == 'audit' else '' return entry.get(f'{suffix}{field.path}') + if self.analysis_form: + _fields = self.analysis_form.insert_analysis_fields(_fields) + # Ensure that fields are filtered if they've been specified, otherwise # carry on as usual if self.filter_fields: @@ -436,10 +438,6 @@ def _get_value_from_entry( field for field in _fields if field.path in self.filter_fields ) - # TODO: For MVP, just reattach additional fields to their source - if self.analysis_form: - _fields = self.analysis_form.insert_analysis_fields(_fields) - # 'rows' will contain all the formatted entries for the current # section. If you don't have repeat-group, there is only one section # with a row of size one. diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 52c7175e..19c83a2b 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -13,7 +13,13 @@ def test_additional_field_exports_without_labels(): options = { 'versions': 'v1', - 'filter_fields': ['record_a_note'], + 'filter_fields': [ + 'record_a_note', + 'record_a_note/transcript', + 'record_a_note/translation_en', + 'record_a_note/translation_es', + 'record_a_note/acme_timestamp', + ], } export = pack.export(**options) values = export.to_dict(submissions) @@ -47,7 +53,13 @@ def test_additional_field_exports_with_labels(): options = { 'versions': 'v1', - 'filter_fields': ['record_a_note'], + 'filter_fields': [ + 'record_a_note', + 'record_a_note/transcript', + 'record_a_note/translation_en', + 'record_a_note/translation_es', + 'record_a_note/acme_timestamp', + ], 'lang': 'English (en)', } export = pack.export(**options) @@ -410,7 +422,11 @@ def test_additional_field_exports_all_versions_exclude_fields(): options = { 'versions': pack.versions, - 'filter_fields': ['record_a_note', 'name_of_shop', 'name_of_clerk'], + 'filter_fields': [ + 'record_a_note', + 'clerk_details/name_of_shop', + 'clerk_details/name_of_clerk', + ], } export = pack.export(**options) values = export.to_dict(submissions) From 4ea0c16ae4c04c69c8b7da8d8b51ef9d74c7e7d0 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Thu, 12 May 2022 14:59:20 +0000 Subject: [PATCH 47/54] mvp filtering for multi-language transcripts --- src/formpack/reporting/export.py | 14 ++++++++ src/formpack/schema/fields.py | 46 ++++++++++++++++++-------- tests/test_additional_field_exports.py | 22 ++++++------ 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index e72670d4..31594ccc 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -90,6 +90,17 @@ def __init__( tag_cols_for_header = [] self.tag_cols_for_header = tag_cols_for_header + self.t_lang_codes = [] + _filter_fields = [] + for item in self.filter_fields: + item = re.sub(r'^_supplementalDetails/', '', item) + if item.split('/')[-1].startswith('transcript'): + self.t_lang_codes.append(item.split('_')[-1]) + _filter_fields.append(f"{item.split('/')[0]}/transcript") + else: + _filter_fields.append(item) + self.filter_fields = _filter_fields + # If some fields need to be arbitrarily copied, add them # to the first section if copy_fields: @@ -268,6 +279,7 @@ def get_fields_labels_tags_for_all_versions( hierarchy_in_labels=hierarchy_in_labels, multiple_select=self.multiple_select, include_media_url=self.include_media_url, + t_lang_codes=self.t_lang_codes, ) ) @@ -297,6 +309,7 @@ def get_fields_labels_tags_for_all_versions( value_names = field.get_value_names( multiple_select=self.multiple_select, include_media_url=self.include_media_url, + t_lang_codes=self.t_lang_codes, ) name_lists.append(value_names) @@ -487,6 +500,7 @@ def _get_value_from_entry( xls_types_as_text=self.xls_types_as_text, attachment=attachment, include_media_url=self.include_media_url, + t_lang_codes=self.t_lang_codes, ) # save fields value if they match parent mapping fields. diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 5a614bbc..65b830ac 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -484,6 +484,7 @@ def get_labels( group_sep='/', hierarchy_in_labels=False, multiple_select='both', + t_lang_codes=[], *args, **kwargs, ): @@ -493,10 +494,17 @@ def get_labels( if self._is_translation: return [f'{source_label} - translation ({self.language})'] elif self._is_transcript: - return [ - f'{source_label} - transcript ({code})' - for code in self.languages - ] + if t_lang_codes: + return [ + f'{source_label} - transcript ({code})' + for code in self.languages + if code in t_lang_codes + ] + else: + return [ + f'{source_label} - transcript ({code})' + for code in self.languages + ] return [self._get_label(*args)] def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): @@ -514,12 +522,21 @@ def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): return stats - def get_value_names(self, multiple_select='both', *args, **kwargs): + def get_value_names( + self, multiple_select='both', t_lang_codes=[], *args, **kwargs + ): if self._is_transcript: - return [ - f'{self.source_field.name} - transcript ({code})' - for code in self.languages - ] + if t_lang_codes: + return [ + f'{self.source_field.name} - transcript ({code})' + for code in self.languages + if code in t_lang_codes + ] + else: + return [ + f'{self.source_field.name} - transcript ({code})' + for code in self.languages + ] return super().get_value_names() @property @@ -538,6 +555,7 @@ def format( hierarchy_in_labels=False, multiple_select='both', xls_types_as_text=True, + t_lang_codes=[], *args, **kwargs, ): @@ -550,11 +568,13 @@ def format( except KeyError: val = '' elif self._is_transcript: - cells = dict.fromkeys(self.get_value_names(), '') + cells = dict.fromkeys( + self.get_value_names(t_lang_codes=t_lang_codes), '' + ) if isinstance(val, dict): - cells[ - f'{self.source_field.name} - transcript ({val["languageCode"]})' - ] = val['value'] + name = f'{self.source_field.name} - transcript ({val["languageCode"]})' + if name in cells: + cells[name] = val['value'] return cells return {self.name: val} diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 19c83a2b..b9c190ec 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -15,10 +15,11 @@ def test_additional_field_exports_without_labels(): 'versions': 'v1', 'filter_fields': [ 'record_a_note', - 'record_a_note/transcript', - 'record_a_note/translation_en', - 'record_a_note/translation_es', - 'record_a_note/acme_timestamp', + '_supplementalDetails/record_a_note/transcript_en', + '_supplementalDetails/record_a_note/transcript_es', + '_supplementalDetails/record_a_note/translation_en', + '_supplementalDetails/record_a_note/translation_es', + '_supplementalDetails/record_a_note/acme_timestamp', ], } export = pack.export(**options) @@ -55,10 +56,11 @@ def test_additional_field_exports_with_labels(): 'versions': 'v1', 'filter_fields': [ 'record_a_note', - 'record_a_note/transcript', - 'record_a_note/translation_en', - 'record_a_note/translation_es', - 'record_a_note/acme_timestamp', + '_supplementalDetails/record_a_note/transcript_en', + '_supplementalDetails/record_a_note/transcript_es', + '_supplementalDetails/record_a_note/translation_en', + '_supplementalDetails/record_a_note/translation_es', + '_supplementalDetails/record_a_note/acme_timestamp', ], 'lang': 'English (en)', } @@ -424,8 +426,8 @@ def test_additional_field_exports_all_versions_exclude_fields(): 'versions': pack.versions, 'filter_fields': [ 'record_a_note', - 'clerk_details/name_of_shop', - 'clerk_details/name_of_clerk', + '_supplementalDetails/clerk_details/name_of_shop', + '_supplementalDetails/clerk_details/name_of_clerk', ], } export = pack.export(**options) From 0569abe231f80718c16e22bee8489ee4ed88f092 Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 17 May 2022 14:01:59 +0000 Subject: [PATCH 48/54] change translations field name from `translation_` to `translated_` --- src/formpack/reporting/export.py | 6 ++++-- tests/fixtures/analysis_form/analysis_form.json | 12 ++++++------ tests/test_additional_field_exports.py | 8 ++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 31594ccc..92825946 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -420,8 +420,10 @@ def _get_value_from_supplemental_details( if not _sup_details: return - # TODO: Fix this on KPI side so that names are consistent - if ANALYSIS_TYPE_TRANSLATION in name: + # The names for translation fields are `translated_` + # which must be stripped to get the value from the supplemental + # details dict + if re.match(r'^translated_', name): name = 'translated' val = _sup_details.get(name) diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 56c46a8a..431d1605 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -22,32 +22,32 @@ }, { "type": "translation", - "name": "record_a_note/translation_en", + "name": "record_a_note/translated_en", "label": "record_a_note Translated (en)", "language": "en", "path": [ "record_a_note", - "translation_en" + "translated_en" ], "source": "record_a_note", "settings": { "mode": "manual", - "engine": "engines/translation_manual" + "engine": "engines/translated_manual" } }, { "type": "translation", - "name": "record_a_note/translation_es", + "name": "record_a_note/translated_es", "label": "record_a_note Translated (es)", "language": "es", "path": [ "record_a_note", - "translation_es" + "translated_es" ], "source": "record_a_note", "settings": { "mode": "manual", - "engine": "engines/translation_manual" + "engine": "engines/translated_manual" } }, { diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index b9c190ec..f0a7cd89 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -17,8 +17,8 @@ def test_additional_field_exports_without_labels(): 'record_a_note', '_supplementalDetails/record_a_note/transcript_en', '_supplementalDetails/record_a_note/transcript_es', - '_supplementalDetails/record_a_note/translation_en', - '_supplementalDetails/record_a_note/translation_es', + '_supplementalDetails/record_a_note/translated_en', + '_supplementalDetails/record_a_note/translated_es', '_supplementalDetails/record_a_note/acme_timestamp', ], } @@ -58,8 +58,8 @@ def test_additional_field_exports_with_labels(): 'record_a_note', '_supplementalDetails/record_a_note/transcript_en', '_supplementalDetails/record_a_note/transcript_es', - '_supplementalDetails/record_a_note/translation_en', - '_supplementalDetails/record_a_note/translation_es', + '_supplementalDetails/record_a_note/translated_en', + '_supplementalDetails/record_a_note/translated_es', '_supplementalDetails/record_a_note/acme_timestamp', ], 'lang': 'English (en)', From 4da4ec94b0b0c1b8b5dc7d9d9b95929e1a8abd7f Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Wed, 1 Jun 2022 10:25:46 +0000 Subject: [PATCH 49/54] remove duplicate inserting of analysis fields --- src/formpack/reporting/export.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 92825946..b3c6ae68 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -387,9 +387,6 @@ def format_one_submission( row = self._row_cache[_section_name] _fields = tuple(current_section.fields.values()) - if self.analysis_form: - _fields = self.analysis_form.insert_analysis_fields(_fields) - def _get_attachment(val, field, attachments): """ Filter attachments for filenames that match the submission field's From 17d53b9dd1c29f435bdb3013fa97bbc0aea8d753 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Mon, 3 Oct 2022 12:28:51 -0400 Subject: [PATCH 50/54] split out languages into individual fields in analysis_form_json for transcripts and translations fixtures --- tests/fixtures/analysis_form/analysis_form.json | 17 ++++++++++++++++- .../analysis_form_advanced/analysis_form.json | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 431d1605..e07661f4 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -13,7 +13,22 @@ "record_a_note", "transcript" ], - "languages": ["en", "es"], + "language": "en", + "source": "record_a_note", + "settings": { + "mode": "manual", + "engine": "engines/transcript_manual" + } + }, + { + "type": "transcript", + "name": "record_a_note/transcript", + "label": "record_a_note Transcript", + "path": [ + "record_a_note", + "transcript" + ], + "language": "es", "source": "record_a_note", "settings": { "mode": "manual", diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 00904534..2fa46409 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -10,7 +10,7 @@ "name": "record_a_note/transcript", "path": ["record_a_note", "transcript"], "source": "record_a_note", - "languages": ["en"], + "language": "en", "settings": { "mode": "auto", "engine": "engines/acme_1_speech2text" From ba1fd1440eb9471ab4746d91bda8a7d6d36535a1 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Mon, 3 Oct 2022 12:54:38 -0400 Subject: [PATCH 51/54] unique name and path for transcript languages in json fixture --- tests/fixtures/analysis_form/analysis_form.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index e07661f4..55d382d4 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -7,11 +7,11 @@ "additional_fields": [ { "type": "transcript", - "name": "record_a_note/transcript", - "label": "record_a_note Transcript", + "name": "record_a_note/transcript_en", + "label": "record_a_note Transcript (en)", "path": [ "record_a_note", - "transcript" + "transcript_en" ], "language": "en", "source": "record_a_note", @@ -22,11 +22,11 @@ }, { "type": "transcript", - "name": "record_a_note/transcript", - "label": "record_a_note Transcript", + "name": "record_a_note/transcript_es", + "label": "record_a_note Transcript (es)", "path": [ "record_a_note", - "transcript" + "transcript_es" ], "language": "es", "source": "record_a_note", From b5d18d686ac8207ec8ed0c18ca4b324c5cedbc7c Mon Sep 17 00:00:00 2001 From: Joshua Beretta Date: Tue, 4 Oct 2022 13:17:56 +0000 Subject: [PATCH 52/54] update handling of transcript fields with new format and update fixtures --- src/formpack/reporting/export.py | 20 ++--- src/formpack/schema/fields.py | 73 ++++++------------- .../fixtures/analysis_form/analysis_form.json | 12 +-- tests/fixtures/analysis_form/v1.json | 2 +- .../analysis_form_advanced/analysis_form.json | 4 +- tests/test_additional_field_exports.py | 8 +- 6 files changed, 40 insertions(+), 79 deletions(-) diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index b3c6ae68..069d55be 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -90,15 +90,10 @@ def __init__( tag_cols_for_header = [] self.tag_cols_for_header = tag_cols_for_header - self.t_lang_codes = [] _filter_fields = [] for item in self.filter_fields: item = re.sub(r'^_supplementalDetails/', '', item) - if item.split('/')[-1].startswith('transcript'): - self.t_lang_codes.append(item.split('_')[-1]) - _filter_fields.append(f"{item.split('/')[0]}/transcript") - else: - _filter_fields.append(item) + _filter_fields.append(item) self.filter_fields = _filter_fields # If some fields need to be arbitrarily copied, add them @@ -279,7 +274,6 @@ def get_fields_labels_tags_for_all_versions( hierarchy_in_labels=hierarchy_in_labels, multiple_select=self.multiple_select, include_media_url=self.include_media_url, - t_lang_codes=self.t_lang_codes, ) ) @@ -309,7 +303,6 @@ def get_fields_labels_tags_for_all_versions( value_names = field.get_value_names( multiple_select=self.multiple_select, include_media_url=self.include_media_url, - t_lang_codes=self.t_lang_codes, ) name_lists.append(value_names) @@ -417,11 +410,11 @@ def _get_value_from_supplemental_details( if not _sup_details: return - # The names for translation fields are `translated_` - # which must be stripped to get the value from the supplemental - # details dict - if re.match(r'^translated_', name): - name = 'translated' + # The names for translation and transcript fields are in the format + # of `translated_` which must be stripped to get the + # value from the supplemental details dict + if _name := re.match(r'^(translation|transcript)_', name): + name = _name.groups()[0] val = _sup_details.get(name) if val is None: @@ -499,7 +492,6 @@ def _get_value_from_entry( xls_types_as_text=self.xls_types_as_text, attachment=attachment, include_media_url=self.include_media_url, - t_lang_codes=self.t_lang_codes, ) # save fields value if they match parent mapping fields. diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 65b830ac..672eb83b 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -50,10 +50,11 @@ def __init__( self.analysis_type = kwargs.get('analysis_type') self.analysis_path = kwargs.get('analysis_path') self.settings = kwargs.get('settings') - if self.analysis_type == ANALYSIS_TYPE_TRANSCRIPT: - self.languages = kwargs.get('languages') - if self.analysis_type == ANALYSIS_TYPE_TRANSLATION: - self.language = kwargs.get('language') + if self.analysis_type in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: + self.language = kwargs['language'] hierarchy = list(hierarchy) if hierarchy is not None else [None] self.hierarchy = hierarchy + [self] @@ -484,27 +485,17 @@ def get_labels( group_sep='/', hierarchy_in_labels=False, multiple_select='both', - t_lang_codes=[], *args, **kwargs, ): args = lang, group_sep, hierarchy_in_labels, multiple_select - if hasattr(self, 'source'): + if getattr(self, 'analysis_type', None) in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: source_label = self.source_field._get_label(*args) - if self._is_translation: - return [f'{source_label} - translation ({self.language})'] - elif self._is_transcript: - if t_lang_codes: - return [ - f'{source_label} - transcript ({code})' - for code in self.languages - if code in t_lang_codes - ] - else: - return [ - f'{source_label} - transcript ({code})' - for code in self.languages - ] + _type = 'translation' if self._is_translation else 'transcript' + return [f'{source_label} - {_type} ({self.language})'] return [self._get_label(*args)] def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): @@ -522,23 +513,6 @@ def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): return stats - def get_value_names( - self, multiple_select='both', t_lang_codes=[], *args, **kwargs - ): - if self._is_transcript: - if t_lang_codes: - return [ - f'{self.source_field.name} - transcript ({code})' - for code in self.languages - if code in t_lang_codes - ] - else: - return [ - f'{self.source_field.name} - transcript ({code})' - for code in self.languages - ] - return super().get_value_names() - @property def _is_transcript(self): return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSCRIPT @@ -555,27 +529,22 @@ def format( hierarchy_in_labels=False, multiple_select='both', xls_types_as_text=True, - t_lang_codes=[], *args, **kwargs, ): if val is None: val = '' - if self._is_translation and isinstance(val, dict): - try: - val = val[self.language]['value'] - except KeyError: - val = '' - elif self._is_transcript: - cells = dict.fromkeys( - self.get_value_names(t_lang_codes=t_lang_codes), '' - ) - if isinstance(val, dict): - name = f'{self.source_field.name} - transcript ({val["languageCode"]})' - if name in cells: - cells[name] = val['value'] - return cells + if isinstance(val, dict): + if self._is_translation: + try: + val = val[self.language]['value'] + except KeyError: + val = '' + elif self._is_transcript: + val = ( + val['value'] if val['languageCode'] == self.language else '' + ) return {self.name: val} diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 55d382d4..40d1eef0 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -37,32 +37,32 @@ }, { "type": "translation", - "name": "record_a_note/translated_en", + "name": "record_a_note/translation_en", "label": "record_a_note Translated (en)", "language": "en", "path": [ "record_a_note", - "translated_en" + "translation_en" ], "source": "record_a_note", "settings": { "mode": "manual", - "engine": "engines/translated_manual" + "engine": "engines/translation_manual" } }, { "type": "translation", - "name": "record_a_note/translated_es", + "name": "record_a_note/translation_es", "label": "record_a_note Translated (es)", "language": "es", "path": [ "record_a_note", - "translated_es" + "translation_es" ], "source": "record_a_note", "settings": { "mode": "manual", - "engine": "engines/translated_manual" + "engine": "engines/translation_manual" } }, { diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index 7aac0100..e81985cf 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -57,7 +57,7 @@ "value": "Saluton, kiel mi povas helpi vin?", "languageCode": "es" }, - "translated": { + "translation": { "en": { "value": "Hello how may I help you?" } diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 2fa46409..26d3210d 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -7,8 +7,8 @@ "additional_fields": [ { "type": "transcript", - "name": "record_a_note/transcript", - "path": ["record_a_note", "transcript"], + "name": "record_a_note/transcript_en", + "path": ["record_a_note", "transcript_en"], "source": "record_a_note", "language": "en", "settings": { diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index f0a7cd89..b9c190ec 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -17,8 +17,8 @@ def test_additional_field_exports_without_labels(): 'record_a_note', '_supplementalDetails/record_a_note/transcript_en', '_supplementalDetails/record_a_note/transcript_es', - '_supplementalDetails/record_a_note/translated_en', - '_supplementalDetails/record_a_note/translated_es', + '_supplementalDetails/record_a_note/translation_en', + '_supplementalDetails/record_a_note/translation_es', '_supplementalDetails/record_a_note/acme_timestamp', ], } @@ -58,8 +58,8 @@ def test_additional_field_exports_with_labels(): 'record_a_note', '_supplementalDetails/record_a_note/transcript_en', '_supplementalDetails/record_a_note/transcript_es', - '_supplementalDetails/record_a_note/translated_en', - '_supplementalDetails/record_a_note/translated_es', + '_supplementalDetails/record_a_note/translation_en', + '_supplementalDetails/record_a_note/translation_es', '_supplementalDetails/record_a_note/acme_timestamp', ], 'lang': 'English (en)', From b71c7c69614dc564a951e6f10b6c14bca4b8206e Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Mon, 10 Oct 2022 15:14:58 -0400 Subject: [PATCH 53/54] key in supplementalData is qpath --- src/formpack/schema/fields.py | 4 ++++ src/formpack/version.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 672eb83b..1ac05c57 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -70,6 +70,10 @@ def __init__( # do not include the root section in the path self.path = '/'.join(info.name for info in self.hierarchy[1:]) + @property + def qpath(self): + return self.path.replace('/', '-') + def get_labels( self, lang=UNSPECIFIED_TRANSLATION, diff --git a/src/formpack/version.py b/src/formpack/version.py index 4873b689..c08b0c13 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -140,7 +140,7 @@ def _map_sections_to_analysis_fields( self, survey_field: FormField ) -> List[FormField]: _fields = [] - for analysis_field in self.fields_by_source[survey_field.name]: + for analysis_field in self.fields_by_source[survey_field.qpath]: analysis_field.section = survey_field.section analysis_field.source_field = survey_field _fields.append(analysis_field) @@ -152,7 +152,7 @@ def insert_analysis_fields( _fields = [] for field in fields: _fields.append(field) - if field.name in self.fields_by_source: + if field.qpath in self.fields_by_source: _fields += self._map_sections_to_analysis_fields(field) return _fields From 15b2d44eb405f85366bb9942d37f01c4b71534b5 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Thu, 10 Nov 2022 20:56:21 -0500 Subject: [PATCH 54/54] Update tests to match qpath in supplementalData See b71c7c69614dc564a951e6f10b6c14bca4b8206e --- .../fixtures/analysis_form/analysis_form.json | 8 +++--- tests/fixtures/analysis_form/v1.json | 2 +- tests/fixtures/analysis_form/v2.json | 2 +- .../analysis_form_advanced/analysis_form.json | 28 +++++++++++++------ tests/fixtures/analysis_form_advanced/v1.json | 10 +++---- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json index 40d1eef0..9c5819d3 100644 --- a/tests/fixtures/analysis_form/analysis_form.json +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -81,25 +81,25 @@ "type": "text", "name": "name_of_clerk/comment", "path": [ - "name_of_clerk", + "clerk_details-name_of_clerk", "comment" ], "label": [ "Comment on the name of the clerk" ], - "source": "name_of_clerk" + "source": "clerk_details-name_of_clerk" }, { "type": "text", "name": "name_of_shop/comment", "path": [ - "name_of_shop", + "clerk_details-name_of_shop", "comment" ], "label": [ "Comment on the name of the shop" ], - "source": "name_of_shop" + "source": "clerk_details-name_of_shop" } ], "translations": [ diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json index e81985cf..f411725c 100644 --- a/tests/fixtures/analysis_form/v1.json +++ b/tests/fixtures/analysis_form/v1.json @@ -64,7 +64,7 @@ }, "acme_timestamp": "2021-11-01Z" }, - "name_of_clerk": { + "clerk_details-name_of_clerk": { "comment": "Sounds like an interesting person" } } diff --git a/tests/fixtures/analysis_form/v2.json b/tests/fixtures/analysis_form/v2.json index f4bb2961..bdb0cfc1 100644 --- a/tests/fixtures/analysis_form/v2.json +++ b/tests/fixtures/analysis_form/v2.json @@ -59,7 +59,7 @@ }, "acme_timestamp": "2021-11-01Z" }, - "name_of_shop": { + "clerk_details-name_of_shop": { "comment": "Pretty cliche" } } diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 26d3210d..5da3a4b0 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -8,8 +8,11 @@ { "type": "transcript", "name": "record_a_note/transcript_en", - "path": ["record_a_note", "transcript_en"], - "source": "record_a_note", + "path": [ + "clerk_interactions-record_a_note", + "transcript_en" + ], + "source": "clerk_interactions-record_a_note", "language": "en", "settings": { "mode": "auto", @@ -20,33 +23,42 @@ "type": "select_multiple", "select_from_list_name": "record_a_note_tones", "name": "record_a_note/tone_of_voice", - "path": ["record_a_note", "tone_of_voice"], + "path": [ + "clerk_interactions-record_a_note", + "tone_of_voice" + ], "label": [ "How was the tone of the clerk's voice?", "Kiel estis la tono de la voĉo de la oficisto?" ], - "source": "record_a_note" + "source": "clerk_interactions-record_a_note" }, { "type": "text", "name": "goods_sold/comment", - "path": ["goods_sold", "comment"], + "path": [ + "clerk_interactions-goods_sold", + "comment" + ], "label": [ "Comment on the goods sold at the store", "Komentu la varojn venditajn en la vendejo" ], - "source": "goods_sold" + "source": "clerk_interactions-goods_sold" }, { "type": "select_one", "select_from_list_name": "goods_sold_ratings", "name": "goods_sold/rating", - "path": ["goods_sold", "rating"], + "path": [ + "clerk_interactions-goods_sold", + "rating" + ], "label": [ "Rate the quality of the goods sold at the store", "Komentu la varojn vendojn en la vendejo" ], - "source": "goods_sold" + "source": "clerk_interactions-goods_sold" } ], "additional_choices": [ diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index 16ec1297..69a56222 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -78,14 +78,14 @@ } ], "_supplementalDetails": { - "record_a_note": { + "clerk_interactions-record_a_note": { "transcript": { "value": "Hello how may I help you?", "languageCode": "en" }, "tone_of_voice": "excited confused" }, - "goods_sold": { + "clerk_interactions-goods_sold": { "comment": "Not much diversity", "rating": "3" } @@ -101,14 +101,14 @@ } ], "_supplementalDetails": { - "record_a_note": { + "clerk_interactions-record_a_note": { "transcript": { "value": "Thank you for your business", "languageCode": "en" }, "tone_of_voice": "anxious excited" }, - "goods_sold": { + "clerk_interactions-goods_sold": { "rating": "2" } } @@ -123,7 +123,7 @@ } ], "_supplementalDetails": { - "goods_sold": { + "clerk_interactions-goods_sold": { "rating": "3" } }