From 9a47613bce6eb47235352f352e9486dd44fb56e9 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Mon, 28 Oct 2024 14:08:04 +0000 Subject: [PATCH 001/115] Adding columns and conditional formatting --- config/bulk_submission_excel/columns.yml | 20 +++++++++++++++++++ .../conditional_formattings.yml | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/config/bulk_submission_excel/columns.yml b/config/bulk_submission_excel/columns.yml index dc9a63ab64..5538d6ece9 100644 --- a/config/bulk_submission_excel/columns.yml +++ b/config/bulk_submission_excel/columns.yml @@ -269,6 +269,26 @@ requested_flowcell_type: conditional_formattings: empty_cell: is_error: +number_of_pools: + heading: scRNA Core Number Pools + attribute: :number_of_pools + unlocked: true + type: :integer + validation: + options: + type: :whole + operator: :between + formula1: "2" + formula2: "8" + allowBlank: true + showInputMessage: true + promptTitle: "Number of pools" + prompt: "The requested number pools (between 2 and 8 inclusive)" + conditional_formattings: + empty_cell: + is_text: + is_num_of_pools_in_valid_range: + is_num_of_pools_outside_valid_range: number_of_samples_per_pool: heading: scRNA Core Number of Samples per Pool attribute: :number_of_samples_per_pool diff --git a/config/bulk_submission_excel/conditional_formattings.yml b/config/bulk_submission_excel/conditional_formattings.yml index 4ce4122128..2c539d457b 100644 --- a/config/bulk_submission_excel/conditional_formattings.yml +++ b/config/bulk_submission_excel/conditional_formattings.yml @@ -53,6 +53,26 @@ number_greater_than_one: operator: :lessThan formula: "1" priority: 2 +is_num_of_pools_in_valid_range: + style: + bg_color: "00FF00" + type: :dxf + options: + type: :cellIs + operator: :between + formula1: "5" + formula2: "25" + priority: 2 +is_num_of_pools_outside_valid_range: + style: + bg_color: "FF0000" + type: :dxf + options: + type: :cellIs + operator: :notBetween + formula1: "5" + formula2: "25" + priority: 2 is_num_samples_per_pool_in_valid_range: style: bg_color: "00FF00" From ae562ca7e9b298893e34b50ea51775092a1a4d4c Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 29 Oct 2024 11:31:50 +0000 Subject: [PATCH 002/115] Removing samples_per_pool column --- config/bulk_submission_excel/columns.yml | 40 +++++++++---------- .../conditional_formattings.yml | 40 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/config/bulk_submission_excel/columns.yml b/config/bulk_submission_excel/columns.yml index 5538d6ece9..2aafa653ee 100644 --- a/config/bulk_submission_excel/columns.yml +++ b/config/bulk_submission_excel/columns.yml @@ -289,26 +289,26 @@ number_of_pools: is_text: is_num_of_pools_in_valid_range: is_num_of_pools_outside_valid_range: -number_of_samples_per_pool: - heading: scRNA Core Number of Samples per Pool - attribute: :number_of_samples_per_pool - unlocked: true - type: :integer - validation: - options: - type: :whole - operator: :between - formula1: "5" - formula2: "25" - allowBlank: true - showInputMessage: true - promptTitle: "Samples per Pool" - prompt: "The requested number of samples per pool (between 5 and 25 inclusive)" - conditional_formattings: - empty_cell: - is_text: - is_num_samples_per_pool_in_valid_range: - is_num_samples_per_pool_outside_valid_range: +#number_of_samples_per_pool: +# heading: scRNA Core Number of Samples per Pool +# attribute: :number_of_samples_per_pool +# unlocked: true +# type: :integer +# validation: +# options: +# type: :whole +# operator: :between +# formula1: "5" +# formula2: "25" +# allowBlank: true +# showInputMessage: true +# promptTitle: "Samples per Pool" +# prompt: "The requested number of samples per pool (between 5 and 25 inclusive)" +# conditional_formattings: +# empty_cell: +# is_text: +# is_num_samples_per_pool_in_valid_range: +# is_num_samples_per_pool_outside_valid_range: cells_per_chip_well: heading: scRNA Core Cells per Chip Well attribute: :cells_per_chip_well diff --git a/config/bulk_submission_excel/conditional_formattings.yml b/config/bulk_submission_excel/conditional_formattings.yml index 2c539d457b..d729c2e295 100644 --- a/config/bulk_submission_excel/conditional_formattings.yml +++ b/config/bulk_submission_excel/conditional_formattings.yml @@ -73,26 +73,26 @@ is_num_of_pools_outside_valid_range: formula1: "5" formula2: "25" priority: 2 -is_num_samples_per_pool_in_valid_range: - style: - bg_color: "00FF00" - type: :dxf - options: - type: :cellIs - operator: :between - formula1: "5" - formula2: "25" - priority: 2 -is_num_samples_per_pool_outside_valid_range: - style: - bg_color: "FF0000" - type: :dxf - options: - type: :cellIs - operator: :notBetween - formula1: "5" - formula2: "25" - priority: 2 +#is_num_samples_per_pool_in_valid_range: +# style: +# bg_color: "00FF00" +# type: :dxf +# options: +# type: :cellIs +# operator: :between +# formula1: "5" +# formula2: "25" +# priority: 2 +#is_num_samples_per_pool_outside_valid_range: +# style: +# bg_color: "FF0000" +# type: :dxf +# options: +# type: :cellIs +# operator: :notBetween +# formula1: "5" +# formula2: "25" +# priority: 2 is_num_cells_per_chip_well_in_valid_range: style: bg_color: "00FF00" From f4ea40aa395113d99784e4d3afabef5945b26315 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 30 Oct 2024 13:34:17 +0000 Subject: [PATCH 003/115] Adding study-project uniqueness validation --- .../validations_by_template_name.rb | 2 ++ config/bulk_submission_excel/columns.yml | 20 ------------------- .../conditional_formattings.yml | 20 ------------------- 3 files changed, 2 insertions(+), 40 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 36ee7710a4..05ee3d482e 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -9,6 +9,7 @@ module Submission::ValidationsByTemplateName HEADER_PROJECT_NAME = 'project name' HEADER_NUM_SAMPLES = 'scrna core number of samples per pool' HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' + HEADER_NUM_POOLS = "scrna core number of pools" # Applies additional validations based on the submission template type. # @@ -34,6 +35,7 @@ def apply_additional_validations_by_template_name when SCRNA_CORE_CDNA_PREP_GEM_X_5P validate_consistent_column_value(HEADER_NUM_SAMPLES) validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) + validate_consistent_column_value(HEADER_NUM_POOLS) end end diff --git a/config/bulk_submission_excel/columns.yml b/config/bulk_submission_excel/columns.yml index 2aafa653ee..35186db6b7 100644 --- a/config/bulk_submission_excel/columns.yml +++ b/config/bulk_submission_excel/columns.yml @@ -289,26 +289,6 @@ number_of_pools: is_text: is_num_of_pools_in_valid_range: is_num_of_pools_outside_valid_range: -#number_of_samples_per_pool: -# heading: scRNA Core Number of Samples per Pool -# attribute: :number_of_samples_per_pool -# unlocked: true -# type: :integer -# validation: -# options: -# type: :whole -# operator: :between -# formula1: "5" -# formula2: "25" -# allowBlank: true -# showInputMessage: true -# promptTitle: "Samples per Pool" -# prompt: "The requested number of samples per pool (between 5 and 25 inclusive)" -# conditional_formattings: -# empty_cell: -# is_text: -# is_num_samples_per_pool_in_valid_range: -# is_num_samples_per_pool_outside_valid_range: cells_per_chip_well: heading: scRNA Core Cells per Chip Well attribute: :cells_per_chip_well diff --git a/config/bulk_submission_excel/conditional_formattings.yml b/config/bulk_submission_excel/conditional_formattings.yml index d729c2e295..c72a7bd671 100644 --- a/config/bulk_submission_excel/conditional_formattings.yml +++ b/config/bulk_submission_excel/conditional_formattings.yml @@ -73,26 +73,6 @@ is_num_of_pools_outside_valid_range: formula1: "5" formula2: "25" priority: 2 -#is_num_samples_per_pool_in_valid_range: -# style: -# bg_color: "00FF00" -# type: :dxf -# options: -# type: :cellIs -# operator: :between -# formula1: "5" -# formula2: "25" -# priority: 2 -#is_num_samples_per_pool_outside_valid_range: -# style: -# bg_color: "FF0000" -# type: :dxf -# options: -# type: :cellIs -# operator: :notBetween -# formula1: "5" -# formula2: "25" -# priority: 2 is_num_cells_per_chip_well_in_valid_range: style: bg_color: "00FF00" From 73b3530fbf347184d12cba529c96572886238828 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 30 Oct 2024 13:41:40 +0000 Subject: [PATCH 004/115] Refactoring with the column name change --- app/models/bulk_submission.rb | 3 ++- app/models/submission/validations_by_template_name.rb | 2 +- config/bulk_submission_excel/columns.yml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 3c8c5fa313..0d5e5e828f 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -232,7 +232,8 @@ def process # rubocop:todo Metrics/CyclomaticComplexity 'priority', 'flowcell type', 'scrna core number of samples per pool', - 'scrna core cells per chip well' + 'scrna core cells per chip well', + 'scrna core number of pools' ].freeze ALIAS_FIELDS = { 'plate barcode' => 'barcode', 'tube barcode' => 'barcode' }.freeze diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 05ee3d482e..878a62ec76 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -9,7 +9,7 @@ module Submission::ValidationsByTemplateName HEADER_PROJECT_NAME = 'project name' HEADER_NUM_SAMPLES = 'scrna core number of samples per pool' HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' - HEADER_NUM_POOLS = "scrna core number of pools" + HEADER_NUM_POOLS = 'scrna core number of pools' # Applies additional validations based on the submission template type. # diff --git a/config/bulk_submission_excel/columns.yml b/config/bulk_submission_excel/columns.yml index 35186db6b7..e6df62c015 100644 --- a/config/bulk_submission_excel/columns.yml +++ b/config/bulk_submission_excel/columns.yml @@ -270,7 +270,7 @@ requested_flowcell_type: empty_cell: is_error: number_of_pools: - heading: scRNA Core Number Pools + heading: scRNA Core of Number Pools attribute: :number_of_pools unlocked: true type: :integer From 273d79a1a4949053c5d9293a3f3e437518c3a8e3 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 30 Oct 2024 14:37:27 +0000 Subject: [PATCH 005/115] Refactoring with the column name change --- app/models/bulk_submission.rb | 2 +- .../validations_by_template_name.rb | 3 +- .../scrna_additional_validations_valid.csv | 2 +- spec/models/bulk_submission_spec.rb | 53 +------------------ 4 files changed, 4 insertions(+), 56 deletions(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 0d5e5e828f..2033febf24 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -231,7 +231,7 @@ def process # rubocop:todo Metrics/CyclomaticComplexity 'gigabases expected', 'priority', 'flowcell type', - 'scrna core number of samples per pool', + 'scrna core number of pools', 'scrna core cells per chip well', 'scrna core number of pools' ].freeze diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 878a62ec76..25c924cdb6 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -33,9 +33,8 @@ def apply_additional_validations_by_template_name case submission_template_name # this validation is for the scRNA pipeline cDNA submission when SCRNA_CORE_CDNA_PREP_GEM_X_5P - validate_consistent_column_value(HEADER_NUM_SAMPLES) - validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) validate_consistent_column_value(HEADER_NUM_POOLS) + validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) end end diff --git a/spec/data/submission/scrna_additional_validations_valid.csv b/spec/data/submission/scrna_additional_validations_valid.csv index abebd6f92c..149cc2316f 100644 --- a/spec/data/submission/scrna_additional_validations_valid.csv +++ b/spec/data/submission/scrna_additional_validations_valid.csv @@ -1,4 +1,4 @@ -User Login,template name,study name,project name,submission name,asset names,asset group name,read length,fragment size from,fragment size to,library type,comments,pre-capture plex level,pre-capture group,gigabases expected,scrna core number of samples per pool,scrna core cells per chip well +User Login,template name,study name,project name,submission name,asset names,asset group name,read length,fragment size from,fragment size to,library type,comments,pre-capture plex level,pre-capture group,gigabases expected,scrna core number of pools,scrna core cells per chip well user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Test project,sub1,,assetgroup123,100,,,Standard,hello there,,,1.35,15,10000 user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Test project,sub2,,assetgroup123,100,,,Standard,hello there,,,1.35,15,10000 user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Project 1,sub2,,assetgroup2,100,,,Standard,hello there,,,1.35,10,20000 diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 7dc83a2147..491d92cca7 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -259,6 +259,7 @@ } } end + let(:spreadsheet_filename) { 'scrna_additional_validations_valid.csv' } before { SubmissionSerializer.construct!(submission_template_hash) } @@ -277,57 +278,5 @@ expect(generated_submission.orders.count).to eq(1) end end - - context 'when invalid for scRNA template on samples per pool' do - let(:submission_template_hash) do - { - name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', - submission_class_name: 'LinearSubmission', - product_catalogue: 'Generic', - submission_parameters: { - request_options: { - }, - request_types: request_types.map(&:key) - } - } - end - let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_samples_per_pool.csv' } - - before { SubmissionSerializer.construct!(submission_template_hash) } - - it 'raises an error and sets an error message' do - expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) - expect(subject.errors.messages[:spreadsheet][0]).to eq( - "Inconsistent values for column 'scrna core number of samples per pool' for Study name 'abc123_study' " \ - "and Project name 'Test project', all rows for a specific study and project must have the same value" - ) - end - end - - context 'when invalid for scRNA template on cells per chip well' do - let(:submission_template_hash) do - { - name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', - submission_class_name: 'LinearSubmission', - product_catalogue: 'Generic', - submission_parameters: { - request_options: { - }, - request_types: request_types.map(&:key) - } - } - end - let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_cells_per_chip_well.csv' } - - before { SubmissionSerializer.construct!(submission_template_hash) } - - it 'raises an error and sets an error message' do - expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) - expect(subject.errors.messages[:spreadsheet][0]).to eq( - "Inconsistent values for column 'scrna core cells per chip well' for Study name 'abc123_study' " \ - "and Project name 'Test project', all rows for a specific study and project must have the same value" - ) - end - end end end From 7b9d50e652274d878f891d64462a5a6c12ebdefc Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 31 Oct 2024 11:18:33 +0000 Subject: [PATCH 006/115] Refactoring with the column name change --- app/models/bulk_submission.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 2033febf24..6eedfc75b0 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -154,6 +154,8 @@ def process # rubocop:todo Metrics/CyclomaticComplexity raise ActiveRecord::RecordInvalid, self if errors.count > 0 + binding.pry + # Within a single transaction process each of the rows of the CSV file as a separate submission. Any name # fields need to be mapped to IDs, and the 'assets' field needs to be split up and processed if present. # rubocop:todo Metrics/BlockLength @@ -313,6 +315,13 @@ def add_study_to_assets(assets, study) assets.map(&:samples).flatten.uniq.each { |sample| sample.studies << study unless sample.studies.include?(study) } end + + # Assigns a value from the source object to the target object if the source value is present. + # + # @param [Hash] source_obj The source object containing the value to be assigned. + # @param [String, Symbol] source_key The key to look up the value in the source object. + # @param [Hash] target_obj The target object where the value will be assigned. + # @param [String, Symbol] target_key The key to assign the value in the target object. def assign_value_if_source_present(source_obj, source_key, target_obj, target_key) target_obj[target_key] = source_obj[source_key] if source_obj[source_key].present? end @@ -330,7 +339,7 @@ def extract_request_options(details) ['gigabases expected', 'gigabases_expected'], ['primer panel', 'primer_panel_name'], ['flowcell type', 'requested_flowcell_type'], - ['scrna core number of samples per pool', 'number_of_samples_per_pool'], + ['scrna core number of pools', 'number_of_samples_per_pool'], ['scrna core cells per chip well', 'cells_per_chip_well'] ].each do |source_key, target_key| assign_value_if_source_present(details, source_key, request_options, target_key) From 7ced46956c54cd5f7652c38a7334341e54ebb4aa Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Mon, 4 Nov 2024 15:41:13 +0000 Subject: [PATCH 007/115] New tests --- .rubocop_todo.yml | 1 + .../validations_by_template_name.rb | 20 +++- .../data/submission/scRNA_bulk_submission.csv | 98 +++++++++++++++++++ spec/models/bulk_submission_scrna_spec.rb | 59 +++++++++++ spec/models/bulk_submission_spec.rb | 52 ++++++++++ 5 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 spec/data/submission/scRNA_bulk_submission.csv create mode 100644 spec/models/bulk_submission_scrna_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 87034510a8..e01f9114a1 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -454,6 +454,7 @@ Naming/VariableNumber: - 'spec/models/api/messages/pac_bio_run_io_spec.rb' - 'spec/models/api/messages/pac_bio_run_with_tag2_io_spec.rb' - 'spec/models/bulk_submission_spec.rb' + - 'spec/models/bulk_submission_scrna_spec.rb' - 'spec/models/heron/factories/tube_rack_spec.rb' - 'spec/models/labware_spec.rb' - 'spec/models/labwhere_reception_spec.rb' diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 25c924cdb6..9894fdc780 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -38,6 +38,18 @@ def apply_additional_validations_by_template_name end end + def apply_number_of_samples_per_pool_validation + # Creates groups of rows based on the study and project name (i.e., study-project combinations) + group_rows_by_study_and_project + + end + + def group_rows_by_study_and_project + index_of_study_name = headers.index(HEADER_STUDY_NAME) + index_of_project_name = headers.index(HEADER_PROJECT_NAME) + csv_data_rows.group_by { |row| [row[index_of_study_name], row[index_of_project_name]] } + end + # Validates that the specified column is consistent for all rows with the same study and project name. # # This method groups the rows in the CSV data by the study name and project name, and checks if the specified column @@ -46,13 +58,11 @@ def apply_additional_validations_by_template_name # # @param column_header [String] The header of the column to validate. # @return [void] - # rubocop:disable Metrics/MethodLength,Metrics/AbcSize + # rubocop:disable Metrics/MethodLength def validate_consistent_column_value(column_header) - index_of_study_name = headers.index(HEADER_STUDY_NAME) - index_of_project_name = headers.index(HEADER_PROJECT_NAME) index_of_column = headers.index(column_header) - grouped_rows = csv_data_rows.group_by { |row| [row[index_of_study_name], row[index_of_project_name]] } + grouped_rows = group_rows_by_study_and_project grouped_rows.each do |study_project, rows| unique_values = rows.pluck(index_of_column).uniq @@ -66,5 +76,5 @@ def validate_consistent_column_value(column_header) ) end end - # rubocop:enable Metrics/MethodLength,Metrics/AbcSize + # rubocop:enable Metrics/MethodLength end diff --git a/spec/data/submission/scRNA_bulk_submission.csv b/spec/data/submission/scRNA_bulk_submission.csv new file mode 100644 index 0000000000..956d407de2 --- /dev/null +++ b/spec/data/submission/scRNA_bulk_submission.csv @@ -0,0 +1,98 @@ +Bulk Submissions Form,,,,,,,,,,,,,,,,,,,,,,, +User Login,Template Name,Project Name,Study Name,Submission name,Barcode,Plate Well,Asset Group Name,Fragment Size From,Fragment Size To,PCR Cycles,Library Type,Bait Library Name,Pre-capture Plex Level,Pre-capture Group,Read Length,Number of lanes,Priority,Primer Panel,Comments,Gigabases Expected,Flowcell Type,scRNA Core Number of pools,scRNA Core Cells per Chip Well +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H1,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H2,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H3,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H4,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H5,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H6,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H7,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H8,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H9,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H10,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H11,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,A12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,B12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,C12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,D12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,E12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,F12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,G12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,SQPD-12345,H12,assetgroup,,,,Standard,,,,100,1,,,Sample Comment,1.35,,7,13000 \ No newline at end of file diff --git a/spec/models/bulk_submission_scrna_spec.rb b/spec/models/bulk_submission_scrna_spec.rb new file mode 100644 index 0000000000..23f0322e52 --- /dev/null +++ b/spec/models/bulk_submission_scrna_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe BulkSubmission, with: :uploader do + subject(:bulk_submission) { described_class.new(spreadsheet: submission_file, encoding: encoding) } + + let(:encoding) { 'Windows-1252' } + let(:spreadsheet_path) { Rails.root.join('spec', 'data', 'submission', spreadsheet_filename) } + + # NB. fixture_file_upload is a Rails method on ActionDispatch::TestProcess::FixtureFile + let(:submission_file) { fixture_file_upload(spreadsheet_path) } + + let(:number_submissions_created) { subject.completed_submissions.first.length } + let(:generated_submissions) { Submission.find(subject.completed_submissions.first) } + let(:generated_submission) { generated_submissions.first } + let(:request_types) { create_list(:well_request_type, 2) } + + let!(:study) { create(:study, name: 'Test Study') } + let!(:plate) { create(:plate_with_tagged_wells, sample_count: 96, barcode: 'SQPD-12345') } + let(:asset_group) do + create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) + end + let!(:library_type) { create(:library_type, name: 'Standard') } + + after { submission_file.close } + + before do + create(:user, login: 'user') + create(:project, name: 'Test project') + end + + context 'when an scRNA Bulk Submission' do + let(:spreadsheet_filename) { 'scRNA_bulk_submission.csv' } + let(:submission_template_hash) do + { + name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + submission_class_name: 'LinearSubmission', + product_catalogue: 'Generic', + submission_parameters: { + request_options: {}, + request_types: request_types.map(&:key) + } + } + end + + before { SubmissionSerializer.construct!(submission_template_hash) } + + it 'is valid' do + expect(bulk_submission).to be_valid + end + + it 'generates submissions when processed' do + bulk_submission.process + expect(number_submissions_created).to eq(1) + end + end + +end diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 491d92cca7..49c86fa6fc 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -278,5 +278,57 @@ expect(generated_submission.orders.count).to eq(1) end end + + # context 'when invalid for scRNA template on samples per pool' do + # let(:submission_template_hash) do + # { + # name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + # submission_class_name: 'LinearSubmission', + # product_catalogue: 'Generic', + # submission_parameters: { + # request_options: { + # }, + # request_types: request_types.map(&:key) + # } + # } + # end + # let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_samples_per_pool.csv' } + # + # before { SubmissionSerializer.construct!(submission_template_hash) } + # + # it 'raises an error and sets an error message' do + # expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) + # expect(subject.errors.messages[:spreadsheet][0]).to eq( + # "Inconsistent values for column 'scrna core number of samples per pool' for Study name 'abc123_study' " \ + # "and Project name 'Test project', all rows for a specific study and project must have the same value" + # ) + # end + # end + + # context 'when invalid for scRNA template on cells per chip well' do + # let(:submission_template_hash) do + # { + # name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + # submission_class_name: 'LinearSubmission', + # product_catalogue: 'Generic', + # submission_parameters: { + # request_options: { + # }, + # request_types: request_types.map(&:key) + # } + # } + # end + # let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_cells_per_chip_well.csv' } + # + # before { SubmissionSerializer.construct!(submission_template_hash) } + # + # it 'raises an error and sets an error message' do + # expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) + # expect(subject.errors.messages[:spreadsheet][0]).to eq( + # "Inconsistent values for column 'scrna core cells per chip well' for Study name 'abc123_study' " \ + # "and Project name 'Test project', all rows for a specific study and project must have the same value" + # ) + # end + # end end end From af5ef2a4b5e251e3f21f7116f96100dccf08bfc4 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Mon, 4 Nov 2024 18:52:25 +0000 Subject: [PATCH 008/115] More tests --- app/models/bulk_submission.rb | 2 -- .../validations_by_template_name.rb | 36 ++++++++++++++++++- spec/models/bulk_submission_scrna_spec.rb | 4 +-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 6eedfc75b0..57b01ecd8a 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -154,8 +154,6 @@ def process # rubocop:todo Metrics/CyclomaticComplexity raise ActiveRecord::RecordInvalid, self if errors.count > 0 - binding.pry - # Within a single transaction process each of the rows of the CSV file as a separate submission. Any name # fields need to be mapped to IDs, and the 'assets' field needs to be split up and processed if present. # rubocop:todo Metrics/BlockLength diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 9894fdc780..2811348f79 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -7,6 +7,9 @@ module Submission::ValidationsByTemplateName HEADER_TEMPLATE_NAME = 'template name' HEADER_STUDY_NAME = 'study name' HEADER_PROJECT_NAME = 'project name' + HEADER_BARCODE = 'barcode' + HEADER_PLATE_WELLS = 'plate well' + HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' HEADER_NUM_SAMPLES = 'scrna core number of samples per pool' HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' HEADER_NUM_POOLS = 'scrna core number of pools' @@ -39,7 +42,7 @@ def apply_additional_validations_by_template_name end def apply_number_of_samples_per_pool_validation - # Creates groups of rows based on the study and project name (i.e., study-project combinations) + # Creates groups of rows based on the study and project name (pool_number.e., study-project combinations) group_rows_by_study_and_project end @@ -50,6 +53,35 @@ def group_rows_by_study_and_project csv_data_rows.group_by { |row| [row[index_of_study_name], row[index_of_project_name]] } end + def calculate_samples_per_pool_for_tube_or_plate + grouped_rows = group_rows_by_study_and_project + grouped_rows.each_value do |rows| + barcodes = rows.pluck(headers.index(HEADER_BARCODE)) + well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) + # Skip if the asset is not a plate or tube + next unless (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) + plate = Plate.find_from_any_barcode(barcodes.uniq.first) + next if plate.nil? + wells = plate.wells.for_bulk_submission.located_at(well_locations) + total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i + number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + + # Perform the calculation for the number of samples per pool + int_division = total_number_of_samples_per_study_project / number_of_pools + remainder = total_number_of_samples_per_study_project % number_of_pools + + number_of_pools.times do |pool_number| + samples_per_pool = int_division + samples_per_pool += 1 if pool_number < remainder + next unless samples_per_pool > 25 || samples_per_pool < 5 + errors.add(:spreadsheet, + "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ + "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ + "is less than 5 or greater than 25 for pool number #{pool_number}") + end + end + end + # Validates that the specified column is consistent for all rows with the same study and project name. # # This method groups the rows in the CSV data by the study name and project name, and checks if the specified column @@ -62,6 +94,8 @@ def group_rows_by_study_and_project def validate_consistent_column_value(column_header) index_of_column = headers.index(column_header) + calculate_samples_per_pool_for_tube_or_plate + grouped_rows = group_rows_by_study_and_project grouped_rows.each do |study_project, rows| diff --git a/spec/models/bulk_submission_scrna_spec.rb b/spec/models/bulk_submission_scrna_spec.rb index 23f0322e52..0c6020d710 100644 --- a/spec/models/bulk_submission_scrna_spec.rb +++ b/spec/models/bulk_submission_scrna_spec.rb @@ -18,7 +18,7 @@ let!(:study) { create(:study, name: 'Test Study') } let!(:plate) { create(:plate_with_tagged_wells, sample_count: 96, barcode: 'SQPD-12345') } - let(:asset_group) do + let!(:asset_group) do create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) end let!(:library_type) { create(:library_type, name: 'Standard') } @@ -30,7 +30,7 @@ create(:project, name: 'Test project') end - context 'when an scRNA Bulk Submission' do + context 'when an scRNA Bulk Submission for plate' do let(:spreadsheet_filename) { 'scRNA_bulk_submission.csv' } let(:submission_template_hash) do { From 0883d04c03f4571916d043c9a834d68af5de0294 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 08:22:31 +0000 Subject: [PATCH 009/115] Making prettier happy --- app/models/bulk_submission.rb | 1 - app/models/submission/validations_by_template_name.rb | 11 ++++++----- spec/models/bulk_submission_scrna_spec.rb | 8 +++----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 57b01ecd8a..1d3498a124 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -313,7 +313,6 @@ def add_study_to_assets(assets, study) assets.map(&:samples).flatten.uniq.each { |sample| sample.studies << study unless sample.studies.include?(study) } end - # Assigns a value from the source object to the target object if the source value is present. # # @param [Hash] source_obj The source object containing the value to be assigned. diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 2811348f79..ad630be4e5 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -44,7 +44,6 @@ def apply_additional_validations_by_template_name def apply_number_of_samples_per_pool_validation # Creates groups of rows based on the study and project name (pool_number.e., study-project combinations) group_rows_by_study_and_project - end def group_rows_by_study_and_project @@ -74,10 +73,12 @@ def calculate_samples_per_pool_for_tube_or_plate samples_per_pool = int_division samples_per_pool += 1 if pool_number < remainder next unless samples_per_pool > 25 || samples_per_pool < 5 - errors.add(:spreadsheet, - "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ - "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ - "is less than 5 or greater than 25 for pool number #{pool_number}") + errors.add( + :spreadsheet, + "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ + "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ + "is less than 5 or greater than 25 for pool number #{pool_number}" + ) end end end diff --git a/spec/models/bulk_submission_scrna_spec.rb b/spec/models/bulk_submission_scrna_spec.rb index 0c6020d710..30097a9216 100644 --- a/spec/models/bulk_submission_scrna_spec.rb +++ b/spec/models/bulk_submission_scrna_spec.rb @@ -18,9 +18,7 @@ let!(:study) { create(:study, name: 'Test Study') } let!(:plate) { create(:plate_with_tagged_wells, sample_count: 96, barcode: 'SQPD-12345') } - let!(:asset_group) do - create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) - end + let!(:asset_group) { create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) } let!(:library_type) { create(:library_type, name: 'Standard') } after { submission_file.close } @@ -38,7 +36,8 @@ submission_class_name: 'LinearSubmission', product_catalogue: 'Generic', submission_parameters: { - request_options: {}, + request_options: { + }, request_types: request_types.map(&:key) } } @@ -55,5 +54,4 @@ expect(number_submissions_created).to eq(1) end end - end From 5f0691f9ae0786c7caba12682b35c08e04cf9a1e Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 08:26:42 +0000 Subject: [PATCH 010/115] Making prettier happy --- .../validations_by_template_name.rb | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index ad630be4e5..aa7b42f146 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -53,33 +53,41 @@ def group_rows_by_study_and_project end def calculate_samples_per_pool_for_tube_or_plate - grouped_rows = group_rows_by_study_and_project - grouped_rows.each_value do |rows| - barcodes = rows.pluck(headers.index(HEADER_BARCODE)) - well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) - # Skip if the asset is not a plate or tube - next unless (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) - plate = Plate.find_from_any_barcode(barcodes.uniq.first) - next if plate.nil? - wells = plate.wells.for_bulk_submission.located_at(well_locations) - total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i - number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + unless headers.index(HEADER_BARCODE).nil? && + headers + .index(HEADER_PLATE_WELLS) + .nil? { |_| + grouped_rows = group_rows_by_study_and_project + grouped_rows.each_value do |rows| + barcodes = rows.pluck(headers.index(HEADER_BARCODE)) + well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) + # Skip if the asset is not a plate or tube + unless (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) + next + end + plate = Plate.find_from_any_barcode(barcodes.uniq.first) + next if plate.nil? + wells = plate.wells.for_bulk_submission.located_at(well_locations) + total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i + number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i - # Perform the calculation for the number of samples per pool - int_division = total_number_of_samples_per_study_project / number_of_pools - remainder = total_number_of_samples_per_study_project % number_of_pools + # Perform the calculation for the number of samples per pool + int_division = total_number_of_samples_per_study_project / number_of_pools + remainder = total_number_of_samples_per_study_project % number_of_pools - number_of_pools.times do |pool_number| - samples_per_pool = int_division - samples_per_pool += 1 if pool_number < remainder - next unless samples_per_pool > 25 || samples_per_pool < 5 - errors.add( - :spreadsheet, - "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ - "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ - "is less than 5 or greater than 25 for pool number #{pool_number}" - ) - end + number_of_pools.times do |pool_number| + samples_per_pool = int_division + samples_per_pool += 1 if pool_number < remainder + next unless samples_per_pool > 25 || samples_per_pool < 5 + errors.add( + :spreadsheet, + "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ + "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ + "is less than 5 or greater than 25 for pool number #{pool_number}" + ) + end + end + } end end From cb268501681eb98644fe714efc96ef4269fd58c1 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 08:36:35 +0000 Subject: [PATCH 011/115] Fixing linters --- .rubocop_todo.yml | 3 ++- app/models/submission/validations_by_template_name.rb | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index e01f9114a1..122313a1a6 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -297,6 +297,7 @@ Metrics/AbcSize: - 'app/controllers/api/v2/transfers/transfers_controller.rb' - 'app/jobs/export_pool_xp_to_traction_job.rb' - 'app/sample_manifest_excel/sample_manifest_excel/manifest_type_list.rb' + - 'spec/models/bulk_submission_scrna_spec.rb' # Offense count: 1 # Configuration parameters: CountComments, Max, CountAsOne. @@ -454,7 +455,6 @@ Naming/VariableNumber: - 'spec/models/api/messages/pac_bio_run_io_spec.rb' - 'spec/models/api/messages/pac_bio_run_with_tag2_io_spec.rb' - 'spec/models/bulk_submission_spec.rb' - - 'spec/models/bulk_submission_scrna_spec.rb' - 'spec/models/heron/factories/tube_rack_spec.rb' - 'spec/models/labware_spec.rb' - 'spec/models/labwhere_reception_spec.rb' @@ -858,6 +858,7 @@ RSpec/LetSetup: - 'spec/models/transfer_request_collection_spec.rb' - 'spec/requests/api/v2/comments_spec.rb' - 'spec/requests/api/v2/plates_spec.rb' + - 'spec/models/bulk_submission_scrna_spec.rb' # Offense count: 40 # Configuration parameters: EnforcedStyle. diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index aa7b42f146..6404e29a71 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -52,6 +52,11 @@ def group_rows_by_study_and_project csv_data_rows.group_by { |row| [row[index_of_study_name], row[index_of_project_name]] } end + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + # rubocop:disable Metrics/BlockLength def calculate_samples_per_pool_for_tube_or_plate unless headers.index(HEADER_BARCODE).nil? && headers @@ -90,6 +95,11 @@ def calculate_samples_per_pool_for_tube_or_plate } end end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity + # rubocop:enable Metrics/BlockLength # Validates that the specified column is consistent for all rows with the same study and project name. # From 04c49ae035ea2b8e1b19496a3dea95c7d65d1fc0 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 08:41:43 +0000 Subject: [PATCH 012/115] Fixing linters --- .../validations_by_template_name.rb | 101 +++++++++--------- 1 file changed, 53 insertions(+), 48 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 6404e29a71..0957b45237 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -52,54 +52,6 @@ def group_rows_by_study_and_project csv_data_rows.group_by { |row| [row[index_of_study_name], row[index_of_project_name]] } end - # rubocop:disable Metrics/MethodLength - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - # rubocop:disable Metrics/BlockLength - def calculate_samples_per_pool_for_tube_or_plate - unless headers.index(HEADER_BARCODE).nil? && - headers - .index(HEADER_PLATE_WELLS) - .nil? { |_| - grouped_rows = group_rows_by_study_and_project - grouped_rows.each_value do |rows| - barcodes = rows.pluck(headers.index(HEADER_BARCODE)) - well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) - # Skip if the asset is not a plate or tube - unless (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) - next - end - plate = Plate.find_from_any_barcode(barcodes.uniq.first) - next if plate.nil? - wells = plate.wells.for_bulk_submission.located_at(well_locations) - total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i - number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i - - # Perform the calculation for the number of samples per pool - int_division = total_number_of_samples_per_study_project / number_of_pools - remainder = total_number_of_samples_per_study_project % number_of_pools - - number_of_pools.times do |pool_number| - samples_per_pool = int_division - samples_per_pool += 1 if pool_number < remainder - next unless samples_per_pool > 25 || samples_per_pool < 5 - errors.add( - :spreadsheet, - "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ - "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ - "is less than 5 or greater than 25 for pool number #{pool_number}" - ) - end - end - } - end - end - # rubocop:enable Metrics/MethodLength - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/CyclomaticComplexity - # rubocop:enable Metrics/PerceivedComplexity - # rubocop:enable Metrics/BlockLength # Validates that the specified column is consistent for all rows with the same study and project name. # @@ -130,4 +82,57 @@ def validate_consistent_column_value(column_header) end end # rubocop:enable Metrics/MethodLength + + def calculate_samples_per_pool_for_tube_or_plate + return if headers.index(HEADER_BARCODE).nil? && headers.index(HEADER_PLATE_WELLS).nil? + + grouped_rows = group_rows_by_study_and_project + grouped_rows.each_value do |rows| + process_rows(rows) + end + end + + private + + # rubocop:disable Metrics/AbcSize + def process_rows(rows) + barcodes = rows.pluck(headers.index(HEADER_BARCODE)) + well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) + + return unless valid_asset?(barcodes, well_locations) + + plate = Plate.find_from_any_barcode(barcodes.uniq.first) + return if plate.nil? + + wells = plate.wells.for_bulk_submission.located_at(well_locations) + total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i + number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + + validate_samples_per_pool(rows, total_number_of_samples_per_study_project, number_of_pools) + end + # rubocop:enable Metrics/AbcSize + + def valid_asset?(barcodes, well_locations) + (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) + end + + # rubocop:disable Metrics/MethodLength + def validate_samples_per_pool(rows, total_samples, number_of_pools) + int_division = total_samples / number_of_pools + remainder = total_samples % number_of_pools + + number_of_pools.times do |pool_number| + samples_per_pool = int_division + samples_per_pool += 1 if pool_number < remainder + next unless samples_per_pool > 25 || samples_per_pool < 5 + + errors.add( + :spreadsheet, + "Number of samples per pool for Study name '#{rows.first[headers.index(HEADER_STUDY_NAME)]}' " \ + "and Project name '#{rows.first[headers.index(HEADER_PROJECT_NAME)]}' " \ + "is less than 5 or greater than 25 for pool number #{pool_number}" + ) + end + end + # rubocop:enable Metrics/MethodLength end From 8887732efe550acf06af815d88c92a1e42d9126b Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 08:45:03 +0000 Subject: [PATCH 013/115] Refactoring for tubes and plates --- app/models/submission/validations_by_template_name.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 0957b45237..a3359b37b6 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -38,6 +38,7 @@ def apply_additional_validations_by_template_name when SCRNA_CORE_CDNA_PREP_GEM_X_5P validate_consistent_column_value(HEADER_NUM_POOLS) validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) + validate_samples_per_pool_for_tube_or_plate end end @@ -52,7 +53,6 @@ def group_rows_by_study_and_project csv_data_rows.group_by { |row| [row[index_of_study_name], row[index_of_project_name]] } end - # Validates that the specified column is consistent for all rows with the same study and project name. # # This method groups the rows in the CSV data by the study name and project name, and checks if the specified column @@ -65,8 +65,6 @@ def group_rows_by_study_and_project def validate_consistent_column_value(column_header) index_of_column = headers.index(column_header) - calculate_samples_per_pool_for_tube_or_plate - grouped_rows = group_rows_by_study_and_project grouped_rows.each do |study_project, rows| @@ -83,13 +81,11 @@ def validate_consistent_column_value(column_header) end # rubocop:enable Metrics/MethodLength - def calculate_samples_per_pool_for_tube_or_plate + def validate_samples_per_pool_for_tube_or_plate return if headers.index(HEADER_BARCODE).nil? && headers.index(HEADER_PLATE_WELLS).nil? grouped_rows = group_rows_by_study_and_project - grouped_rows.each_value do |rows| - process_rows(rows) - end + grouped_rows.each_value { |rows| process_rows(rows) } end private From 5aa28cb530b35577a2d436338cb2b4611bf8cbbb Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 08:56:07 +0000 Subject: [PATCH 014/115] Refactoring for tubes and plates --- .../validations_by_template_name.rb | 4 +- ...alidations_invalid_cells_per_chip_well.csv | 7 +- spec/models/bulk_submission_spec.rb | 76 ++++++------------- 3 files changed, 30 insertions(+), 57 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index a3359b37b6..584197a70b 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -10,9 +10,7 @@ module Submission::ValidationsByTemplateName HEADER_BARCODE = 'barcode' HEADER_PLATE_WELLS = 'plate well' HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' - HEADER_NUM_SAMPLES = 'scrna core number of samples per pool' HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' - HEADER_NUM_POOLS = 'scrna core number of pools' # Applies additional validations based on the submission template type. # @@ -36,7 +34,7 @@ def apply_additional_validations_by_template_name case submission_template_name # this validation is for the scRNA pipeline cDNA submission when SCRNA_CORE_CDNA_PREP_GEM_X_5P - validate_consistent_column_value(HEADER_NUM_POOLS) + validate_consistent_column_value(HEADER_NUMBER_OF_POOLS) validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) validate_samples_per_pool_for_tube_or_plate end diff --git a/spec/data/submission/scrna_additional_validations_invalid_cells_per_chip_well.csv b/spec/data/submission/scrna_additional_validations_invalid_cells_per_chip_well.csv index c483aa5a04..b64202afa1 100644 --- a/spec/data/submission/scrna_additional_validations_invalid_cells_per_chip_well.csv +++ b/spec/data/submission/scrna_additional_validations_invalid_cells_per_chip_well.csv @@ -1,3 +1,4 @@ -User Login,template name,study name,project name,submission name,asset names,asset group name,read length,fragment size from,fragment size to,library type,comments,pre-capture plex level,pre-capture group,gigabases expected,scrna core number of samples per pool,scrna core cells per chip well -user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Test project,sub1,,assetgroup123,100,,,Standard,hello there,,,1.35,15,10000 -user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Test project,sub2,,assetgroup123,100,,,Standard,hello there,,,1.35,15,20000 +User Login,template name,study name,project name,submission name,asset names,asset group name,read length,fragment size from,fragment size to,library type,comments,pre-capture plex level,pre-capture group,gigabases expected,scrna core number of pools,scrna core cells per chip well +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Test project,sub1,,assetgroup123,100,,,Standard,hello there,,,1.35,7,10000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,abc123_study,Test project,sub2,,assetgroup123,100,,,Standard,hello there,,,1.35,7,20000 +,,,,,,,,,,,,,,, \ No newline at end of file diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 49c86fa6fc..8f8b1e8c92 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -279,56 +279,30 @@ end end - # context 'when invalid for scRNA template on samples per pool' do - # let(:submission_template_hash) do - # { - # name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', - # submission_class_name: 'LinearSubmission', - # product_catalogue: 'Generic', - # submission_parameters: { - # request_options: { - # }, - # request_types: request_types.map(&:key) - # } - # } - # end - # let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_samples_per_pool.csv' } - # - # before { SubmissionSerializer.construct!(submission_template_hash) } - # - # it 'raises an error and sets an error message' do - # expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) - # expect(subject.errors.messages[:spreadsheet][0]).to eq( - # "Inconsistent values for column 'scrna core number of samples per pool' for Study name 'abc123_study' " \ - # "and Project name 'Test project', all rows for a specific study and project must have the same value" - # ) - # end - # end - - # context 'when invalid for scRNA template on cells per chip well' do - # let(:submission_template_hash) do - # { - # name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', - # submission_class_name: 'LinearSubmission', - # product_catalogue: 'Generic', - # submission_parameters: { - # request_options: { - # }, - # request_types: request_types.map(&:key) - # } - # } - # end - # let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_cells_per_chip_well.csv' } - # - # before { SubmissionSerializer.construct!(submission_template_hash) } - # - # it 'raises an error and sets an error message' do - # expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) - # expect(subject.errors.messages[:spreadsheet][0]).to eq( - # "Inconsistent values for column 'scrna core cells per chip well' for Study name 'abc123_study' " \ - # "and Project name 'Test project', all rows for a specific study and project must have the same value" - # ) - # end - # end + context 'when invalid for scRNA template on cells per chip well' do + let(:submission_template_hash) do + { + name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + submission_class_name: 'LinearSubmission', + product_catalogue: 'Generic', + submission_parameters: { + request_options: { + }, + request_types: request_types.map(&:key) + } + } + end + let(:spreadsheet_filename) { 'scrna_additional_validations_invalid_cells_per_chip_well.csv' } + + before { SubmissionSerializer.construct!(submission_template_hash) } + + it 'raises an error and sets an error message' do + expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(subject.errors.messages[:spreadsheet][0]).to eq( + "Inconsistent values for column 'scrna core cells per chip well' for Study name 'abc123_study' " \ + "and Project name 'Test project', all rows for a specific study and project must have the same value" + ) + end + end end end From 7aba1862f50a40f85cb4ef24f755ff3aa4ada916 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 11:45:59 +0000 Subject: [PATCH 015/115] Reverting some changes that aren't required --- app/models/bulk_submission.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 1d3498a124..85259ae203 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -336,7 +336,7 @@ def extract_request_options(details) ['gigabases expected', 'gigabases_expected'], ['primer panel', 'primer_panel_name'], ['flowcell type', 'requested_flowcell_type'], - ['scrna core number of pools', 'number_of_samples_per_pool'], + ['scrna core number of samples per pool', 'number_of_samples_per_pool'], ['scrna core cells per chip well', 'cells_per_chip_well'] ].each do |source_key, target_key| assign_value_if_source_present(details, source_key, request_options, target_key) From 8223742be77684b3585e74a523d2ec47b69ac752 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 12:19:59 +0000 Subject: [PATCH 016/115] Replacing no of sample per pool with no of pools --- app/models/bulk_submission.rb | 2 +- app/models/submission/validations_by_template_name.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 85259ae203..3182265705 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -336,7 +336,7 @@ def extract_request_options(details) ['gigabases expected', 'gigabases_expected'], ['primer panel', 'primer_panel_name'], ['flowcell type', 'requested_flowcell_type'], - ['scrna core number of samples per pool', 'number_of_samples_per_pool'], + ['scrna core number of pools', 'number_of_pools'], ['scrna core cells per chip well', 'cells_per_chip_well'] ].each do |source_key, target_key| assign_value_if_source_present(details, source_key, request_options, target_key) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 584197a70b..43e6c874a4 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -106,6 +106,7 @@ def process_rows(rows) end # rubocop:enable Metrics/AbcSize + # Checks if the asset is either a tube or a plate. def valid_asset?(barcodes, well_locations) (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) end From ce715de816ae106e72be7d268be0056376d47b60 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 14:25:31 +0000 Subject: [PATCH 017/115] Adding validation logic for tubes --- .../validations_by_template_name.rb | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 43e6c874a4..8c4c13f2ce 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +# rubocop:todo Metrics/ModuleLength module Submission::ValidationsByTemplateName # Template names SCRNA_CORE_CDNA_PREP_GEM_X_5P = 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p' @@ -89,12 +90,19 @@ def validate_samples_per_pool_for_tube_or_plate private # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/MethodLength def process_rows(rows) barcodes = rows.pluck(headers.index(HEADER_BARCODE)) well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) return unless valid_asset?(barcodes, well_locations) + if plate?(barcodes, well_locations) + validate_for_plates(barcodes, well_locations, rows) + elsif tube?(barcodes, well_locations) + validate_for_tubes(barcodes, rows) + end + plate = Plate.find_from_any_barcode(barcodes.uniq.first) return if plate.nil? @@ -105,6 +113,50 @@ def process_rows(rows) validate_samples_per_pool(rows, total_number_of_samples_per_study_project, number_of_pools) end # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/MethodLength + + # rubocop:disable Metrics/AbcSize + def validate_for_plates(barcodes, well_locations, rows) + plate = Plate.find_from_any_barcode(barcodes.uniq.first) + return if plate.nil? + + wells = plate.wells.for_bulk_submission.located_at(well_locations) + total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i + number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + + validate_samples_per_pool(rows, total_number_of_samples_per_study_project, number_of_pools) + end + # rubocop:enable Metrics/AbcSize + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/MethodLength + def validate_for_tubes(barcodes, rows) + tubes = + Receptacle + .on_a(Tube) + .for_bulk_submission + .with_barcode(barcodes) + .tap do |found| + missing = details['barcode'].reject { |barcode| found.any? { |tube| tube.any_barcode_matching?(barcode) } } + if missing.present? + raise ActiveRecord::RecordNotFound, "Could not find Tubes with barcodes #{missing.inspect}" + end + end + total_number_of_samples_per_study_project = tubes.map(&:samples).flatten.count.to_i + number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + + validate_samples_per_pool(rows, total_number_of_samples_per_study_project, number_of_pools) + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/MethodLength + + def plate?(barcodes, well_locations) + barcodes.present? && well_locations.present? + end + + def tube?(barcodes, well_locations) + barcodes.present? && well_locations.blank? + end # Checks if the asset is either a tube or a plate. def valid_asset?(barcodes, well_locations) @@ -131,3 +183,4 @@ def validate_samples_per_pool(rows, total_samples, number_of_pools) end # rubocop:enable Metrics/MethodLength end +# rubocop:enable Metrics/ModuleLength From 217a62f48e72deace3aa60f83afca4c758729d81 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 14:31:01 +0000 Subject: [PATCH 018/115] Adding validation logic for tubes --- .../submission/validations_by_template_name.rb | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 8c4c13f2ce..5188f311aa 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -89,31 +89,16 @@ def validate_samples_per_pool_for_tube_or_plate private - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/MethodLength def process_rows(rows) barcodes = rows.pluck(headers.index(HEADER_BARCODE)) well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) - return unless valid_asset?(barcodes, well_locations) - if plate?(barcodes, well_locations) validate_for_plates(barcodes, well_locations, rows) elsif tube?(barcodes, well_locations) validate_for_tubes(barcodes, rows) end - - plate = Plate.find_from_any_barcode(barcodes.uniq.first) - return if plate.nil? - - wells = plate.wells.for_bulk_submission.located_at(well_locations) - total_number_of_samples_per_study_project = wells.map(&:samples).flatten.count.to_i - number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i - - validate_samples_per_pool(rows, total_number_of_samples_per_study_project, number_of_pools) end - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/MethodLength # rubocop:disable Metrics/AbcSize def validate_for_plates(barcodes, well_locations, rows) From 89a86bd92221604281e469739e9dd21e21eaae91 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 15:29:09 +0000 Subject: [PATCH 019/115] Changing column name --- config/bulk_submission_excel/columns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/bulk_submission_excel/columns.yml b/config/bulk_submission_excel/columns.yml index e6df62c015..c00f5a51e1 100644 --- a/config/bulk_submission_excel/columns.yml +++ b/config/bulk_submission_excel/columns.yml @@ -270,7 +270,7 @@ requested_flowcell_type: empty_cell: is_error: number_of_pools: - heading: scRNA Core of Number Pools + heading: scRNA Core Number of Pools attribute: :number_of_pools unlocked: true type: :integer From f4649f86e34ec54d99e7e61293d168aa17193647 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 15:31:41 +0000 Subject: [PATCH 020/115] Refactoring the existing bulk submission tests to include the new ones --- spec/models/bulk_submission_scrna_spec.rb | 57 ----------------------- spec/models/bulk_submission_spec.rb | 32 +++++++++++++ 2 files changed, 32 insertions(+), 57 deletions(-) delete mode 100644 spec/models/bulk_submission_scrna_spec.rb diff --git a/spec/models/bulk_submission_scrna_spec.rb b/spec/models/bulk_submission_scrna_spec.rb deleted file mode 100644 index 30097a9216..0000000000 --- a/spec/models/bulk_submission_scrna_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -describe BulkSubmission, with: :uploader do - subject(:bulk_submission) { described_class.new(spreadsheet: submission_file, encoding: encoding) } - - let(:encoding) { 'Windows-1252' } - let(:spreadsheet_path) { Rails.root.join('spec', 'data', 'submission', spreadsheet_filename) } - - # NB. fixture_file_upload is a Rails method on ActionDispatch::TestProcess::FixtureFile - let(:submission_file) { fixture_file_upload(spreadsheet_path) } - - let(:number_submissions_created) { subject.completed_submissions.first.length } - let(:generated_submissions) { Submission.find(subject.completed_submissions.first) } - let(:generated_submission) { generated_submissions.first } - let(:request_types) { create_list(:well_request_type, 2) } - - let!(:study) { create(:study, name: 'Test Study') } - let!(:plate) { create(:plate_with_tagged_wells, sample_count: 96, barcode: 'SQPD-12345') } - let!(:asset_group) { create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) } - let!(:library_type) { create(:library_type, name: 'Standard') } - - after { submission_file.close } - - before do - create(:user, login: 'user') - create(:project, name: 'Test project') - end - - context 'when an scRNA Bulk Submission for plate' do - let(:spreadsheet_filename) { 'scRNA_bulk_submission.csv' } - let(:submission_template_hash) do - { - name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', - submission_class_name: 'LinearSubmission', - product_catalogue: 'Generic', - submission_parameters: { - request_options: { - }, - request_types: request_types.map(&:key) - } - } - end - - before { SubmissionSerializer.construct!(submission_template_hash) } - - it 'is valid' do - expect(bulk_submission).to be_valid - end - - it 'generates submissions when processed' do - bulk_submission.process - expect(number_submissions_created).to eq(1) - end - end -end diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 8f8b1e8c92..2d157eb45e 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -304,5 +304,37 @@ ) end end + + context 'when an scRNA Bulk Submission for plate' do + let!(:study) { create(:study, name: 'Test Study') } + let!(:plate) { create(:plate_with_tagged_wells, sample_count: 96, barcode: 'SQPD-12345') } + let!(:asset_group) { create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) } + let!(:library_type) { create(:library_type, name: 'Standard') } + + let(:spreadsheet_filename) { 'scRNA_bulk_submission.csv' } + let(:submission_template_hash) do + { + name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + submission_class_name: 'LinearSubmission', + product_catalogue: 'Generic', + submission_parameters: { + request_options: { + }, + request_types: request_types.map(&:key) + } + } + end + + before { SubmissionSerializer.construct!(submission_template_hash) } + + it 'is valid' do + expect(subject).to be_valid + end + + it 'generates submissions when processed' do + subject.process + expect(number_submissions_created).to eq(1) + end + end end end From 7058a7ca1c6edebc2d3fc4d82feb2f152b9d6098 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 5 Nov 2024 15:49:02 +0000 Subject: [PATCH 021/115] Adding number of pools to the UI --- app/models/pbmc_pooling_customer_request.rb | 2 +- config/locales/metadata/en.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/models/pbmc_pooling_customer_request.rb b/app/models/pbmc_pooling_customer_request.rb index b98639e7e1..f5a594a5cf 100644 --- a/app/models/pbmc_pooling_customer_request.rb +++ b/app/models/pbmc_pooling_customer_request.rb @@ -3,7 +3,7 @@ # A class for customer requests that need the extra metadata fields used for PBMC pooling calculations class PbmcPoolingCustomerRequest < CustomerRequest has_metadata as: Request do - custom_attribute(:number_of_samples_per_pool, integer: true, required: false, default: nil) custom_attribute(:cells_per_chip_well, integer: true, required: false, default: nil) + custom_attribute(:number_of_pools, integer: true, required: false, default: nil) end end diff --git a/config/locales/metadata/en.yml b/config/locales/metadata/en.yml index 2abb785013..fa02c9f98b 100644 --- a/config/locales/metadata/en.yml +++ b/config/locales/metadata/en.yml @@ -61,6 +61,9 @@ en: number_of_samples_per_pool: label: Number of samples per pool + number_of_pools: + label: Number of pools + cells_per_chip_well: label: Cells per chip well From 6b25c81ddf20fadc34397e427742fe305c06af42 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 7 Nov 2024 08:00:15 +0000 Subject: [PATCH 022/115] Removing redundant column validations --- app/models/bulk_submission.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 3182265705..f399c063e4 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -232,8 +232,7 @@ def process # rubocop:todo Metrics/CyclomaticComplexity 'priority', 'flowcell type', 'scrna core number of pools', - 'scrna core cells per chip well', - 'scrna core number of pools' + 'scrna core cells per chip well' ].freeze ALIAS_FIELDS = { 'plate barcode' => 'barcode', 'tube barcode' => 'barcode' }.freeze From 0f93a2daa0f45e2cee1c4719f8e592337691e8d5 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 7 Nov 2024 14:19:36 +0000 Subject: [PATCH 023/115] Refactoring for reviews --- .../submission/validations_by_template_name.rb | 13 +++++++++---- .../conditional_formattings.yml | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 5188f311aa..02a035e495 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -13,6 +13,11 @@ module Submission::ValidationsByTemplateName HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' + SAMPLES_PER_POOL = { + max: 25, + min: 5 + }.freeze + # Applies additional validations based on the submission template type. # # This method determines the submission template type from the CSV data and calls the appropriate @@ -37,7 +42,7 @@ def apply_additional_validations_by_template_name when SCRNA_CORE_CDNA_PREP_GEM_X_5P validate_consistent_column_value(HEADER_NUMBER_OF_POOLS) validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) - validate_samples_per_pool_for_tube_or_plate + validate_samples_per_pool_for_labware end end @@ -80,7 +85,7 @@ def validate_consistent_column_value(column_header) end # rubocop:enable Metrics/MethodLength - def validate_samples_per_pool_for_tube_or_plate + def validate_samples_per_pool_for_labware return if headers.index(HEADER_BARCODE).nil? && headers.index(HEADER_PLATE_WELLS).nil? grouped_rows = group_rows_by_study_and_project @@ -144,7 +149,7 @@ def tube?(barcodes, well_locations) end # Checks if the asset is either a tube or a plate. - def valid_asset?(barcodes, well_locations) + def valid_labware?(barcodes, well_locations) (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) end @@ -156,7 +161,7 @@ def validate_samples_per_pool(rows, total_samples, number_of_pools) number_of_pools.times do |pool_number| samples_per_pool = int_division samples_per_pool += 1 if pool_number < remainder - next unless samples_per_pool > 25 || samples_per_pool < 5 + next unless samples_per_pool > SAMPLES_PER_POOL[:max] || samples_per_pool < SAMPLES_PER_POOL[:min] errors.add( :spreadsheet, diff --git a/config/bulk_submission_excel/conditional_formattings.yml b/config/bulk_submission_excel/conditional_formattings.yml index c72a7bd671..1be93b552d 100644 --- a/config/bulk_submission_excel/conditional_formattings.yml +++ b/config/bulk_submission_excel/conditional_formattings.yml @@ -60,8 +60,8 @@ is_num_of_pools_in_valid_range: options: type: :cellIs operator: :between - formula1: "5" - formula2: "25" + formula1: "2" + formula2: "8" priority: 2 is_num_of_pools_outside_valid_range: style: From 3ddc5b920768b1d1a3b072a6afd96277406388bf Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 7 Nov 2024 14:19:49 +0000 Subject: [PATCH 024/115] Refactoring for reviews --- config/bulk_submission_excel/conditional_formattings.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/bulk_submission_excel/conditional_formattings.yml b/config/bulk_submission_excel/conditional_formattings.yml index 1be93b552d..69dfc9f707 100644 --- a/config/bulk_submission_excel/conditional_formattings.yml +++ b/config/bulk_submission_excel/conditional_formattings.yml @@ -70,8 +70,8 @@ is_num_of_pools_outside_valid_range: options: type: :cellIs operator: :notBetween - formula1: "5" - formula2: "25" + formula1: "2" + formula2: "8" priority: 2 is_num_cells_per_chip_well_in_valid_range: style: From d47250d17988067d4adf3cc67fd87d741b68a2c5 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 7 Nov 2024 14:23:12 +0000 Subject: [PATCH 025/115] Linting --- app/models/submission/validations_by_template_name.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 02a035e495..831c29d3d0 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -13,10 +13,7 @@ module Submission::ValidationsByTemplateName HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' - SAMPLES_PER_POOL = { - max: 25, - min: 5 - }.freeze + SAMPLES_PER_POOL = { max: 25, min: 5 }.freeze # Applies additional validations based on the submission template type. # From c2c5e2065a688be1db29c7c6026facc1c35f5e07 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 14 Nov 2024 13:10:44 +0000 Subject: [PATCH 026/115] Update number of pools validation range in columns.yml --- config/bulk_submission_excel/columns.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/bulk_submission_excel/columns.yml b/config/bulk_submission_excel/columns.yml index c00f5a51e1..127294cc19 100644 --- a/config/bulk_submission_excel/columns.yml +++ b/config/bulk_submission_excel/columns.yml @@ -278,12 +278,12 @@ number_of_pools: options: type: :whole operator: :between - formula1: "2" + formula1: "1" formula2: "8" allowBlank: true showInputMessage: true promptTitle: "Number of pools" - prompt: "The requested number pools (between 2 and 8 inclusive)" + prompt: "The requested number pools (between 1 and 8 inclusive)" conditional_formattings: empty_cell: is_text: From bdd639d8a93293fc2fa196119806704776f3521f Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 26 Nov 2024 14:42:50 +0000 Subject: [PATCH 027/115] Raising exception for invalid labware types --- app/models/submission/validations_by_template_name.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 831c29d3d0..e8fdc9320c 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -99,6 +99,11 @@ def process_rows(rows) validate_for_plates(barcodes, well_locations, rows) elsif tube?(barcodes, well_locations) validate_for_tubes(barcodes, rows) + else + errors.add( + :spreadsheet, + 'Invalid labware type. Please provide either a plate barcode with well locations or tube barcodes only' + ) end end From eaacc488af76953b85e54c5a666039c6e7f4414c Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 26 Nov 2024 14:45:37 +0000 Subject: [PATCH 028/115] [skip ci] Update comment --- app/models/submission/validations_by_template_name.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index e8fdc9320c..1e2ecf284f 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -44,7 +44,7 @@ def apply_additional_validations_by_template_name end def apply_number_of_samples_per_pool_validation - # Creates groups of rows based on the study and project name (pool_number.e., study-project combinations) + # Creates groups of rows based on the study and project name (pool_number, study-project) combinations group_rows_by_study_and_project end From 933afad0f35c664b2073b67285c31399e6fe3f82 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 26 Nov 2024 14:49:00 +0000 Subject: [PATCH 029/115] Fixing rubocop issue --- app/models/submission/validations_by_template_name.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 1e2ecf284f..0df90dce32 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -91,6 +91,7 @@ def validate_samples_per_pool_for_labware private + # rubocop:disable Metrics/MethodLength def process_rows(rows) barcodes = rows.pluck(headers.index(HEADER_BARCODE)) well_locations = rows.pluck(headers.index(HEADER_PLATE_WELLS)) @@ -106,6 +107,7 @@ def process_rows(rows) ) end end + # rubocop:enable Metrics/MethodLength # rubocop:disable Metrics/AbcSize def validate_for_plates(barcodes, well_locations, rows) From b8817452ac055fa6540db1af57a60a14e1fcd8ab Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Tue, 26 Nov 2024 15:58:17 +0000 Subject: [PATCH 030/115] Adding more tests --- .../submission/scRNA_bulk_submission_tube.csv | 4 ++ spec/models/bulk_submission_spec.rb | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 spec/data/submission/scRNA_bulk_submission_tube.csv diff --git a/spec/data/submission/scRNA_bulk_submission_tube.csv b/spec/data/submission/scRNA_bulk_submission_tube.csv new file mode 100644 index 0000000000..5c386c1862 --- /dev/null +++ b/spec/data/submission/scRNA_bulk_submission_tube.csv @@ -0,0 +1,4 @@ +Bulk Submissions Form,,,,,,,,,,,,,,,,,,,,,,, +User Login,Template Name,Project Name,Study Name,Submission name,Barcode,Plate Well,Asset Group Name,Fragment Size From,Fragment Size To,PCR Cycles,Library Type,Bait Library Name,Pre-capture Plex Level,Pre-capture Group,Read Length,Number of lanes,Priority,Primer Panel,Comments,Gigabases Expected,Flowcell Type,scRNA Core Number of pools,scRNA Core Cells per Chip Well +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT12W,,assetgroup,,,,Standard,,,,108,1,,,Sample Comment,1.35,,7,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT13A,,assetgroup2,,,,Standard,,,,108,1,,,Sample Comment,1.35,,7,13000 \ No newline at end of file diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 2d157eb45e..eebf89cac4 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -336,5 +336,43 @@ expect(number_submissions_created).to eq(1) end end + + context 'when an scRNA Bulk Submission for tube' do + # Add another similar tube to the asset group + let(:request_types) { create_list(:sequencing_request_type, 2) } + let!(:tube_barcode) { create(:sanger_ean13_tube, barcode_number: '12') } + let!(:study) { create(:study, name: 'Test Study') } + let!(:tube) { create(:phi_x_stock_tube, barcodes: [tube_barcode]) } + let!(:asset_group) { create(:asset_group, name: 'assetgroup', study: study, assets: [tube.receptacle]) } + let!(:tube_barcode_2) { create(:sanger_ean13_tube, barcode_number: '13') } + let!(:tube_2) { create(:phi_x_stock_tube, barcodes: [tube_barcode_2]) } + let!(:asset_group_2) { create(:asset_group, name: 'assetgroup2', study: study, assets: [tube_2.receptacle]) } + let!(:library_type) { create(:library_type, name: 'Standard') } + + let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube.csv' } + let(:submission_template_hash) do + { + name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + submission_class_name: 'LinearSubmission', + product_catalogue: 'Generic', + submission_parameters: { + request_options: { + }, + request_types: request_types.map(&:key) + } + } + end + + before { SubmissionSerializer.construct!(submission_template_hash) } + + it 'is valid' do + expect(subject).to be_valid + end + + it 'generates submissions when processed' do + subject.process + expect(number_submissions_created).to eq(1) + end + end end end From 682785cf0ca45a08474c81eab483c8cd50ef892d Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 08:44:01 +0000 Subject: [PATCH 031/115] Adding method docs --- .../validations_by_template_name.rb | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 0df90dce32..482c2577de 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -82,6 +82,14 @@ def validate_consistent_column_value(column_header) end # rubocop:enable Metrics/MethodLength + # Validates the number of samples per pool for labware. + # + # This method checks if the headers for barcode and plate wells are present. + # If they are, it groups the rows by study and project, and processes each group. + # The processing involves determining if the labware is a plate or tube and + # validating the number of samples per pool accordingly. + # + # @return [void] def validate_samples_per_pool_for_labware return if headers.index(HEADER_BARCODE).nil? && headers.index(HEADER_PLATE_WELLS).nil? @@ -91,6 +99,13 @@ def validate_samples_per_pool_for_labware private + # Processes the rows to determine the type of labware and validate accordingly. + # + # This method extracts the barcodes and well locations from the rows and determines if the labware is a plate or tube. + # It then calls the appropriate validation method based on the labware type. + # + # @param rows [Array>] The rows of CSV data to process. + # @return [void] # rubocop:disable Metrics/MethodLength def process_rows(rows) barcodes = rows.pluck(headers.index(HEADER_BARCODE)) @@ -109,6 +124,17 @@ def process_rows(rows) end # rubocop:enable Metrics/MethodLength + # Validates the number of samples per pool for plates. + # + # This method finds the plate using the provided barcodes and retrieves the wells located at the specified well + # locations. + # It then calculates the total number of samples per study and project and the number of pools. + # Finally, it validates the number of samples per pool. + # + # @param barcodes [Array] The barcodes of the plates. + # @param well_locations [Array] The well locations on the plate. + # @param rows [Array>] The rows of CSV data to process. + # @return [void] # rubocop:disable Metrics/AbcSize def validate_for_plates(barcodes, well_locations, rows) plate = Plate.find_from_any_barcode(barcodes.uniq.first) @@ -144,10 +170,26 @@ def validate_for_tubes(barcodes, rows) # rubocop:enable Metrics/AbcSize # rubocop:enable Metrics/MethodLength + # Determines if the labware is a plate based on the presence of barcodes and well locations. + # + # This method checks if both barcodes and well locations are present to determine if the labware is a plate. + # + # @param barcodes [Array] The barcodes of the labware. + # @param well_locations [Array] The well locations on the labware. + # @return [Boolean] Returns true if both barcodes and well locations are present, indicating the labware is a plate. def plate?(barcodes, well_locations) barcodes.present? && well_locations.present? end + # Determines if the labware is a tube based on the presence of barcodes and absence of well locations. + # + # This method checks if barcodes are present and well locations are absent to determine if the labware is a tube. + # + # @param barcodes [Array] The barcodes of the labware. + # @param well_locations [Array] The well locations on the labware. + # @return [Boolean] Returns true if barcodes are present and well locations are absent, indicating the labware is a + # tube. + def tube?(barcodes, well_locations) barcodes.present? && well_locations.blank? end @@ -157,6 +199,17 @@ def valid_labware?(barcodes, well_locations) (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) end + # Validates the number of samples per pool. + # + # This method calculates the number of samples per pool by dividing the total number of samples by the number of + # pools. + # It then iterates through each pool and checks if the number of samples per pool is within the allowed range. + # If the number of samples per pool is less than the minimum or greater than the maximum allowed, an error is added. + # + # @param rows [Array>] The rows of CSV data to process. + # @param total_samples [Integer] The total number of samples. + # @param number_of_pools [Integer] The number of pools. + # @return [void] # rubocop:disable Metrics/MethodLength def validate_samples_per_pool(rows, total_samples, number_of_pools) int_division = total_samples / number_of_pools From dfce47972010da0f626f87fa11b15432d921777f Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 08:48:03 +0000 Subject: [PATCH 032/115] Refactor validation logic --- .../validations_by_template_name.rb | 77 +++++++++++++++---- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 482c2577de..dad2e0950b 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -148,27 +148,70 @@ def validate_for_plates(barcodes, well_locations, rows) end # rubocop:enable Metrics/AbcSize - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/MethodLength + # Validates the number of samples per pool for tubes. + # + # This method finds the tubes using the provided barcodes and calculates the total number of samples per study and + # project. + # It then retrieves the number of pools from the rows and validates the number of samples per pool. + # + # @param barcodes [Array] The barcodes of the tubes. + # @param rows [Array>] The rows of CSV data to process. + # @return [void] def validate_for_tubes(barcodes, rows) - tubes = - Receptacle - .on_a(Tube) - .for_bulk_submission - .with_barcode(barcodes) - .tap do |found| - missing = details['barcode'].reject { |barcode| found.any? { |tube| tube.any_barcode_matching?(barcode) } } - if missing.present? - raise ActiveRecord::RecordNotFound, "Could not find Tubes with barcodes #{missing.inspect}" - end - end - total_number_of_samples_per_study_project = tubes.map(&:samples).flatten.count.to_i - number_of_pools = rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + tubes = find_tubes(barcodes) + total_number_of_samples_per_study_project = calculate_total_samples(tubes) + number_of_pools = extract_number_of_pools(rows) validate_samples_per_pool(rows, total_number_of_samples_per_study_project, number_of_pools) end - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/MethodLength + + # Finds the tubes using the provided barcodes. + # + # This method retrieves the tubes that match the provided barcodes and raises an error if any barcodes are missing. + # + # @param barcodes [Array] The barcodes of the tubes. + # @return [Array] The found tubes. + def find_tubes(barcodes) + Receptacle + .on_a(Tube) + .for_bulk_submission + .with_barcode(barcodes) + .tap do |found| + missing = find_missing_barcodes(barcodes, found) + raise ActiveRecord::RecordNotFound, "Could not find Tubes with barcodes #{missing.inspect}" if missing.present? + end + end + + # Finds the missing barcodes from the found tubes. + # + # This method checks which barcodes are not present in the found tubes. + # + # @param barcodes [Array] The barcodes of the tubes. + # @param found [Array] The found tubes. + # @return [Array] The missing barcodes. + def find_missing_barcodes(barcodes, found) + barcodes.reject { |barcode| found.any? { |tube| tube.any_barcode_matching?(barcode) } } + end + + # Calculates the total number of samples from the tubes. + # + # This method calculates the total number of samples by flattening the samples from the tubes and counting them. + # + # @param tubes [Array] The tubes to calculate samples from. + # @return [Integer] The total number of samples. + def calculate_total_samples(tubes) + tubes.map(&:samples).flatten.count.to_i + end + + # Extracts the number of pools from the rows. + # + # This method retrieves the number of pools from the specified column in the rows. + # + # @param rows [Array>] The rows of CSV data to process. + # @return [Integer] The number of pools. + def extract_number_of_pools(rows) + rows.pluck(headers.index(HEADER_NUMBER_OF_POOLS)).uniq.first.to_i + end # Determines if the labware is a plate based on the presence of barcodes and well locations. # From 8c75444c5060bf8f47527526f2339fe7a1aa690d Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 11:13:42 +0000 Subject: [PATCH 033/115] Adding tests for tubes --- .../validations_by_template_name.rb | 9 ++------ .../conditional_formattings.yml | 4 ++-- .../submission/scRNA_bulk_submission_tube.csv | 8 +++++-- spec/models/bulk_submission_spec.rb | 22 ++++++++++++------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index dad2e0950b..199b4d67a3 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -221,7 +221,7 @@ def extract_number_of_pools(rows) # @param well_locations [Array] The well locations on the labware. # @return [Boolean] Returns true if both barcodes and well locations are present, indicating the labware is a plate. def plate?(barcodes, well_locations) - barcodes.present? && well_locations.present? + barcodes.present? && well_locations.none?(&:nil?) end # Determines if the labware is a tube based on the presence of barcodes and absence of well locations. @@ -234,12 +234,7 @@ def plate?(barcodes, well_locations) # tube. def tube?(barcodes, well_locations) - barcodes.present? && well_locations.blank? - end - - # Checks if the asset is either a tube or a plate. - def valid_labware?(barcodes, well_locations) - (barcodes.present? && well_locations.present?) || (barcodes.present? && well_locations.blank?) + barcodes.present? && well_locations.all?(&:nil?) end # Validates the number of samples per pool. diff --git a/config/bulk_submission_excel/conditional_formattings.yml b/config/bulk_submission_excel/conditional_formattings.yml index 69dfc9f707..c97a0bacca 100644 --- a/config/bulk_submission_excel/conditional_formattings.yml +++ b/config/bulk_submission_excel/conditional_formattings.yml @@ -60,7 +60,7 @@ is_num_of_pools_in_valid_range: options: type: :cellIs operator: :between - formula1: "2" + formula1: "1" formula2: "8" priority: 2 is_num_of_pools_outside_valid_range: @@ -70,7 +70,7 @@ is_num_of_pools_outside_valid_range: options: type: :cellIs operator: :notBetween - formula1: "2" + formula1: "1" formula2: "8" priority: 2 is_num_cells_per_chip_well_in_valid_range: diff --git a/spec/data/submission/scRNA_bulk_submission_tube.csv b/spec/data/submission/scRNA_bulk_submission_tube.csv index 5c386c1862..e30bdfd196 100644 --- a/spec/data/submission/scRNA_bulk_submission_tube.csv +++ b/spec/data/submission/scRNA_bulk_submission_tube.csv @@ -1,4 +1,8 @@ Bulk Submissions Form,,,,,,,,,,,,,,,,,,,,,,, User Login,Template Name,Project Name,Study Name,Submission name,Barcode,Plate Well,Asset Group Name,Fragment Size From,Fragment Size To,PCR Cycles,Library Type,Bait Library Name,Pre-capture Plex Level,Pre-capture Group,Read Length,Number of lanes,Priority,Primer Panel,Comments,Gigabases Expected,Flowcell Type,scRNA Core Number of pools,scRNA Core Cells per Chip Well -user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT12W,,assetgroup,,,,Standard,,,,108,1,,,Sample Comment,1.35,,7,13000 -user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT13A,,assetgroup2,,,,Standard,,,,108,1,,,Sample Comment,1.35,,7,13000 \ No newline at end of file +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT1,,ag1,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT2,,ag2,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT3,,ag3,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT4,,ag4,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT5,,ag5,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT6,,ag6,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 \ No newline at end of file diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index eebf89cac4..107cb1c848 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -337,16 +337,17 @@ end end - context 'when an scRNA Bulk Submission for tube' do + context 'when an scRNA Bulk Submission for tubes' do # Add another similar tube to the asset group let(:request_types) { create_list(:sequencing_request_type, 2) } - let!(:tube_barcode) { create(:sanger_ean13_tube, barcode_number: '12') } + # Create a list of tubes with samples + + let!(:tubes) do + create_list(:phi_x_stock_tube, 6) do |tube, i| + tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 100}") + end + end let!(:study) { create(:study, name: 'Test Study') } - let!(:tube) { create(:phi_x_stock_tube, barcodes: [tube_barcode]) } - let!(:asset_group) { create(:asset_group, name: 'assetgroup', study: study, assets: [tube.receptacle]) } - let!(:tube_barcode_2) { create(:sanger_ean13_tube, barcode_number: '13') } - let!(:tube_2) { create(:phi_x_stock_tube, barcodes: [tube_barcode_2]) } - let!(:asset_group_2) { create(:asset_group, name: 'assetgroup2', study: study, assets: [tube_2.receptacle]) } let!(:library_type) { create(:library_type, name: 'Standard') } let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube.csv' } @@ -363,7 +364,12 @@ } end - before { SubmissionSerializer.construct!(submission_template_hash) } + before do + SubmissionSerializer.construct!(submission_template_hash) + tubes.each_with_index.map do |tube, i| + create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) + end + end it 'is valid' do expect(subject).to be_valid From ff64214fe2402a1401cef667b46fa3e9bc40cc36 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 13:23:01 +0000 Subject: [PATCH 034/115] Adding a test for invalid scenario --- .../validations_by_template_name.rb | 1 - .../scRNA_bulk_submission_tube_invalid.csv | 6 +++ spec/models/bulk_submission_spec.rb | 41 ++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 spec/data/submission/scRNA_bulk_submission_tube_invalid.csv diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 199b4d67a3..aeff6aa438 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -232,7 +232,6 @@ def plate?(barcodes, well_locations) # @param well_locations [Array] The well locations on the labware. # @return [Boolean] Returns true if barcodes are present and well locations are absent, indicating the labware is a # tube. - def tube?(barcodes, well_locations) barcodes.present? && well_locations.all?(&:nil?) end diff --git a/spec/data/submission/scRNA_bulk_submission_tube_invalid.csv b/spec/data/submission/scRNA_bulk_submission_tube_invalid.csv new file mode 100644 index 0000000000..7fe6d2a692 --- /dev/null +++ b/spec/data/submission/scRNA_bulk_submission_tube_invalid.csv @@ -0,0 +1,6 @@ +Bulk Submissions Form,,,,,,,,,,,,,,,,,,,,,,, +User Login,Template Name,Project Name,Study Name,Submission name,Barcode,Plate Well,Asset Group Name,Fragment Size From,Fragment Size To,PCR Cycles,Library Type,Bait Library Name,Pre-capture Plex Level,Pre-capture Group,Read Length,Number of lanes,Priority,Primer Panel,Comments,Gigabases Expected,Flowcell Type,scRNA Core Number of pools,scRNA Core Cells per Chip Well +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT1,,ag1,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT2,,ag2,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT3,,ag3,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT4,,ag4,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 \ No newline at end of file diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 107cb1c848..37ffb3058b 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -344,7 +344,7 @@ let!(:tubes) do create_list(:phi_x_stock_tube, 6) do |tube, i| - tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 100}") + tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") end end let!(:study) { create(:study, name: 'Test Study') } @@ -380,5 +380,44 @@ expect(number_submissions_created).to eq(1) end end + + context 'when an scRNA Bulk Submission for tubes with incorrect number of samples per pool' do + # Add another similar tube to the asset group + let(:request_types) { create_list(:sequencing_request_type, 2) } + # Create a list of tubes with samples + + let!(:tubes) do + create_list(:phi_x_stock_tube, 6) do |tube, i| + tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") + end + end + let!(:study) { create(:study, name: 'Test Study') } + let!(:library_type) { create(:library_type, name: 'Standard') } + + let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube_invalid.csv' } + let(:submission_template_hash) do + { + name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + submission_class_name: 'LinearSubmission', + product_catalogue: 'Generic', + submission_parameters: { + request_options: { + }, + request_types: request_types.map(&:key) + } + } + end + + before do + SubmissionSerializer.construct!(submission_template_hash) + tubes.each_with_index.map do |tube, i| + create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) + end + end + + it 'is invalid' do + expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) + end + end end end From fea780fc50cf81e42124ceffd8d15692e990cc82 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 13:25:40 +0000 Subject: [PATCH 035/115] [skip ci] refactoring some comments in tests --- spec/models/bulk_submission_spec.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index 37ffb3058b..b0474da764 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -338,10 +338,8 @@ end context 'when an scRNA Bulk Submission for tubes' do - # Add another similar tube to the asset group let(:request_types) { create_list(:sequencing_request_type, 2) } # Create a list of tubes with samples - let!(:tubes) do create_list(:phi_x_stock_tube, 6) do |tube, i| tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") @@ -382,10 +380,8 @@ end context 'when an scRNA Bulk Submission for tubes with incorrect number of samples per pool' do - # Add another similar tube to the asset group let(:request_types) { create_list(:sequencing_request_type, 2) } # Create a list of tubes with samples - let!(:tubes) do create_list(:phi_x_stock_tube, 6) do |tube, i| tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") From 33750013865b6eb4646d530b589acd0bed265329 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 13:40:49 +0000 Subject: [PATCH 036/115] Updating tests to add shared examples for invalid scenarios --- ...A_bulk_submission_tube_invalid_greater.csv | 34 +++++++++ spec/models/bulk_submission_spec.rb | 72 +++++++++++-------- 2 files changed, 76 insertions(+), 30 deletions(-) create mode 100644 spec/data/submission/scRNA_bulk_submission_tube_invalid_greater.csv diff --git a/spec/data/submission/scRNA_bulk_submission_tube_invalid_greater.csv b/spec/data/submission/scRNA_bulk_submission_tube_invalid_greater.csv new file mode 100644 index 0000000000..3f1a36aef5 --- /dev/null +++ b/spec/data/submission/scRNA_bulk_submission_tube_invalid_greater.csv @@ -0,0 +1,34 @@ +Bulk Submissions Form,,,,,,,,,,,,,,,,,,,,,,, +User Login,Template Name,Project Name,Study Name,Submission name,Barcode,Plate Well,Asset Group Name,Fragment Size From,Fragment Size To,PCR Cycles,Library Type,Bait Library Name,Pre-capture Plex Level,Pre-capture Group,Read Length,Number of lanes,Priority,Primer Panel,Comments,Gigabases Expected,Flowcell Type,scRNA Core Number of pools,scRNA Core Cells per Chip Well +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT1,,ag1,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT2,,ag2,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT3,,ag3,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT4,,ag4,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT5,,ag5,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT6,,ag6,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT7,,ag7,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT8,,ag8,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT9,,ag9,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT10,,ag10,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT11,,ag11,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT12,,ag12,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT13,,ag13,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT14,,ag14,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT15,,ag15,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT16,,ag16,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT17,,ag17,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT18,,ag18,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT19,,ag19,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT20,,ag20,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT21,,ag21,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT22,,ag22,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT23,,ag23,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT24,,ag24,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT25,,ag25,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT26,,ag26,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT27,,ag27,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT28,,ag28,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT29,,ag29,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT30,,ag30,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT31,,ag31,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 +user,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Test Project,Test Study,sub1,NT32,,ag32,,,,Standard,,,,108,1,,,Sample Comment,1.35,,1,13000 \ No newline at end of file diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index b0474da764..fe89c40de9 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -2,6 +2,41 @@ require 'rails_helper' +shared_examples 'an invalid scRNA Bulk Submission for tubes' do |_, tube_count| + let(:request_types) { create_list(:sequencing_request_type, 2) } + let!(:tubes) do + create_list(:phi_x_stock_tube, tube_count) do |tube, i| + tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") + end + end + let!(:study) { create(:study, name: 'Test Study') } + let!(:library_type) { create(:library_type, name: 'Standard') } + + let(:submission_template_hash) do + { + name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', + submission_class_name: 'LinearSubmission', + product_catalogue: 'Generic', + submission_parameters: { + request_options: { + }, + request_types: request_types.map(&:key) + } + } + end + + before do + SubmissionSerializer.construct!(submission_template_hash) + tubes.each_with_index.map do |tube, i| + create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) + end + end + + it 'is invalid' do + expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) + end +end + describe BulkSubmission, with: :uploader do subject { described_class.new(spreadsheet: submission_file, encoding: encoding) } @@ -379,40 +414,17 @@ end end - context 'when an scRNA Bulk Submission for tubes with incorrect number of samples per pool' do - let(:request_types) { create_list(:sequencing_request_type, 2) } - # Create a list of tubes with samples - let!(:tubes) do - create_list(:phi_x_stock_tube, 6) do |tube, i| - tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") - end - end - let!(:study) { create(:study, name: 'Test Study') } - let!(:library_type) { create(:library_type, name: 'Standard') } + context 'when an scRNA Bulk Submission given with invalid number of samples per pool' do + context 'number of samples per pool < 5' do + let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube_invalid.csv' } - let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube_invalid.csv' } - let(:submission_template_hash) do - { - name: 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p', - submission_class_name: 'LinearSubmission', - product_catalogue: 'Generic', - submission_parameters: { - request_options: { - }, - request_types: request_types.map(&:key) - } - } + include_examples 'an invalid scRNA Bulk Submission for tubes', 'scRNA_bulk_submission_tube_invalid', 4 end - before do - SubmissionSerializer.construct!(submission_template_hash) - tubes.each_with_index.map do |tube, i| - create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) - end - end + context 'number of samples per pool > 25' do + let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube_invalid_greater.csv' } - it 'is invalid' do - expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) + include_examples 'an invalid scRNA Bulk Submission for tubes', 'scRNA_bulk_submission_tube_invalid_greater', 32 end end end From d58bd71840bb80bb252e7d95e9a768d3bec8d8d1 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 13:41:13 +0000 Subject: [PATCH 037/115] [skip ci] Updating tests to add shared examples for invalid scenarios --- spec/models/bulk_submission_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index fe89c40de9..f8edb21428 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' -shared_examples 'an invalid scRNA Bulk Submission for tubes' do |_, tube_count| +shared_examples 'an invalid scRNA Bulk Submission' do |_, tube_count| let(:request_types) { create_list(:sequencing_request_type, 2) } let!(:tubes) do create_list(:phi_x_stock_tube, tube_count) do |tube, i| From e409e5e1d5be7a0c9f5f0ea62dfa6fb8b85ac1dd Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Wed, 27 Nov 2024 13:42:08 +0000 Subject: [PATCH 038/115] Changing shared example names --- spec/models/bulk_submission_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index f8edb21428..db3c5ea948 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -418,13 +418,13 @@ context 'number of samples per pool < 5' do let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube_invalid.csv' } - include_examples 'an invalid scRNA Bulk Submission for tubes', 'scRNA_bulk_submission_tube_invalid', 4 + include_examples 'an invalid scRNA Bulk Submission', 'scRNA_bulk_submission_tube_invalid', 4 end context 'number of samples per pool > 25' do let(:spreadsheet_filename) { 'scRNA_bulk_submission_tube_invalid_greater.csv' } - include_examples 'an invalid scRNA Bulk Submission for tubes', 'scRNA_bulk_submission_tube_invalid_greater', 32 + include_examples 'an invalid scRNA Bulk Submission', 'scRNA_bulk_submission_tube_invalid_greater', 32 end end end From e44c0ced4013dcfcf9a6c88c856d0f861a717549 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 28 Nov 2024 09:04:09 +0000 Subject: [PATCH 039/115] Refactoring long methods --- .../validations_by_template_name.rb | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index aeff6aa438..1f6069194b 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -62,25 +62,14 @@ def group_rows_by_study_and_project # # @param column_header [String] The header of the column to validate. # @return [void] - # rubocop:disable Metrics/MethodLength def validate_consistent_column_value(column_header) index_of_column = headers.index(column_header) - grouped_rows = group_rows_by_study_and_project grouped_rows.each do |study_project, rows| - unique_values = rows.pluck(index_of_column).uniq - - next unless unique_values.size > 1 - errors.add( - :spreadsheet, - "Inconsistent values for column '#{column_header}' for Study name '#{study_project[0]}' and Project name " \ - "'#{study_project[1]}', " \ - 'all rows for a specific study and project must have the same value' - ) + validate_unique_values(study_project, rows, index_of_column, column_header) end end - # rubocop:enable Metrics/MethodLength # Validates the number of samples per pool for labware. # @@ -99,6 +88,39 @@ def validate_samples_per_pool_for_labware private + # Validates that the specified column has unique values for each study and project. + # + # This method checks if the specified column has unique values for each study and project. + # If inconsistencies are found, an error is added to the errors collection. + # + # @param study_project [Array] The study and project names. + # @param rows [Array>] The rows of CSV data to process. + # @param index_of_column [Integer] The index of the column to validate. + # @param column_header [String] The header of the column to validate. + # @return [void] + def validate_unique_values(study_project, rows, index_of_column, column_header) + unique_values = rows.pluck(index_of_column).uniq + return unless unique_values.size > 1 + + add_inconsistent_value_error(study_project, column_header) + end + + # Adds an error for inconsistent column values. + # + # This method adds an error to the errors collection for inconsistent column values + # for the specified study and project. + # + # @param study_project [Array] The study and project names. + # @param column_header [String] The header of the column with inconsistent values. + # @return [void] + def add_inconsistent_value_error(study_project, column_header) + errors.add( + :spreadsheet, + "Inconsistent values for column '#{column_header}' for Study name '#{study_project[0]}' and Project name " \ + "'#{study_project[1]}', all rows for a specific study and project must have the same value" + ) + end + # Processes the rows to determine the type of labware and validate accordingly. # # This method extracts the barcodes and well locations from the rows and determines if the labware is a plate or tube. From 9ef7c10e55fc8a279de6a33bba58b3175098e489 Mon Sep 17 00:00:00 2001 From: Dasun Pubudumal Date: Thu, 28 Nov 2024 09:06:34 +0000 Subject: [PATCH 040/115] Refactoring long methods --- .../submission/validations_by_template_name.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 1f6069194b..f99c48bca7 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -102,18 +102,6 @@ def validate_unique_values(study_project, rows, index_of_column, column_header) unique_values = rows.pluck(index_of_column).uniq return unless unique_values.size > 1 - add_inconsistent_value_error(study_project, column_header) - end - - # Adds an error for inconsistent column values. - # - # This method adds an error to the errors collection for inconsistent column values - # for the specified study and project. - # - # @param study_project [Array] The study and project names. - # @param column_header [String] The header of the column with inconsistent values. - # @return [void] - def add_inconsistent_value_error(study_project, column_header) errors.add( :spreadsheet, "Inconsistent values for column '#{column_header}' for Study name '#{study_project[0]}' and Project name " \ From 5bc22bf9a04afc1b02e0285477ff3d048a60084f Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Tue, 3 Dec 2024 14:19:48 +0000 Subject: [PATCH 041/115] feat(studies): Updates new/edit study questions --- app/models/study.rb | 31 +++++++++++++++---- .../shared/metadata/edit/_study.html.erb | 2 +- config/locales/metadata/en.yml | 7 ++--- ...eeds_to_be_reverted_to_old_version.feature | 2 +- .../8447221_data_release_help_text.feature | 9 ++---- features/studies/data_release_timings.feature | 10 +++--- spec/features/studies/create_study_spec.rb | 6 ++-- spec/features/studies/edit_study_spec.rb | 4 +-- spec/features/studies/manage_study_spec.rb | 4 +-- .../studies/view_study_properties_spec.rb | 2 +- 10 files changed, 46 insertions(+), 31 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index ebfd589446..0a291fba0c 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -82,11 +82,25 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength DATA_RELEASE_TIMING_IMMEDIATE, DATA_RELEASE_TIMING_DELAYED ].freeze - DATA_RELEASE_PREVENTION_REASONS = ['data validity', 'legal', 'replication of data subset'].freeze + + OLD_DATA_RELEASE_PREVENTION_REASONS = ['data validity', 'legal', 'replication of data subset'].freeze + DATA_RELEASE_PREVENTION_REASONS = [ + 'Pilot or validation studies - DAC approval not required', + 'Collaborators will share data in a research repository - DAC approval not required', + 'Prevent harm (e.g sensitive studies or biosecurity) - DAC approval required', + 'Protecting IP - DAC approval required', + 'Other (please specify)' + ].freeze DATA_RELEASE_DELAY_FOR_OTHER = 'other' - DATA_RELEASE_DELAY_REASONS_STANDARD = ['phd study', DATA_RELEASE_DELAY_FOR_OTHER].freeze DATA_RELEASE_DELAY_REASONS_ASSAY = ['phd study', 'assay of no other use', DATA_RELEASE_DELAY_FOR_OTHER].freeze + DATA_RELEASE_DELAY_REASONS_STANDARD = [ + 'PhD study', + 'Capacity building', + 'Intellectual property protection', + 'Additional time to make data FAIR', + DATA_RELEASE_DELAY_FOR_OTHER + ].freeze DATA_RELEASE_DELAY_PERIODS = ['3 months', '6 months', '9 months', '12 months', '18 months'].freeze @@ -217,6 +231,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength in: DATA_RELEASE_DELAY_REASONS_ASSAY, if: :delayed_release? ) + custom_attribute(:data_release_delay_period, required: true, in: DATA_RELEASE_DELAY_PERIODS, if: :delayed_release?) custom_attribute(:bam, default: true) @@ -235,10 +250,14 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute(:data_release_delay_approval, in: YES_OR_NO, default: NO) end - with_options(if: :never_release?, required: true) do - custom_attribute(:data_release_prevention_reason, in: DATA_RELEASE_PREVENTION_REASONS) - custom_attribute(:data_release_prevention_approval, in: YES_OR_NO) - custom_attribute(:data_release_prevention_reason_comment) + with_options(if: :never_release?) do + custom_attribute( + :data_release_prevention_reason, + in: DATA_RELEASE_PREVENTION_REASONS + OLD_DATA_RELEASE_PREVENTION_REASONS, + required: true + ) + custom_attribute(:data_release_prevention_reason_comment, required: true) + custom_attribute(:data_release_prevention_approval) end # NOTE: Additional validation in Study::Metadata Class to validate_presence_of :data_access_group, if: :managed diff --git a/app/views/shared/metadata/edit/_study.html.erb b/app/views/shared/metadata/edit/_study.html.erb index 4f16c90e52..aa1e599762 100644 --- a/app/views/shared/metadata/edit/_study.html.erb +++ b/app/views/shared/metadata/edit/_study.html.erb @@ -60,7 +60,7 @@ <% metadata_fields.related_fields(to: :data_release_strategy, when: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> <%= group.select(:data_release_prevention_reason, Study::DATA_RELEASE_PREVENTION_REASONS) %> - <%= group.radio_select(:data_release_prevention_approval, Study::YES_OR_NO) %> + <%= group.text_area(:data_release_prevention_approval) %> <%= group.text_area(:data_release_prevention_reason_comment) %> <% end %> <% metadata_fields.related_fields(to: :data_release_strategy, in: Study::DATA_RELEASE_STRATEGIES, not: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> diff --git a/config/locales/metadata/en.yml b/config/locales/metadata/en.yml index 2abb785013..85e636230a 100644 --- a/config/locales/metadata/en.yml +++ b/config/locales/metadata/en.yml @@ -422,7 +422,7 @@ en: values: open: "Open (ENA)" managed: "Managed (EGA)" - not_applicable: "Not Applicable (Contact Datasharing)" + not_applicable: "Not Applicable" data_release_standard_agreement: label: "Will you be using WTSI's standard access agreement?" @@ -432,13 +432,13 @@ en: data_release_timing: label: How is the data release to be timed? - help: "Choose from:

Immediate: To be released as soon as possible.

Standard: To be released:

  • For managed (EGA) studies: 6 months
  • For open (ENA) studies: 12 months
  • Transcriptomics studies: on request only

Delayed:

  • For managed (EGA) studies: 6 months plus delay for period
  • For open (ENA) studies: 12 months plus delay for period.

Never: This option is only available if the data release strategy is set to 'Not applicable.'

" + help: "Choose from:

Immediate: To be released as soon as possible.

Standard: To be released:

  • For managed (EGA) studies: 12 months
  • For open (ENA) studies: 12 months
  • Transcriptomics studies: 12 months

Delayed:

  • For managed (EGA) studies: 12 months plus delay for period
  • For open (ENA) studies: 12 months plus delay for period.

Never: This option is only available if the data release strategy is set to 'Not applicable.'

" data_release_prevention_reason: label: What is the reason for preventing data release? data_release_prevention_approval: - label: Has this been approved? + label: If reason for exemption requires DAC approval, what is the approval number? help: "If this is for data validity reasons: approval from the sponsor is required
If this is for legal reasons: approval from the Data Sharing Committee is required (please contact sd4)
" data_release_prevention_reason_comment: @@ -446,7 +446,6 @@ en: data_release_delay_reason: label: Reason for delaying release - help: 'To apply for a delay, please contact <%= configatron.data_sharing_contact.email %>' data_release_delay_other_comment: label: Please explain the reason for delaying release diff --git a/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature b/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature index 331c614d46..59e640bd67 100644 --- a/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature +++ b/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature @@ -115,7 +115,7 @@ Feature: The XML for the sequencescape API Not specified - Has this been approved? + If reason for exemption requires DAC approval, what is the approval number? diff --git a/features/studies/8447221_data_release_help_text.feature b/features/studies/8447221_data_release_help_text.feature index a245f126d6..0b20d4bab8 100644 --- a/features/studies/8447221_data_release_help_text.feature +++ b/features/studies/8447221_data_release_help_text.feature @@ -23,10 +23,7 @@ Feature: Update the data release fields for creating a study When I choose "" from "What is the data release strategy for this study?" When I select "delayed" from "How is the data release to be timed?" When I select "other" from "Reason for delaying release" - Then the help text for "Reason for delaying release" should contain: - """ - To apply for a delay, please contact datasharing@example.com - """ + Then I should exactly see "Reason for delaying release" Examples: | release strategy | @@ -34,9 +31,9 @@ Feature: Update the data release fields for creating a study | Open (ENA) | Scenario: Add help text to has this been approved for never release (4044343) - When I choose "Not Applicable (Contact Datasharing)" from "What is the data release strategy for this study?" + When I choose "Not Applicable" from "What is the data release strategy for this study?" When I select "never" from "How is the data release to be timed?" - Then the help text for "Has this been approved?" should contain: + Then the help text for "If reason for exemption requires DAC approval, what is the approval number?" should contain: """ If this is for data validity reasons: approval from the sponsor is required If this is for legal reasons: approval from the Data Sharing Committee is required (please contact sd4) diff --git a/features/studies/data_release_timings.feature b/features/studies/data_release_timings.feature index c8e210b365..15ba11a340 100644 --- a/features/studies/data_release_timings.feature +++ b/features/studies/data_release_timings.feature @@ -28,7 +28,7 @@ Feature: Studies have timings for release of their data Scenario: When the data release is delayed for PhD study Given I select "delayed" from "How is the data release to be timed?" - And I select "phd study" from "Reason for delaying release" + And I select "PhD study" from "Reason for delaying release" Then the "Comment regarding data release timing and approval" field is hidden When I select "6 months" from "Delay for" And I press "Create" @@ -70,19 +70,19 @@ Feature: Studies have timings for release of their data | 12 months | Scenario: When the data release is never but the comment is not supplied - When I choose "Not Applicable (Contact Datasharing)" from "What is the data release strategy for this study?" + When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" - And I choose "Yes" from "Has this been approved?" + And I fill in "12345" from "If reason for exemption requires DAC approval, what is the approval number?" When I press "Create" Then I should be on the studies page # Again, ideally without study metadata And I should see "Study metadata data release prevention reason comment can't be blank" Scenario: When the data release is never and the comment is supplied - When I choose "Not Applicable (Contact Datasharing)" from "What is the data release strategy for this study?" + When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" And I fill in "Comment regarding prevention of data release and approval" with "Some reason" - And I choose "Yes" from "Has this been approved?" + And I fill in "12345" from "If reason for exemption requires DAC approval, what is the approval number? When I press "Create" Then I should be on the study information page for "Testing data release strategies" And I should see "Your study has been created" diff --git a/spec/features/studies/create_study_spec.rb b/spec/features/studies/create_study_spec.rb index 4f63079c5b..e4545f5182 100644 --- a/spec/features/studies/create_study_spec.rb +++ b/spec/features/studies/create_study_spec.rb @@ -40,7 +40,7 @@ within_fieldset('What is the data release strategy for this study?') do expect(page).to have_field('Open (ENA)', type: :radio) expect(page).to have_field('Managed (EGA)', type: :radio) - expect(page).to have_field('Not Applicable (Contact Datasharing)', type: :radio) + expect(page).to have_field('Not Applicable', type: :radio) end within_fieldset('Study Visibility') do @@ -132,8 +132,8 @@ expect(page).to have_field('HuMFre approval number', type: :text) end - it 'displays HuMFre approval number when Not Applicable (Contact Datasharing) is clicked' do - choose('Not Applicable (Contact Datasharing)', allow_label_click: true) + it 'displays HuMFre approval number when Not Applicable is clicked' do + choose('Not Applicable', allow_label_click: true) expect(page).to have_field('HuMFre approval number', type: :text) end end diff --git a/spec/features/studies/edit_study_spec.rb b/spec/features/studies/edit_study_spec.rb index c8a29b8138..a21ac64c62 100644 --- a/spec/features/studies/edit_study_spec.rb +++ b/spec/features/studies/edit_study_spec.rb @@ -69,8 +69,8 @@ expect(page).to have_field('HuMFre approval number', type: :text) end - it 'displays HuMFre approval number when Not Applicable (Contact Datasharing) is clicked' do - choose('Not Applicable (Contact Datasharing)', allow_label_click: true) + it 'displays HuMFre approval number when Not Applicable is clicked' do + choose('Not Applicable', allow_label_click: true) expect(page).to have_field('HuMFre approval number', type: :text) end end diff --git a/spec/features/studies/manage_study_spec.rb b/spec/features/studies/manage_study_spec.rb index a6a1a094c3..a6e4b6f5de 100644 --- a/spec/features/studies/manage_study_spec.rb +++ b/spec/features/studies/manage_study_spec.rb @@ -34,8 +34,8 @@ expect(page).to have_field('HuMFre approval number', type: :text) end - it 'displays HuMFre approval number when Not Applicable (Contact Datasharing) is clicked' do - choose('Not Applicable (Contact Datasharing)', allow_label_click: true) + it 'displays HuMFre approval number when Not Applicable is clicked' do + choose('Not Applicable', allow_label_click: true) expect(page).to have_field('HuMFre approval number', type: :text) end end diff --git a/spec/features/studies/view_study_properties_spec.rb b/spec/features/studies/view_study_properties_spec.rb index a3359f72ac..b64305bacd 100644 --- a/spec/features/studies/view_study_properties_spec.rb +++ b/spec/features/studies/view_study_properties_spec.rb @@ -52,7 +52,7 @@ expect(page).to have_content('HuMFre approval number: 12345') end - it 'displays HuMFre approval number for Not Applicable (Contact Datasharing) data release strategy' do + it 'displays HuMFre approval number for Not Applicable data release strategy' do study.study_metadata.data_release_strategy = Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE study.study_metadata.data_release_timing = Study::DATA_RELEASE_TIMING_NEVER study.study_metadata.data_release_prevention_reason = Study::DATA_RELEASE_PREVENTION_REASONS[0] From 9f6cbcd84154d45e5f082fd256930b644cc26f9c Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 4 Dec 2024 10:10:21 +0000 Subject: [PATCH 042/115] feat(studies): corrects undefined cucumber tests in data release timing --- app/models/study.rb | 2 +- features/studies/data_release_timings.feature | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index 0a291fba0c..f891e86125 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -228,7 +228,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute( :data_release_delay_reason, required: true, - in: DATA_RELEASE_DELAY_REASONS_ASSAY, + in: DATA_RELEASE_DELAY_REASONS_ASSAY + DATA_RELEASE_DELAY_REASONS_STANDARD, if: :delayed_release? ) diff --git a/features/studies/data_release_timings.feature b/features/studies/data_release_timings.feature index 15ba11a340..b59a19fc87 100644 --- a/features/studies/data_release_timings.feature +++ b/features/studies/data_release_timings.feature @@ -72,7 +72,7 @@ Feature: Studies have timings for release of their data Scenario: When the data release is never but the comment is not supplied When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" - And I fill in "12345" from "If reason for exemption requires DAC approval, what is the approval number?" + And I fill in "If reason for exemption requires DAC approval, what is the approval number?" with "12345" When I press "Create" Then I should be on the studies page # Again, ideally without study metadata @@ -82,7 +82,7 @@ Feature: Studies have timings for release of their data When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" And I fill in "Comment regarding prevention of data release and approval" with "Some reason" - And I fill in "12345" from "If reason for exemption requires DAC approval, what is the approval number? + And I fill in "If reason for exemption requires DAC approval, what is the approval number?" with "12345" When I press "Create" Then I should be on the study information page for "Testing data release strategies" And I should see "Your study has been created" From 12d0368ef42cd37b04bfa680d18156accff610e0 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 4 Dec 2024 14:39:57 +0000 Subject: [PATCH 043/115] feat(study): removes redundant old data release prevention reasons --- app/models/study.rb | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index f891e86125..0f50fdc6c0 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -83,7 +83,6 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength DATA_RELEASE_TIMING_DELAYED ].freeze - OLD_DATA_RELEASE_PREVENTION_REASONS = ['data validity', 'legal', 'replication of data subset'].freeze DATA_RELEASE_PREVENTION_REASONS = [ 'Pilot or validation studies - DAC approval not required', 'Collaborators will share data in a research repository - DAC approval not required', @@ -93,7 +92,6 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength ].freeze DATA_RELEASE_DELAY_FOR_OTHER = 'other' - DATA_RELEASE_DELAY_REASONS_ASSAY = ['phd study', 'assay of no other use', DATA_RELEASE_DELAY_FOR_OTHER].freeze DATA_RELEASE_DELAY_REASONS_STANDARD = [ 'PhD study', 'Capacity building', @@ -101,6 +99,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength 'Additional time to make data FAIR', DATA_RELEASE_DELAY_FOR_OTHER ].freeze + DATA_RELEASE_DELAY_REASONS_ASSAY = ['assay of no other use', *DATA_RELEASE_DELAY_REASONS_STANDARD].freeze DATA_RELEASE_DELAY_PERIODS = ['3 months', '6 months', '9 months', '12 months', '18 months'].freeze @@ -228,7 +227,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute( :data_release_delay_reason, required: true, - in: DATA_RELEASE_DELAY_REASONS_ASSAY + DATA_RELEASE_DELAY_REASONS_STANDARD, + in: DATA_RELEASE_DELAY_REASONS_ASSAY, if: :delayed_release? ) @@ -251,11 +250,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength end with_options(if: :never_release?) do - custom_attribute( - :data_release_prevention_reason, - in: DATA_RELEASE_PREVENTION_REASONS + OLD_DATA_RELEASE_PREVENTION_REASONS, - required: true - ) + custom_attribute(:data_release_prevention_reason, in: DATA_RELEASE_PREVENTION_REASONS, required: true) custom_attribute(:data_release_prevention_reason_comment, required: true) custom_attribute(:data_release_prevention_approval) end From ea5deb33d50de2bcd60b47d7929d621430eb857f Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 4 Dec 2024 16:36:52 +0000 Subject: [PATCH 044/115] tests(studies): updates study metadata test data to reflect data release changes --- spec/factories/study_factories.rb | 2 +- spec/models/study_spec.rb | 4 ++-- test/unit/data_release_test.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/factories/study_factories.rb b/spec/factories/study_factories.rb index 8ab0fcae97..9603449b12 100644 --- a/spec/factories/study_factories.rb +++ b/spec/factories/study_factories.rb @@ -54,7 +54,7 @@ new_field_values = { data_release_strategy: 'not applicable', data_release_timing: 'never', - data_release_prevention_reason: 'data validity', + data_release_prevention_reason: 'Protecting IP - DAC approval required', data_release_prevention_approval: 'Yes', data_release_prevention_reason_comment: 'This is the reason context' } diff --git a/spec/models/study_spec.rb b/spec/models/study_spec.rb index e6547b6917..3b146ee736 100644 --- a/spec/models/study_spec.rb +++ b/spec/models/study_spec.rb @@ -445,7 +445,7 @@ data_release_strategy: 'open', data_release_standard_agreement: 'Yes', data_release_timing: 'standard', - data_release_delay_reason: 'phd study', + data_release_delay_reason: 'PhD study', data_release_delay_period: '3 months', bam: true, data_release_delay_other_comment: 'Data Release delay other comment', @@ -456,7 +456,7 @@ ega_policy_accession_number: 'POL123456', array_express_accession_number: 'ARR123456', data_release_delay_approval: 'Yes', - data_release_prevention_reason: 'data validity', + data_release_prevention_reason: 'Protecting IP - DAC approval required', data_release_prevention_approval: 'Yes', data_release_prevention_reason_comment: 'Data Release prevention reason comment', data_access_group: 'something', diff --git a/test/unit/data_release_test.rb b/test/unit/data_release_test.rb index f1dee7e4af..328b3bc9d4 100644 --- a/test/unit/data_release_test.rb +++ b/test/unit/data_release_test.rb @@ -93,7 +93,7 @@ class DataReleaseTest < ActiveSupport::TestCase setup do @study.study_metadata.data_release_strategy = 'never' @study.study_metadata.data_release_timing = 'never' - @study.study_metadata.data_release_prevention_reason = 'legal' + @study.study_metadata.data_release_prevention_reason = 'Protecting IP - DAC approval required' @study.study_metadata.data_release_prevention_approval = 'Yes' @study.study_metadata.data_release_prevention_reason_comment = 'It just is' end @@ -106,7 +106,7 @@ class DataReleaseTest < ActiveSupport::TestCase context 'and release timing is delayed' do setup do @study.study_metadata.data_release_timing = 'delayed' - @study.study_metadata.data_release_delay_reason = 'phd study' + @study.study_metadata.data_release_delay_reason = 'PhD study' end data_release_strategies.each do |strategy| From 5ffaa681784da12565e5594e2d18bdb91c75d767 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Mon, 9 Dec 2024 11:01:58 +0000 Subject: [PATCH 045/115] fix(studies): adds blank values to changed study data release values in edit form to fail validation by default when editing an old study --- app/views/shared/metadata/edit/_study.html.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/shared/metadata/edit/_study.html.erb b/app/views/shared/metadata/edit/_study.html.erb index aa1e599762..70942e2638 100644 --- a/app/views/shared/metadata/edit/_study.html.erb +++ b/app/views/shared/metadata/edit/_study.html.erb @@ -59,13 +59,13 @@ %> <% metadata_fields.related_fields(to: :data_release_strategy, when: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> - <%= group.select(:data_release_prevention_reason, Study::DATA_RELEASE_PREVENTION_REASONS) %> + <%= group.select(:data_release_prevention_reason, [''] + Study::DATA_RELEASE_PREVENTION_REASONS) %> <%= group.text_area(:data_release_prevention_approval) %> <%= group.text_area(:data_release_prevention_reason_comment) %> <% end %> <% metadata_fields.related_fields(to: :data_release_strategy, in: Study::DATA_RELEASE_STRATEGIES, not: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> <% metadata_fields.related_fields(to: :data_release_timing, when: Study::DATA_RELEASE_TIMING_DELAYED) do %> - <%= group.select(:data_release_delay_reason, Study::DATA_RELEASE_DELAY_REASONS_STANDARD) %> + <%= group.select(:data_release_delay_reason, [''] + Study::DATA_RELEASE_DELAY_REASONS_STANDARD) %> <% group.change_select_options_for(:data_release_delay_reason, when: :data_release_study_type_id, values: { DataReleaseStudyType.assay_types.map(&:id) => [ '' ] + Study::DATA_RELEASE_DELAY_REASONS_ASSAY, From 6966d2a3a0dade1d9bbc42f230e13781aef0fc34 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Mon, 9 Dec 2024 11:46:09 +0000 Subject: [PATCH 046/115] tests(studies): updates cucumber studies data release test after removal of valid default values --- features/studies/data_release_timings.feature | 2 ++ 1 file changed, 2 insertions(+) diff --git a/features/studies/data_release_timings.feature b/features/studies/data_release_timings.feature index b59a19fc87..4c0c2f3961 100644 --- a/features/studies/data_release_timings.feature +++ b/features/studies/data_release_timings.feature @@ -72,6 +72,7 @@ Feature: Studies have timings for release of their data Scenario: When the data release is never but the comment is not supplied When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" + And I select "Protecting IP - DAC approval required" from "What is the reason for preventing data release?" And I fill in "If reason for exemption requires DAC approval, what is the approval number?" with "12345" When I press "Create" Then I should be on the studies page @@ -81,6 +82,7 @@ Feature: Studies have timings for release of their data Scenario: When the data release is never and the comment is supplied When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" + And I select "Protecting IP - DAC approval required" from "What is the reason for preventing data release?" And I fill in "Comment regarding prevention of data release and approval" with "Some reason" And I fill in "If reason for exemption requires DAC approval, what is the approval number?" with "12345" When I press "Create" From fe53ead11ff06d6c75d976852bfb88bea0e801cb Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 16 Dec 2024 11:03:31 +0000 Subject: [PATCH 047/115] Append the scRNA Core cDNA Prep feasibility validation to the additional validations --- app/models/submission/validations_by_template_name.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index f99c48bca7..c2dd94d770 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true # rubocop:todo Metrics/ModuleLength module Submission::ValidationsByTemplateName + + include Submission::ScrnaCoreCdnaPrepFeasibilityValidation + # Template names SCRNA_CORE_CDNA_PREP_GEM_X_5P = 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p' @@ -40,6 +43,7 @@ def apply_additional_validations_by_template_name validate_consistent_column_value(HEADER_NUMBER_OF_POOLS) validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) validate_samples_per_pool_for_labware + validate_scrna_core_cdna_prep_feasibility end end From 4ddbb27f456734655dbb89e6f17d05524f7d0a18 Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 16 Dec 2024 11:06:22 +0000 Subject: [PATCH 048/115] Add total number of samples, pools, and per-pool error message templates of the scRNA Core cDNA Prep feasibility validation to the locale file --- config/locales/metadata/en.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/config/locales/metadata/en.yml b/config/locales/metadata/en.yml index fa02c9f98b..eed3dbc917 100644 --- a/config/locales/metadata/en.yml +++ b/config/locales/metadata/en.yml @@ -560,3 +560,18 @@ en: metadata: release_reason: label: Reason for releasing data + submission: + scrna_core_cdna_prep_feasibility_validation: + errors: + total_number_of_samples: + zero: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, no samples were found." + one: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, 1 sample was found." + other: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, %{count} samples were found." + total_number_of_pools: + zero: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, no pools were found." + one: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, 1 pool was found." + other: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, %{count} pools were found." + number_of_samples_in_pool: + zero: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, no samples were found for the study %{study_name} and the project %{project_name}." + one: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, 1 sample was found for the study %{study_name} and the project %{project_name}." + other: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, %{count} samples were found for the study %{study_name} and the project %{project_name}." From c9d410308e50163c245394a941a9ddc180d5087a Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 16 Dec 2024 13:26:32 +0000 Subject: [PATCH 049/115] Add validations for total number of samples, pools, smallest pools size and biggest pool size --- ...a_core_cdna_prep_feasibility_validation.rb | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb new file mode 100644 index 0000000000..61a7bf549f --- /dev/null +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Submission::ScrnaCoreCdnaPrepFeasibilityValidation + HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) + HEADER_PLATE_WELL = 'plate well' unless defined?(HEADER_PLATE_WELL) + HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' unless defined?(HEADER_NUMBER_OF_POOLS) + + TOTAL_NUMBER_OF_SAMPLES = { min: 5, max: 96 }.freeze + TOTAL_NUMBER_OF_POOLS = { min: 1, max: 8 }.freeze + NUMBER_OF_SAMPLES_PER_POOL = { min: 5, max: 25 }.freeze + + def validate_scrna_core_cdna_prep_feasibility + required = [HEADER_BARCODE, HEADER_PLATE_WELL, HEADER_NUMBER_OF_POOLS] + return unless required.all? { |header| headers.include?(header) } + + validate_scrna_core_cdna_prep_total_number_of_samples + validate_scrna_core_cdna_prep_total_number_of_pools + validate_scrna_core_cdna_prep_feasibility_by_samples + validate_scrna_core_cdna_prep_feasibility_by_donors + validate_scrna_core_cdna_prep_full_allowance + end + + private + + def validate_scrna_core_cdna_prep_total_number_of_samples + barcodes = csv_data_rows.pluck(headers.index(HEADER_BARCODE)) + well_locations = csv_data_rows.pluck(headers.index(HEADER_PLATE_WELL)) + count = calculate_total_number_of_samples(barcodes, well_locations) + min = TOTAL_NUMBER_OF_SAMPLES[:min] + max = TOTAL_NUMBER_OF_SAMPLES[:max] + + return if count.between?(min, max) # inclusive + + message = + I18n.t( + 'errors.total_number_of_samples', + min: min, + max: max, + count: count, + scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + ) + + errors.add(:spreadsheet, message) + end + + def validate_scrna_core_cdna_prep_total_number_of_pools + first_rows = group_rows_by_study_and_project.map { |_study_project, rows| rows.first } + count = first_rows.sum { |row| row[headers.index(HEADER_NUMBER_OF_POOLS)].to_i } + min = TOTAL_NUMBER_OF_POOLS[:min] + max = TOTAL_NUMBER_OF_POOLS[:max] + + return if count.between?(min, max) # inclusive + + message = + I18n.t( + 'errors.total_number_of_pools', + min: min, + max: max, + count: count, + scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + ) + + errors.add(:spreadsheet, message) + end + + def validate_scrna_core_cdna_prep_feasibility_by_samples + group_rows_by_study_and_project.each_value do |rows| + barcodes = rows.pluck(headers.index(HEADER_BARCODE)) + well_locations = rows.pluck(headers.index(HEADER_PLATE_WELL)) + + number_of_samples = calculate_total_number_of_samples(barcodes, well_locations) + number_of_pools = rows.first[headers.index(HEADER_NUMBER_OF_POOLS)].to_i + quotient, remainder = number_of_samples.divmod(number_of_pools) + smallest = biggest = quotient + biggest += 1 if remainder.positive? + + min = NUMBER_OF_SAMPLES_PER_POOL[:min] + max = NUMBER_OF_SAMPLES_PER_POOL[:max] + + unless smallest.between?(min, max) + message = + I18n.t( + 'errors.number_of_samples_in_smallest_pool', + min: min, + max: max, + count: smallest, + pool: 'smallest', + scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + ) + errors.add(:spreadsheet, message) + end + + next unless remainder.positive? && !biggest.between?(min, max) + I18n.t( + 'errors.number_of_samples_in_biggest_pool', + min: min, + max: max, + count: biggest, + pool: 'biggest', + scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + ) + end + end + + def validate_scrna_core_cdna_prep_feasibility_by_donors + end + + def validate_scrna_core_cdna_prep_full_allowance + end + + def calculate_total_number_of_samples(barcodes, well_locations) + receptacles = find_receptacles(barcodes, well_locations) + receptacles.map(&:samples).flatten.count.to_i + end + + def find_receptacles(barcodes, well_locations) + return find_tubes(barcodes) if tube?(barcodes, well_locations) + + plate = Plate.find_from_any_barcode(barcodes.uniq.first) + plate.wells.for_bulk_submission.located_at(well_locations) + end +end From ef2ca6de4eb4d484229f353d6e549a2b78f6dfac Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 16 Dec 2024 13:33:14 +0000 Subject: [PATCH 050/115] Move scRNA Core cDNA Prep feasibility validation locale strings to common file --- config/locales/en.yml | 15 +++++++++++++++ config/locales/metadata/en.yml | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 8a7a0914ca..22dfaf87a9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -169,6 +169,21 @@ en: submissions: no_submissions: "There are no submissions yet. Please create one." + scrna_core_cdna_prep_feasibility_validation: + errors: + total_number_of_samples: + zero: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, no samples were found." + one: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, one sample was found." + other: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, %{count} samples were found." + total_number_of_pools: + zero: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, no pools were found." + one: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, one pool was found." + other: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, %{count} pools were found." + number_of_samples_in_pool: + zero: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, no samples were found for the study %{study_name} and the project %{project_name}." + one: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, one sample was found for the study %{study_name} and the project %{project_name}." + other: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, %{count} samples were found for the study %{study_name} and the project %{project_name}." + cherrypick: picking_by_row: "This cherrypick may take longer as it is picking by rows, rather than columns." diff --git a/config/locales/metadata/en.yml b/config/locales/metadata/en.yml index eed3dbc917..fa02c9f98b 100644 --- a/config/locales/metadata/en.yml +++ b/config/locales/metadata/en.yml @@ -560,18 +560,3 @@ en: metadata: release_reason: label: Reason for releasing data - submission: - scrna_core_cdna_prep_feasibility_validation: - errors: - total_number_of_samples: - zero: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, no samples were found." - one: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, 1 sample was found." - other: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, %{count} samples were found." - total_number_of_pools: - zero: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, no pools were found." - one: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, 1 pool was found." - other: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, %{count} pools were found." - number_of_samples_in_pool: - zero: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, no samples were found for the study %{study_name} and the project %{project_name}." - one: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, 1 sample was found for the study %{study_name} and the project %{project_name}." - other: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, %{count} samples were found for the study %{study_name} and the project %{project_name}." From 11c20fa3eb5e73834e50c04bdb1bf4ae586c75c3 Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 16 Dec 2024 13:33:58 +0000 Subject: [PATCH 051/115] Change namescape for the locale string from submission to submissions to match the section in the locale file --- .../scrna_core_cdna_prep_feasibility_validation.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb index 61a7bf549f..c73d20ed4a 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb @@ -37,7 +37,7 @@ def validate_scrna_core_cdna_prep_total_number_of_samples min: min, max: max, count: count, - scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) errors.add(:spreadsheet, message) @@ -57,7 +57,7 @@ def validate_scrna_core_cdna_prep_total_number_of_pools min: min, max: max, count: count, - scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) errors.add(:spreadsheet, message) @@ -85,7 +85,7 @@ def validate_scrna_core_cdna_prep_feasibility_by_samples max: max, count: smallest, pool: 'smallest', - scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) errors.add(:spreadsheet, message) end @@ -97,7 +97,7 @@ def validate_scrna_core_cdna_prep_feasibility_by_samples max: max, count: biggest, pool: 'biggest', - scope: 'submission.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) end end From 380d1615e43449256b551d0c8ab4663c43362f35 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Mon, 16 Dec 2024 15:41:28 +0000 Subject: [PATCH 052/115] feat(study-metadata): removes data_release_delay_approval for new/edited studies --- app/models/study.rb | 4 ---- app/views/shared/metadata/edit/_study.html.erb | 3 --- .../support/step_definitions/study_steps.rb | 1 - spec/models/study_spec.rb | 17 ----------------- 4 files changed, 25 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index 0f50fdc6c0..dca73ee792 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -245,10 +245,6 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute(:ega_policy_accession_number) custom_attribute(:array_express_accession_number) - with_options(if: :delayed_for_long_time?, required: true) do - custom_attribute(:data_release_delay_approval, in: YES_OR_NO, default: NO) - end - with_options(if: :never_release?) do custom_attribute(:data_release_prevention_reason, in: DATA_RELEASE_PREVENTION_REASONS, required: true) custom_attribute(:data_release_prevention_reason_comment, required: true) diff --git a/app/views/shared/metadata/edit/_study.html.erb b/app/views/shared/metadata/edit/_study.html.erb index 70942e2638..d754aaddb4 100644 --- a/app/views/shared/metadata/edit/_study.html.erb +++ b/app/views/shared/metadata/edit/_study.html.erb @@ -74,9 +74,6 @@ %> <%= group.select(:data_release_delay_period, Study::DATA_RELEASE_DELAY_PERIODS) %> - <% metadata_fields.related_fields(to: :data_release_delay_period, in: Study::DATA_RELEASE_DELAY_PERIODS) do %> - <%= group.radio_select(:data_release_delay_approval, Study::YES_OR_NO) %> - <% end %> <% metadata_fields.related_fields(to: :data_release_delay_reason, when: Study::DATA_RELEASE_DELAY_FOR_OTHER) do %> <%= group.text_area(:data_release_delay_other_comment) %> <% metadata_fields.related_fields(to: :data_release_delay_period, in: Study::DATA_RELEASE_DELAY_PERIODS) do %> diff --git a/features/support/step_definitions/study_steps.rb b/features/support/step_definitions/study_steps.rb index e3bacbb72b..466e42627b 100644 --- a/features/support/step_definitions/study_steps.rb +++ b/features/support/step_definitions/study_steps.rb @@ -162,7 +162,6 @@ def given_study_metadata(attribute, regexp) data_release_delay_reason: 'other', data_release_delay_other_comment: reason, data_release_delay_period: "#{period} months", - data_release_delay_approval: 'Yes', data_release_delay_reason_comment: reason } ) diff --git a/spec/models/study_spec.rb b/spec/models/study_spec.rb index 3b146ee736..a3e27ab54e 100644 --- a/spec/models/study_spec.rb +++ b/spec/models/study_spec.rb @@ -739,23 +739,6 @@ end end - context 'delayed for long time' do - let(:study) do - create( - :study, - study_metadata: - create( - :study_metadata, - metadata.merge(data_release_timing: 'delayed', data_release_delay_period: '6 months') - ) - ) - end - - it 'will have a data_release_delay_approval' do - expect(study.study_metadata.data_release_delay_approval).to eq(metadata[:data_release_delay_approval]) - end - end - context 'never released' do let(:never_release_fields) do { From bac3e8b6744fd6d1a7e59b26449d8cbac1c34f84 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Mon, 16 Dec 2024 15:52:40 +0000 Subject: [PATCH 053/115] tests(study-metadata): removes test references to deprecated release delay approval study metadata --- ...5391_study_xml_needs_to_be_reverted_to_old_version.feature | 4 ---- features/studies/8447221_data_release_help_text.feature | 1 - 2 files changed, 5 deletions(-) diff --git a/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature b/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature index 59e640bd67..8057e24450 100644 --- a/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature +++ b/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature @@ -150,10 +150,6 @@ Feature: The XML for the sequencescape API Are all the samples to be used in this study commercially available, unlinked anonymised cell-lines? No - - Has the delay period been approved by the data sharing committee for this project? - - ENA Project ID diff --git a/features/studies/8447221_data_release_help_text.feature b/features/studies/8447221_data_release_help_text.feature index 0b20d4bab8..603b2c0b4c 100644 --- a/features/studies/8447221_data_release_help_text.feature +++ b/features/studies/8447221_data_release_help_text.feature @@ -43,7 +43,6 @@ Feature: Update the data release fields for creating a study When I select "delayed" from "How is the data release to be timed?" And I select "other" from "Reason for delaying release" And I select "" from "Delay for" - Then I should exactly see "Has the delay period been approved by the data sharing committee for this project?" And I should exactly see "Comment regarding data release timing and approval" When I fill in the following: From 62e18aca77651f6f8be43fc5d09ae54c0300a858 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 00:35:58 +0000 Subject: [PATCH 054/115] Copy scRNA config from Limber and add cDNA Prep parameters --- config/initializers/scrna_config.rb | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 config/initializers/scrna_config.rb diff --git a/config/initializers/scrna_config.rb b/config/initializers/scrna_config.rb new file mode 100644 index 0000000000..abadf8e5d7 --- /dev/null +++ b/config/initializers/scrna_config.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +# Stores constants used in pooling and chip loading calculations for samples in the scRNA Core pipeline. +Rails.application.config.scrna_config = { + # Maximum volume to take into the pools plate for each sample (in microlitres) + maximum_sample_volume: 60.0, + # Minimum volume to take into the pools plate for each sample (in microlitres) + minimum_sample_volume: 5.0, + # Minimum volume required for resuspension in microlitres + minimum_resuspension_volume: 10.0, + # Conversion factor from millilitres to microlitres + millilitres_to_microlitres: 1_000.0, + # Number of cells required for each sample going into the pool + required_number_of_cells_per_sample_in_pool: 30_000, + # Factor accounting for wastage of material when transferring between labware + wastage_factor: 0.95, + # Fixed wastage volume in microlitres + wastage_volume: 5.0, + # Desired concentration of cells per microlitre for chip loading + desired_chip_loading_concentration: 2400, + # Desired volume in the chip well (in microlitres) + desired_chip_loading_volume: 37.5, + # Desired number of runs for full allowance calculations + desired_number_of_runs: 2, + # Volume taken for cell counting in microlitres + volume_taken_for_cell_counting: 10.0, + # Full allowance table, keyed on numbers of samples with values for number of cells per chip well + full_allowance_table: { + 5 => 41_428, + 6 => 55_714, + 7 => 70_000, + 8 => 84_285 + }, + # Key for the required number of cells metadata stored on Study (in poly_metadata) + study_required_number_of_cells_per_sample_in_pool_key: 'scrna_core_pbmc_donor_pooling_required_number_of_cells', + # Default viability threshold when passing/failing samples (in percent) + viability_default_threshold: 65, + # Default total cell count threshold when passing/failing samples + total_cell_count_default_threshold: 50_000, + # Key for the number of cells per chip well metadata stored on pool wells (in poly_metadata) + number_of_cells_per_chip_well_key: 'scrna_core_pbmc_donor_pooling_number_of_cells_per_chip_well', + # + cdna_prep_minimum_total_number_of_samples: 5, + cdna_prep_maximum_total_number_of_samples: 96, + cdna_prep_minimum_total_number_of_pools: 1, + cdna_prep_maximum_total_number_of_samples: 8, + cdna_prep_minimum_number_of_samples_per_pool: 5, + cdna_prep_maximum_number_of_samples_per_pool: 25 +}.freeze From 0e453856066fbdd1111d57bd9aa5b0ad3279866e Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 00:38:07 +0000 Subject: [PATCH 055/115] Add multiline locale strings and comments for scrna_core_cdna_prep_feasibility_validation.errors --- config/locales/en.yml | 64 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 22dfaf87a9..b22f1e3674 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -171,19 +171,61 @@ en: no_submissions: "There are no submissions yet. Please create one." scrna_core_cdna_prep_feasibility_validation: errors: + # Validation: Total number of samples in the submission must be between 5 and 96 (inclusive). + # min and max are for the acceptable range of the total number of samples in the submission. + # count is the total number of samples in the submission. + # zero, one and other are the pluralization keys selected by the count of samples total_number_of_samples: - zero: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, no samples were found." - one: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, one sample was found." - other: "The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, %{count} samples were found." + zero: | + The total number of samples in the submission must be between %{min} and %{max} (inclusive). + However, no samples were found. + one: | + The total number of samples in the submission must be between %{min} and %{max} (inclusive). + However, one sample was found. + other: | + The total number of samples in the submission must be between %{min} and %{max} (inclusive). + However, %{count} samples were found. + # Validation: Total (requested) number of pools must be between 1 and 8 (inclusive). + # min and max are for the acceptable range of the total number of pools in the submission. + # count is the total number of pools in the submission. + # zero, one and other are the pluralization keys selected by the count of pools total_number_of_pools: - zero: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, no pools were found." - one: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, one pool was found." - other: "The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, %{count} pools were found." - number_of_samples_in_pool: - zero: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, no samples were found for the study %{study_name} and the project %{project_name}." - one: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, one sample was found for the study %{study_name} and the project %{project_name}." - other: "The number of samples in the #{pool} pool for a study and project must be between %{min} and %{max} (inclusive). However, %{count} samples were found for the study %{study_name} and the project %{project_name}." - + zero: | + The total number of pools in the submission must be between %{min} and %{max} (inclusive). + However, no pools were found. + one: | + The total number of pools in the submission must be between %{min} and %{max} (inclusive). + However, one pool was found. + other: | + The total number of pools in the submission must be between %{min} and %{max} (inclusive). + However, %{count} pools were found. + # Validation: The number of pools requested must be feasible given the number of samples. + # size_type is either 'smallest' and 'biggest'. + # study_name and project_name are the values of respective fields from the spreadsheet. + # zero, one and other are the pluralization keys selected by the count of samples in the pools. + number_of_pools_by_samples: + zero: | + The number of pools is not feasible for the study %{study_name} and the project %{project_name}. + The number of samples in the #{size_type} pool must be between %{min} and %{max} (inclusive). + However, no samples were found in the pool. + one: | + The number of pools is not feasible for the study %{study_name} and the project %{project_name}. + The number of samples in the #{size_type} pool must be between %{min} and %{max} (inclusive). + However, one sample was found in the pool. + other: | + The number of pools is not feasible for the study %{study_name} and the project %{project_name}. + The number of samples in the #{size_type} pool must be between %{min} and %{max} (inclusive). + However, %{count} samples were found in the pool. + # Validation: The number of pools requested must be feasible having checked for donor clash. + # study_name and project_name are the values of respective fields from the spreadsheet. + # count is the size of the biggest group of samples with the same donor ID. + # number_of_pools is the number of pools requested for the study and project. + # barcodes_or_well_locations is the list of barcodes (tubes) or well locations (plates) with the same donor ID. + number_of_pools_by_donors: | + The number of pools is not feasible for the study %{study_name} and the project %{project_name}. + The number of samples with the same donor ID must be less than or equal to the requested number of pools. + However, %{count} samples with the same donor ID were found, while the number of pools requested was %{number_of_pools}. + The samples with the same donor ID follow: %{barcodes_or_well_locations}. cherrypick: picking_by_row: "This cherrypick may take longer as it is picking by rows, rather than columns." From 477845d535616873b755c2cbe4e3b92894eea781 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 00:41:42 +0000 Subject: [PATCH 056/115] Move constants to scrna config, and separate adding error messages from validations --- ...a_core_cdna_prep_feasibility_validation.rb | 181 +++++++++++++----- 1 file changed, 136 insertions(+), 45 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb index c73d20ed4a..b654937ae1 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb @@ -1,14 +1,11 @@ # frozen_string_literal: true +# rubocop:disable Metrics/ModuleLength module Submission::ScrnaCoreCdnaPrepFeasibilityValidation HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) HEADER_PLATE_WELL = 'plate well' unless defined?(HEADER_PLATE_WELL) HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' unless defined?(HEADER_NUMBER_OF_POOLS) - TOTAL_NUMBER_OF_SAMPLES = { min: 5, max: 96 }.freeze - TOTAL_NUMBER_OF_POOLS = { min: 1, max: 8 }.freeze - NUMBER_OF_SAMPLES_PER_POOL = { min: 5, max: 25 }.freeze - def validate_scrna_core_cdna_prep_feasibility required = [HEADER_BARCODE, HEADER_PLATE_WELL, HEADER_NUMBER_OF_POOLS] return unless required.all? { |header| headers.include?(header) } @@ -23,15 +20,19 @@ def validate_scrna_core_cdna_prep_feasibility private def validate_scrna_core_cdna_prep_total_number_of_samples - barcodes = csv_data_rows.pluck(headers.index(HEADER_BARCODE)) - well_locations = csv_data_rows.pluck(headers.index(HEADER_PLATE_WELL)) + barcodes, well_locations = extract_barcodes_and_well_locations(csv_data_rows) count = calculate_total_number_of_samples(barcodes, well_locations) - min = TOTAL_NUMBER_OF_SAMPLES[:min] - max = TOTAL_NUMBER_OF_SAMPLES[:max] + min = scrna_config[:cdna_prep_minimum_total_number_of_samples] + max = scrna_config[:cdna_prep_maximum_total_number_of_samples] return if count.between?(min, max) # inclusive - message = + add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) + end + + def add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) + errors.add( + :spreadsheet, I18n.t( 'errors.total_number_of_samples', min: min, @@ -39,19 +40,23 @@ def validate_scrna_core_cdna_prep_total_number_of_samples count: count, scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) - - errors.add(:spreadsheet, message) + ) end def validate_scrna_core_cdna_prep_total_number_of_pools first_rows = group_rows_by_study_and_project.map { |_study_project, rows| rows.first } count = first_rows.sum { |row| row[headers.index(HEADER_NUMBER_OF_POOLS)].to_i } - min = TOTAL_NUMBER_OF_POOLS[:min] - max = TOTAL_NUMBER_OF_POOLS[:max] + min = scrna_config[:cdna_prep_minimum_total_number_of_pools] + max = scrna_config[:cdna_prep_maximum_total_number_of_pools] return if count.between?(min, max) # inclusive - message = + add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) + end + + def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) + errors.add( + :spreadsheet, I18n.t( 'errors.total_number_of_pools', min: min, @@ -59,64 +64,150 @@ def validate_scrna_core_cdna_prep_total_number_of_pools count: count, scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) - - errors.add(:spreadsheet, message) + ) end + # rubocop:disable Metrics/AbcSize, Metrics/MethodLength def validate_scrna_core_cdna_prep_feasibility_by_samples group_rows_by_study_and_project.each_value do |rows| - barcodes = rows.pluck(headers.index(HEADER_BARCODE)) - well_locations = rows.pluck(headers.index(HEADER_PLATE_WELL)) - + barcodes, well_locations = extract_barcodes_and_well_locations(rows) number_of_samples = calculate_total_number_of_samples(barcodes, well_locations) number_of_pools = rows.first[headers.index(HEADER_NUMBER_OF_POOLS)].to_i - quotient, remainder = number_of_samples.divmod(number_of_pools) - smallest = biggest = quotient - biggest += 1 if remainder.positive? - - min = NUMBER_OF_SAMPLES_PER_POOL[:min] - max = NUMBER_OF_SAMPLES_PER_POOL[:max] - - unless smallest.between?(min, max) - message = - I18n.t( - 'errors.number_of_samples_in_smallest_pool', - min: min, - max: max, - count: smallest, - pool: 'smallest', - scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' - ) - errors.add(:spreadsheet, message) + pool_sizes = calculate_pool_size_types(number_of_samples, number_of_pools) + min = scrna_config[:cdna_prep_minimum_number_of_samples_per_pool] + max = scrna_config[:cdna_prep_maximum_number_of_samples_per_pool] + pool_sizes.each do |size_type, pool_size| + next if pool_size.between?(min, max) + add_error_scrna_core_cdna_prep_feasibility_by_samples(min, max, pool_size, size_type) end + end + end + # rubocop:enable Metrics/AbcSize, Metrics/MethodLength - next unless remainder.positive? && !biggest.between?(min, max) + def calculate_pool_size_types(number_of_samples, number_of_pools) + quotient, remainder = number_of_samples.divmod(number_of_pools) + smallest = biggest = quotient + biggest += 1 if remainder.positive? + { 'smallest' => smallest, 'biggest' => biggest } + end + + def add_error_scrna_core_cdna_prep_feasibility_by_samples(min, max, count, size_type) + errors.add( + :spreadsheet, I18n.t( - 'errors.number_of_samples_in_biggest_pool', + 'errors.number_of_pools_by_samples', min: min, max: max, - count: biggest, - pool: 'biggest', + count: count, + size_type: size_type, scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' ) - end + ) end + # rubocop:disable Metrics/MethodLength def validate_scrna_core_cdna_prep_feasibility_by_donors + group_rows_by_study_and_project.each do |(study_name, project_name), rows| + number_of_pools = rows.first[headers.index(HEADER_NUMBER_OF_POOLS)].to_i + donor_id_groups = group_rows_by_donor_id(rows) + _donor_id, biggest_group = donor_id_groups.max_by { |_key, value| value.size } + + next if biggest_group.size <= number_of_pools + + barcodes_or_well_locations = list_barcodes_or_well_locations_to_check_for_donors(biggest_group) + add_error_scrna_core_cdna_prep_feasibility_by_donors( + study_name, + project_name, + biggest_group.size, + number_of_pools, + barcodes_or_well_locations + ) + end + end + # rubocop:enable Metrics/MethodLength + + def add_error_scrna_core_cdna_prep_feasibility_by_donors( + study_name, + project_name, + count, + number_of_pools, + barcodes_or_well_locations + ) + errors.add( + :spreadsheet, + I18n.t( + 'errors.number_of_pools_by_donors', + study_name: study_name, + project_name: project_name, + count: count, + number_of_pools: number_of_pools, + barcodes_or_well_locations: barcodes_or_well_locations, + scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' + ) + ) + end + + def list_barcodes_or_well_locations_to_check_for_donors(rows) + barcodes, well_locations = extract_barcodes_and_well_locations(rows) + tube?(barcodes, well_locations) ? barcodes.join(', ') : well_locations.join(', ') end def validate_scrna_core_cdna_prep_full_allowance end + def extract_barcodes_and_well_locations(rows) + barcodes = rows.pluck(headers.index(HEADER_BARCODE)) + well_locations = rows.pluck(headers.index(HEADER_PLATE_WELL)) + [barcodes, well_locations] + end + def calculate_total_number_of_samples(barcodes, well_locations) receptacles = find_receptacles(barcodes, well_locations) receptacles.map(&:samples).flatten.count.to_i end def find_receptacles(barcodes, well_locations) - return find_tubes(barcodes) if tube?(barcodes, well_locations) + if tube?(barcodes, well_locations) + receptacles = find_tubes(barcodes) + sort_tube_receptacles_by_rows(receptacles, barcodes) + else + plate = Plate.find_from_any_barcode(barcodes.uniq.first) + receptacles = plate.wells.for_bulk_submission.located_at(well_locations) + sort_plate_wells_by_rows(receptacles, well_locations) + end + end + + def sort_tube_receptacles_by_rows(receptacles, barcodes) + receptacle_map = {} + receptacles.each do |receptacle| + receptacle.labware.barcodes.each { |barcode| receptacle_map[barcode] = receptacle } + end + barcodes.map { |barcode| receptacle_map[barcode] } + end + + def sort_plate_wells_by_rows(receptacles, well_locations) + receptacle_map = {} + receptacles.each { |receptacle| receptacle_map[receptacle.map_description] = receptacle } + well_locations.map { |well_location| receptacle_map[well_location] } + end + + # rubocop:disable Metrics/AbcSize + def group_rows_by_donor_id(rows) + barcodes, well_locations = extract_barcodes_and_well_locations(rows) + receptacles = find_receptacles(barcodes, well_locations) + + # Create a mapping between receptacles and their corresponding rows. + receptacle_to_row_map = {} + rows.each_with_index { |row, index| receptacle_to_row_map[receptacles[index]] = row } + + # Group receptacles by donor_id and replace values with corresponding rows. + groups = receptacles.group_by { |receptacle| receptacle.aliquots.first.sample.sample_metadata.donor_id } + groups.transform_values! { |array| array.map { |receptacle| receptacle_to_row_map[receptacle] } } + end + # rubocop:enable Metrics/AbcSize - plate = Plate.find_from_any_barcode(barcodes.uniq.first) - plate.wells.for_bulk_submission.located_at(well_locations) + def scrna_config + Rails.application.config.scrna_config end end +# rubocop:enable Metrics/ModuleLength From 3b1e4f10c7862a32d0e740e5b09cf042cfa2dbd4 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 00:42:09 +0000 Subject: [PATCH 057/115] Rubocop --- app/models/submission/validations_by_template_name.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index c2dd94d770..8b6018084d 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true # rubocop:todo Metrics/ModuleLength module Submission::ValidationsByTemplateName - include Submission::ScrnaCoreCdnaPrepFeasibilityValidation # Template names From 8a04d88b6e48d6ed2b01283fed0967dc7933632b Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 00:47:18 +0000 Subject: [PATCH 058/115] Rename to scrna_core_cdna_prep_feasibility_validator --- ...alidation.rb => scrna_core_cdna_prep_feasibility_validator.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/models/submission/{scrna_core_cdna_prep_feasibility_validation.rb => scrna_core_cdna_prep_feasibility_validator.rb} (100%) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb similarity index 100% rename from app/models/submission/scrna_core_cdna_prep_feasibility_validation.rb rename to app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb From e6f44957a6786697fb8205cc3257b9e4ad38ab72 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 00:51:42 +0000 Subject: [PATCH 059/115] Rename I18n scope for the cDNA feasibility validator --- .../scrna_core_cdna_prep_feasibility_validator.rb | 10 +++++----- app/models/submission/validations_by_template_name.rb | 2 +- config/locales/en.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index b654937ae1..c7893b1f5a 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # rubocop:disable Metrics/ModuleLength -module Submission::ScrnaCoreCdnaPrepFeasibilityValidation +module Submission::ScrnaCoreCdnaPrepFeasibilityValidator HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) HEADER_PLATE_WELL = 'plate well' unless defined?(HEADER_PLATE_WELL) HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' unless defined?(HEADER_NUMBER_OF_POOLS) @@ -38,7 +38,7 @@ def add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) min: min, max: max, count: count, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' ) ) end @@ -62,7 +62,7 @@ def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) min: min, max: max, count: count, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' ) ) end @@ -100,7 +100,7 @@ def add_error_scrna_core_cdna_prep_feasibility_by_samples(min, max, count, size_ max: max, count: count, size_type: size_type, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' ) ) end @@ -142,7 +142,7 @@ def add_error_scrna_core_cdna_prep_feasibility_by_donors( count: count, number_of_pools: number_of_pools, barcodes_or_well_locations: barcodes_or_well_locations, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validation' + scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' ) ) end diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index 8b6018084d..a9fd8d6283 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # rubocop:todo Metrics/ModuleLength module Submission::ValidationsByTemplateName - include Submission::ScrnaCoreCdnaPrepFeasibilityValidation + include Submission::ScrnaCoreCdnaPrepFeasibilityValidator # Template names SCRNA_CORE_CDNA_PREP_GEM_X_5P = 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p' diff --git a/config/locales/en.yml b/config/locales/en.yml index b22f1e3674..f8835eefa1 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -169,7 +169,7 @@ en: submissions: no_submissions: "There are no submissions yet. Please create one." - scrna_core_cdna_prep_feasibility_validation: + scrna_core_cdna_prep_feasibility_validator: errors: # Validation: Total number of samples in the submission must be between 5 and 96 (inclusive). # min and max are for the acceptable range of the total number of samples in the submission. From f41d9c3fcd72ae485a8e92a44cf900269367f36b Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 09:34:53 +0000 Subject: [PATCH 060/115] Move I18n scope to constant --- .../scrna_core_cdna_prep_feasibility_validator.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index c7893b1f5a..cba4d30548 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -5,6 +5,7 @@ module Submission::ScrnaCoreCdnaPrepFeasibilityValidator HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) HEADER_PLATE_WELL = 'plate well' unless defined?(HEADER_PLATE_WELL) HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' unless defined?(HEADER_NUMBER_OF_POOLS) + I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR = 'submissions.scrna_core_cdna_prep_feasibility_validator' def validate_scrna_core_cdna_prep_feasibility required = [HEADER_BARCODE, HEADER_PLATE_WELL, HEADER_NUMBER_OF_POOLS] @@ -38,7 +39,7 @@ def add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) min: min, max: max, count: count, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' + scope: I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR ) ) end @@ -62,7 +63,7 @@ def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) min: min, max: max, count: count, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' + scope: I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR ) ) end @@ -100,7 +101,7 @@ def add_error_scrna_core_cdna_prep_feasibility_by_samples(min, max, count, size_ max: max, count: count, size_type: size_type, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' + scope: I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR ) ) end @@ -142,7 +143,7 @@ def add_error_scrna_core_cdna_prep_feasibility_by_donors( count: count, number_of_pools: number_of_pools, barcodes_or_well_locations: barcodes_or_well_locations, - scope: 'submissions.scrna_core_cdna_prep_feasibility_validator' + scope: I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR ) ) end @@ -180,7 +181,7 @@ def find_receptacles(barcodes, well_locations) def sort_tube_receptacles_by_rows(receptacles, barcodes) receptacle_map = {} receptacles.each do |receptacle| - receptacle.labware.barcodes.each { |barcode| receptacle_map[barcode] = receptacle } + receptacle.labware.barcodes.each { |barcode_obj| receptacle_map[barcode_obj.barcode] = receptacle } end barcodes.map { |barcode| receptacle_map[barcode] } end From 83003b4fb46b63974ac76c07c2c96a231cff2e8e Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 09:35:31 +0000 Subject: [PATCH 061/115] Fix config option for cdna prep number of pools --- config/initializers/scrna_config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/scrna_config.rb b/config/initializers/scrna_config.rb index abadf8e5d7..daf1b63422 100644 --- a/config/initializers/scrna_config.rb +++ b/config/initializers/scrna_config.rb @@ -43,7 +43,7 @@ cdna_prep_minimum_total_number_of_samples: 5, cdna_prep_maximum_total_number_of_samples: 96, cdna_prep_minimum_total_number_of_pools: 1, - cdna_prep_maximum_total_number_of_samples: 8, + cdna_prep_maximum_total_number_of_pools: 8, cdna_prep_minimum_number_of_samples_per_pool: 5, cdna_prep_maximum_number_of_samples_per_pool: 25 }.freeze From c897fbf4a1e83cde31e3f6f95e238ec1e0018f82 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 09:38:32 +0000 Subject: [PATCH 062/115] Fix existing bulk submission tests by adding donors to sample metadata --- spec/models/bulk_submission_spec.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index db3c5ea948..d41ba96382 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -30,6 +30,9 @@ tubes.each_with_index.map do |tube, i| create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) end + tubes.each_with_index do |tube, i| + tube.receptacle.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1}") + end end it 'is invalid' do @@ -345,7 +348,6 @@ let!(:plate) { create(:plate_with_tagged_wells, sample_count: 96, barcode: 'SQPD-12345') } let!(:asset_group) { create(:asset_group, name: 'assetgroup', study: study, assets: plate.wells) } let!(:library_type) { create(:library_type, name: 'Standard') } - let(:spreadsheet_filename) { 'scRNA_bulk_submission.csv' } let(:submission_template_hash) do { @@ -360,7 +362,13 @@ } end - before { SubmissionSerializer.construct!(submission_template_hash) } + before do + SubmissionSerializer.construct!(submission_template_hash) + # Assign a donor id to each sample + plate.wells.each_with_index do |well, i| + well.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1}") + end + end it 'is valid' do expect(subject).to be_valid @@ -402,9 +410,14 @@ tubes.each_with_index.map do |tube, i| create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) end + + tubes.each_with_index do |tube, i| + tube.receptacle.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1 }") + end end it 'is valid' do + # binding.pry expect(subject).to be_valid end From b5e5c4a41427248d3309c3dac5ae99c085dad4ff Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Tue, 17 Dec 2024 11:50:24 +0000 Subject: [PATCH 063/115] feat(studies): updates data release other option --- app/models/study.rb | 2 +- app/views/shared/metadata/_related_fields.html.erb | 2 +- features/studies/8447221_data_release_help_text.feature | 4 ++-- features/studies/data_release_timings.feature | 4 ++-- features/support/step_definitions/study_steps.rb | 2 +- spec/models/study_spec.rb | 5 ++++- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index dca73ee792..92ea477ce2 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -91,7 +91,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength 'Other (please specify)' ].freeze - DATA_RELEASE_DELAY_FOR_OTHER = 'other' + DATA_RELEASE_DELAY_FOR_OTHER = 'Other (with free text box)' DATA_RELEASE_DELAY_REASONS_STANDARD = [ 'PhD study', 'Capacity building', diff --git a/app/views/shared/metadata/_related_fields.html.erb b/app/views/shared/metadata/_related_fields.html.erb index d748e8d430..9d59fb9bdc 100644 --- a/app/views/shared/metadata/_related_fields.html.erb +++ b/app/views/shared/metadata/_related_fields.html.erb @@ -23,7 +23,7 @@ var valueFrom = function(element) { if (element === null) { return null } - var value = element.value.toLowerCase().replace(/[^a-z0-9]+/, '_'); + var value = element.value.toLowerCase().replaceAll(/[^a-z0-9]+/g, '_'); return (value.length == 0) ? 'blank' : value; }; diff --git a/features/studies/8447221_data_release_help_text.feature b/features/studies/8447221_data_release_help_text.feature index 603b2c0b4c..dd79263d3e 100644 --- a/features/studies/8447221_data_release_help_text.feature +++ b/features/studies/8447221_data_release_help_text.feature @@ -22,7 +22,7 @@ Feature: Update the data release fields for creating a study Scenario Outline: Add help text opposite delay drop down (4044305) When I choose "" from "What is the data release strategy for this study?" When I select "delayed" from "How is the data release to be timed?" - When I select "other" from "Reason for delaying release" + When I select "Other (with free text box)" from "Reason for delaying release" Then I should exactly see "Reason for delaying release" Examples: @@ -41,7 +41,7 @@ Feature: Update the data release fields for creating a study Scenario Outline: Delaying for 3 months should have the same questions as all other delays (4044273) When I select "delayed" from "How is the data release to be timed?" - And I select "other" from "Reason for delaying release" + And I select "Other (with free text box)" from "Reason for delaying release" And I select "" from "Delay for" And I should exactly see "Comment regarding data release timing and approval" diff --git a/features/studies/data_release_timings.feature b/features/studies/data_release_timings.feature index 4c0c2f3961..cc6562c638 100644 --- a/features/studies/data_release_timings.feature +++ b/features/studies/data_release_timings.feature @@ -37,7 +37,7 @@ Feature: Studies have timings for release of their data Scenario Outline: When the data release is delayed but no reasons are provided Given I select "delayed" from "How is the data release to be timed?" - And I select "other" from "Reason for delaying release" + And I select "Other (with free text box)" from "Reason for delaying release" And I fill in "Please explain the reason for delaying release" with "Some reason" And I select "" from "Delay for" When I press "Create" @@ -54,7 +54,7 @@ Feature: Studies have timings for release of their data Scenario Outline: When the data release is delayed and the reasons are provided Given I select "delayed" from "How is the data release to be timed?" - And I select "other" from "Reason for delaying release" + And I select "Other (with free text box)" from "Reason for delaying release" And I fill in "Please explain the reason for delaying release" with "Some reason" And I select "" from "Delay for" And I fill in "Comment regarding data release timing and approval" with "Because it is ok?" diff --git a/features/support/step_definitions/study_steps.rb b/features/support/step_definitions/study_steps.rb index 466e42627b..86f220fab3 100644 --- a/features/support/step_definitions/study_steps.rb +++ b/features/support/step_definitions/study_steps.rb @@ -159,7 +159,7 @@ def given_study_metadata(attribute, regexp) study.update!( study_metadata_attributes: { data_release_timing: 'delayed', - data_release_delay_reason: 'other', + data_release_delay_reason: 'Other (with free text box)', data_release_delay_other_comment: reason, data_release_delay_period: "#{period} months", data_release_delay_reason_comment: reason diff --git a/spec/models/study_spec.rb b/spec/models/study_spec.rb index a3e27ab54e..696170a97f 100644 --- a/spec/models/study_spec.rb +++ b/spec/models/study_spec.rb @@ -713,7 +713,10 @@ create( :study, study_metadata: - create(:study_metadata, metadata.merge(data_release_timing: 'delayed', data_release_delay_reason: 'other')) + create( + :study_metadata, + metadata.merge(data_release_timing: 'delayed', data_release_delay_reason: 'Other (with free text box)') + ) ) end From c6b5da8d71ec640c2f2722bca6133bdc08a43eef Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 15:26:05 +0000 Subject: [PATCH 064/115] Remove newlines from locale strings --- config/locales/en.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index f8835eefa1..3b3a65c8aa 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -176,13 +176,13 @@ en: # count is the total number of samples in the submission. # zero, one and other are the pluralization keys selected by the count of samples total_number_of_samples: - zero: | + zero: > The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, no samples were found. one: | The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, one sample was found. - other: | + other: > The total number of samples in the submission must be between %{min} and %{max} (inclusive). However, %{count} samples were found. # Validation: Total (requested) number of pools must be between 1 and 8 (inclusive). @@ -190,13 +190,13 @@ en: # count is the total number of pools in the submission. # zero, one and other are the pluralization keys selected by the count of pools total_number_of_pools: - zero: | + zero: > The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, no pools were found. - one: | + one: > The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, one pool was found. - other: | + other: > The total number of pools in the submission must be between %{min} and %{max} (inclusive). However, %{count} pools were found. # Validation: The number of pools requested must be feasible given the number of samples. @@ -204,24 +204,24 @@ en: # study_name and project_name are the values of respective fields from the spreadsheet. # zero, one and other are the pluralization keys selected by the count of samples in the pools. number_of_pools_by_samples: - zero: | + zero: > The number of pools is not feasible for the study %{study_name} and the project %{project_name}. - The number of samples in the #{size_type} pool must be between %{min} and %{max} (inclusive). + The number of samples in the %{size_type} pool must be between %{min} and %{max} (inclusive). However, no samples were found in the pool. - one: | + one: > The number of pools is not feasible for the study %{study_name} and the project %{project_name}. - The number of samples in the #{size_type} pool must be between %{min} and %{max} (inclusive). + The number of samples in the %{size_type} pool must be between %{min} and %{max} (inclusive). However, one sample was found in the pool. - other: | + other: > The number of pools is not feasible for the study %{study_name} and the project %{project_name}. - The number of samples in the #{size_type} pool must be between %{min} and %{max} (inclusive). + The number of samples in the %{size_type} pool must be between %{min} and %{max} (inclusive). However, %{count} samples were found in the pool. # Validation: The number of pools requested must be feasible having checked for donor clash. # study_name and project_name are the values of respective fields from the spreadsheet. # count is the size of the biggest group of samples with the same donor ID. # number_of_pools is the number of pools requested for the study and project. # barcodes_or_well_locations is the list of barcodes (tubes) or well locations (plates) with the same donor ID. - number_of_pools_by_donors: | + number_of_pools_by_donors: > The number of pools is not feasible for the study %{study_name} and the project %{project_name}. The number of samples with the same donor ID must be less than or equal to the requested number of pools. However, %{count} samples with the same donor ID were found, while the number of pools requested was %{number_of_pools}. From 7c93a5ebd54b0574e8646f8de50af47e280f5458 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 15:27:20 +0000 Subject: [PATCH 065/115] Add study_name and project_name to the add_error call --- .../scrna_core_cdna_prep_feasibility_validator.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index cba4d30548..493b7cb3af 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -53,6 +53,7 @@ def validate_scrna_core_cdna_prep_total_number_of_pools return if count.between?(min, max) # inclusive add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) + binding.pry end def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) @@ -70,7 +71,7 @@ def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength def validate_scrna_core_cdna_prep_feasibility_by_samples - group_rows_by_study_and_project.each_value do |rows| + group_rows_by_study_and_project.each do |(study_name, project_name), rows| barcodes, well_locations = extract_barcodes_and_well_locations(rows) number_of_samples = calculate_total_number_of_samples(barcodes, well_locations) number_of_pools = rows.first[headers.index(HEADER_NUMBER_OF_POOLS)].to_i @@ -79,7 +80,7 @@ def validate_scrna_core_cdna_prep_feasibility_by_samples max = scrna_config[:cdna_prep_maximum_number_of_samples_per_pool] pool_sizes.each do |size_type, pool_size| next if pool_size.between?(min, max) - add_error_scrna_core_cdna_prep_feasibility_by_samples(min, max, pool_size, size_type) + add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_name, min, max, pool_size, size_type) end end end @@ -92,11 +93,13 @@ def calculate_pool_size_types(number_of_samples, number_of_pools) { 'smallest' => smallest, 'biggest' => biggest } end - def add_error_scrna_core_cdna_prep_feasibility_by_samples(min, max, count, size_type) + def add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_name, min, max, count, size_type) errors.add( :spreadsheet, I18n.t( 'errors.number_of_pools_by_samples', + study_name: study_name, + project_name: project_name, min: min, max: max, count: count, From f399c836cf27e2b970b62ea35b339bed3ff62006 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 15:32:55 +0000 Subject: [PATCH 066/115] Prettier --- spec/models/bulk_submission_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index d41ba96382..f581380e7e 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -412,7 +412,7 @@ end tubes.each_with_index do |tube, i| - tube.receptacle.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1 }") + tube.receptacle.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1}") end end From 0d5e11d1431d509297302299712606c212ad2cf8 Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 15:34:39 +0000 Subject: [PATCH 067/115] Add tests for total number of samples --- ...re_cdna_prep_feasibility_validator_spec.rb | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb diff --git a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb new file mode 100644 index 0000000000..59bb9b6d17 --- /dev/null +++ b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb @@ -0,0 +1,275 @@ +# frozen_string_literal: true + +require 'rails_helper' + +# This spec tests the Submission::ScrnaCoreCdnaPrepFeasibilityValidator module +# included by the BulkSubmission class through the ValidationsByTemplateName +# module. CSV files are created for bulk submissions and the validations +# provided by the module are tested. +RSpec.describe BulkSubmission, with: :uploader do + # The CSV file headers have been copied form the headings in the config file + # config/bulk_submission_excel/worksheet/data_worksheet.rb + # The test subject is initialised with the uploaded file. + subject(:bulk_submission) { described_class.new(spreadsheet: submission_file) } + + let(:csv_headers) do + [ + 'User Login', + 'Template Name', + 'Project Name', + 'Study Name', + 'Submission name', + 'Barcode', + 'Plate Well', + 'Asset Group Name', + 'Fragment Size From', + 'Fragment Size To', + 'PCR Cycles', + 'Library Type', + 'Bait Library Name', + 'Pre-capture Plex Level', + 'Pre-capture Group', + 'Read Length', + 'Number of lanes', + 'Priority', + 'Primer Panel', + 'Comments', + 'Gigabases Expected', + 'Flowcell Type', + 'scRNA Core Number of Pools', + 'scRNA Core Cells per Chip Well' + ] + end + + # Defaults for the submission template and CSV data. + let(:template_name) { 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p' } + let(:scrna_core_number_of_pools) { 2 } # study-project + let(:scrna_core_cells_per_chip_well) { 90_000 } + # The request types are used to create the submission template record. + let!(:request_types) { [create(:pbmc_pooling_customer_request_type)] } + # User is created for the submission beforehand. + let!(:user) { create(:user, login: 'user1') } + # The submission template hash is used for creating the submission template + # record using the SubmissionSerializer.construct! call. + let(:submission_template_hash) do + { + name: template_name, + submission_class_name: 'LinearSubmission', + product_catalogue: 'scRNA Core', + submission_parameters: { + request_options: { + }, + request_types: request_types.map(&:key) + } + } + end + + # Each test context will specify different rows of data. + let(:csv_data) { [] } + + # CSV headers and data are combined to create the CSV content as string. + let(:csv_content) do + CSV.generate do |csv| + csv << csv_headers + csv_data.each { |row| csv << row } + end + end + + # A temporary file is created with the CSV content. + let(:csv_tempfile) do + # Use the Array form to enforce an extension in the filename. + # Use new to control the lifecycle of the temporary file. + Tempfile + .new(%w[donor_pooling_validator_test .csv]) + .tap do |tempfile| + tempfile.write(csv_content) + tempfile.rewind # to make it ready for reading from the beginning. + end + end + + # The path of the temporary file is assigned to the spreadsheet path, which + # is passed to the fixture_file_upload method to create an uploaded file + # in order to initialise the test subject. + let(:spreadsheet_path) { csv_tempfile.path } + + # The uploaded file is created using the fixture_file_upload method. + let(:submission_file) { fixture_file_upload(spreadsheet_path) } + + # Helper method to create a new CSV row with the defaults and specified values. + # The optional row_data hash argument is used to set the values corresponding + # to the headers. Unspecified fields will be set to nil but they can be set + # later directly on the row object. + # @example Create a new row with defaults, specified values, and more later + # row = new_csv_row('User Login' => 'user1', 'Barcode' => 'FX12345678') + # row['Study Name'] = 'Study 1' + # row['Project Name'] = 'Project 1' + # row.to_s + # => "user1,Limber-Htp - scRNA Core cDNA Prep GEM-X 5p,Project 1,Study 1,,FX12345678,,,,,,,,,,,,,,,,,10,90000\n" + # @param row_data [Hash] The values to be set in the new row: header => value. + # @return [CSV::Row] The new CSV row. + def new_csv_row(**row_data) + CSV::Row + .new(csv_headers, [nil] * csv_headers.size) + .tap do |row| + # Start with the defaults + data = { + 'User Login' => user.login, + 'Template Name' => template_name, + 'scRNA Core Number of Pools' => scrna_core_number_of_pools, + 'scRNA Core Cells per Chip Well' => scrna_core_cells_per_chip_well + } + # Merge specified row_data values. + data.merge!(row_data) + + # Set the values in the row. + data.each { |key, value| row[key] = value } + end + end + + # Helper method to get the scrna_config hash from the Rails application config. + # @return [Hash] The scrna_config hash. + def scrna_config + Rails.application.config.scrna_config + end + + before do + SubmissionSerializer.construct!(submission_template_hash) # Create the template. + end + + after do + csv_tempfile.close! # Close and unlink the temporary file after the test. + end + + context 'with total number of samples in the submission' do + # Total number of samples in the submission must be between 5 and 96 (inclusive). + + let(:group_1_number_of_samples) { 15 } # Study 1, Project 1 + let(:group_2_number_of_samples) { 15 } + let(:group_3_number_of_samples) { 15 } + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + let(:group_1_donors) { group_1_number_of_samples.times.map { |index| "group_1_donor_#{index + 1}" } } + + let(:group_2_donors) { group_2_number_of_samples.times.map { |index| "group_2_donor_#{index + 1}" } } + let(:group_3_donors) { group_3_number_of_samples.times.map { |index| "group_3_donor_#{index + 1}" } } + + let(:group_1_tubes) do + group_1_number_of_samples.times.map do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_1_donors[index]) } + end + end + + let(:group_2_tubes) do + group_2_number_of_samples.times.map do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_2_donors[index]) } + end + end + + let(:group_3_tubes) do + group_3_number_of_samples.times.map do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_3_donors[index]) } + end + end + + let(:group_1_rows) do + group_1_number_of_samples.times.map do |index| + new_csv_row( + 'Study Name' => 'Study 1', + 'Project Name' => 'Project 1', + 'Barcode' => group_1_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 1', + 'scRNA Core Number of Pools' => group_1_number_of_pools + ) + end + end + + let(:group_2_rows) do + group_2_number_of_samples.times.map do |index| + new_csv_row( + 'Study Name' => 'Study 2', + 'Project Name' => 'Project 2', + 'Barcode' => group_2_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 2', + 'scRNA Core Number of Pools' => group_2_number_of_pools + ) + end + end + + let(:group_3_rows) do + group_3_number_of_samples.times.map do |index| + new_csv_row( + 'Study Name' => 'Study 3', + 'Project Name' => 'Project 3', + 'Barcode' => group_3_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 3', + 'scRNA Core Number of Pools' => group_3_number_of_pools + ) + end + end + + let(:csv_data) { group_1_rows + group_2_rows + group_3_rows } + + context 'when the total number of samples is between minimum and maximum allowed' do + it { is_expected.to be_valid } + end + + context 'when the total number of samples is the minimum allowed' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 0 } + let(:group_3_number_of_samples) { 0 } + + it { is_expected.to be_valid } + end + + context 'when the total number of samples is the maximum allowed' do + let(:group_1_number_of_samples) { 31 } + let(:group_2_number_of_samples) { 32 } + let(:group_3_number_of_samples) { 33 } + + let(:group_1_number_of_pools) { 2 } + let(:group_2_number_of_pools) { 2 } + let(:group_3_number_of_pools) { 2 } + + it { is_expected.to be_valid } + end + + context 'when the total number of samples is less than the minimum allowed' do + let(:group_1_number_of_samples) { 4 } + let(:group_2_number_of_samples) { 0 } + let(:group_3_number_of_samples) { 0 } + + it 'sets the error message' do + scope = described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR + min = scrna_config[:cdna_prep_minimum_total_number_of_samples] + max = scrna_config[:cdna_prep_maximum_total_number_of_samples] + count = group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples + error_message = I18n.t('errors.total_number_of_samples', min: min, max: max, count: 4, scope: scope) + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + end + + context 'when the total number of samples is greater than the maximum allowed' do + let(:group_1_number_of_samples) { 32 } + let(:group_2_number_of_samples) { 32 } + let(:group_3_number_of_samples) { 33 } # 97 samples + + let(:group_1_number_of_pools) { 3 } + let(:group_2_number_of_pools) { 3 } + let(:group_3_number_of_pools) { 2 } + + it 'sets the error message' do + scope = described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR + min = scrna_config[:cdna_prep_minimum_total_number_of_samples] + max = scrna_config[:cdna_prep_maximum_total_number_of_samples] + count = group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples + error_message = I18n.t('errors.total_number_of_samples', min: min, max: max, count: 4, scope: scope) + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + end + end +end From 581fe4085e4b98eb604c98c87000904849c3787b Mon Sep 17 00:00:00 2001 From: yoldas Date: Tue, 17 Dec 2024 16:14:49 +0000 Subject: [PATCH 068/115] Rubocop --- ...na_core_cdna_prep_feasibility_validator.rb | 3 +- spec/models/bulk_submission_spec.rb | 1 - ...re_cdna_prep_feasibility_validator_spec.rb | 53 ++++++++++++------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 493b7cb3af..70cb964f07 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -53,7 +53,6 @@ def validate_scrna_core_cdna_prep_total_number_of_pools return if count.between?(min, max) # inclusive add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) - binding.pry end def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) @@ -93,6 +92,7 @@ def calculate_pool_size_types(number_of_samples, number_of_pools) { 'smallest' => smallest, 'biggest' => biggest } end + # rubocop:disable Metrics/ParameterLists def add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_name, min, max, count, size_type) errors.add( :spreadsheet, @@ -108,6 +108,7 @@ def add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_na ) ) end + # rubocop:enable Metrics/ParameterLists # rubocop:disable Metrics/MethodLength def validate_scrna_core_cdna_prep_feasibility_by_donors diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index f581380e7e..c00a8f84de 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -417,7 +417,6 @@ end it 'is valid' do - # binding.pry expect(subject).to be_valid end diff --git a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb index 59bb9b6d17..d860ad93cb 100644 --- a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb +++ b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb @@ -151,31 +151,30 @@ def scrna_config let(:group_2_number_of_pools) { 1 } let(:group_3_number_of_pools) { 1 } - let(:group_1_donors) { group_1_number_of_samples.times.map { |index| "group_1_donor_#{index + 1}" } } - - let(:group_2_donors) { group_2_number_of_samples.times.map { |index| "group_2_donor_#{index + 1}" } } - let(:group_3_donors) { group_3_number_of_samples.times.map { |index| "group_3_donor_#{index + 1}" } } + let(:group_1_donors) { Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } } + let(:group_2_donors) { Array.new(group_2_number_of_samples) { |index| "group_2_donor_#{index + 1}" } } + let(:group_3_donors) { Array.new(group_3_number_of_samples) { |index| "group_3_donor_#{index + 1}" } } let(:group_1_tubes) do - group_1_number_of_samples.times.map do |index| + Array.new(group_1_number_of_samples) do |index| create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_1_donors[index]) } end end let(:group_2_tubes) do - group_2_number_of_samples.times.map do |index| + Array.new(group_2_number_of_samples) do |index| create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_2_donors[index]) } end end let(:group_3_tubes) do - group_3_number_of_samples.times.map do |index| + Array.new(group_3_number_of_samples) do |index| create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_3_donors[index]) } end end let(:group_1_rows) do - group_1_number_of_samples.times.map do |index| + Array.new(group_1_number_of_samples) do |index| new_csv_row( 'Study Name' => 'Study 1', 'Project Name' => 'Project 1', @@ -187,7 +186,7 @@ def scrna_config end let(:group_2_rows) do - group_2_number_of_samples.times.map do |index| + Array.new(group_2_number_of_samples) do |index| new_csv_row( 'Study Name' => 'Study 2', 'Project Name' => 'Project 2', @@ -199,7 +198,7 @@ def scrna_config end let(:group_3_rows) do - group_3_number_of_samples.times.map do |index| + Array.new(group_3_number_of_samples) do |index| new_csv_row( 'Study Name' => 'Study 3', 'Project Name' => 'Project 3', @@ -212,6 +211,10 @@ def scrna_config let(:csv_data) { group_1_rows + group_2_rows + group_3_rows } + let(:i18n_scope) { described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR } + + let(:total_number_of_samples) { group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples } + context 'when the total number of samples is between minimum and maximum allowed' do it { is_expected.to be_valid } end @@ -241,15 +244,20 @@ def scrna_config let(:group_2_number_of_samples) { 0 } let(:group_3_number_of_samples) { 0 } + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations it 'sets the error message' do - scope = described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR - min = scrna_config[:cdna_prep_minimum_total_number_of_samples] - max = scrna_config[:cdna_prep_maximum_total_number_of_samples] - count = group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples - error_message = I18n.t('errors.total_number_of_samples', min: min, max: max, count: 4, scope: scope) + error_message = + I18n.t( + 'errors.total_number_of_samples', + min: scrna_config[:cdna_prep_minimum_total_number_of_samples], + max: scrna_config[:cdna_prep_maximum_total_number_of_samples], + count: total_number_of_samples, + scope: i18n_scope + ) expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) expect(bulk_submission.errors[:spreadsheet]).to include(error_message) end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations end context 'when the total number of samples is greater than the maximum allowed' do @@ -261,15 +269,20 @@ def scrna_config let(:group_2_number_of_pools) { 3 } let(:group_3_number_of_pools) { 2 } + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations it 'sets the error message' do - scope = described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR - min = scrna_config[:cdna_prep_minimum_total_number_of_samples] - max = scrna_config[:cdna_prep_maximum_total_number_of_samples] - count = group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples - error_message = I18n.t('errors.total_number_of_samples', min: min, max: max, count: 4, scope: scope) + error_message = + I18n.t( + 'errors.total_number_of_samples', + min: scrna_config[:cdna_prep_minimum_total_number_of_samples], + max: scrna_config[:cdna_prep_maximum_total_number_of_samples], + count: total_number_of_samples, + scope: i18n_scope + ) expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) expect(bulk_submission.errors[:spreadsheet]).to include(error_message) end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations end end end From 27530dec591df95acf0b12cec3e712fbc23e1b3a Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 19 Dec 2024 01:26:08 +0000 Subject: [PATCH 069/115] Add tests for total number of pools --- ...re_cdna_prep_feasibility_validator_spec.rb | 240 +++++++++++++----- 1 file changed, 171 insertions(+), 69 deletions(-) diff --git a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb index d860ad93cb..7d16283d37 100644 --- a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb +++ b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb @@ -7,11 +7,11 @@ # module. CSV files are created for bulk submissions and the validations # provided by the module are tested. RSpec.describe BulkSubmission, with: :uploader do - # The CSV file headers have been copied form the headings in the config file - # config/bulk_submission_excel/worksheet/data_worksheet.rb # The test subject is initialised with the uploaded file. subject(:bulk_submission) { described_class.new(spreadsheet: submission_file) } + # The CSV headers are used to create the CSV content; copied from headings in + # config/bulk_submission_excel/columns.yml let(:csv_headers) do [ 'User Login', @@ -45,10 +45,13 @@ let(:template_name) { 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p' } let(:scrna_core_number_of_pools) { 2 } # study-project let(:scrna_core_cells_per_chip_well) { 90_000 } + # The request types are used to create the submission template record. let!(:request_types) { [create(:pbmc_pooling_customer_request_type)] } + # User is created for the submission beforehand. let!(:user) { create(:user, login: 'user1') } + # The submission template hash is used for creating the submission template # record using the SubmissionSerializer.construct! call. let(:submission_template_hash) do @@ -64,10 +67,7 @@ } end - # Each test context will specify different rows of data. - let(:csv_data) { [] } - - # CSV headers and data are combined to create the CSV content as string. + # CSV headers and CSV data are combined to create the CSV content as string. let(:csv_content) do CSV.generate do |csv| csv << csv_headers @@ -98,8 +98,8 @@ # Helper method to create a new CSV row with the defaults and specified values. # The optional row_data hash argument is used to set the values corresponding # to the headers. Unspecified fields will be set to nil but they can be set - # later directly on the row object. - # @example Create a new row with defaults, specified values, and more later + # later directly on the row object. The nil fields are rendered as empty cells. + # @example Create a new row with defaults, specified values, and more later. # row = new_csv_row('User Login' => 'user1', 'Barcode' => 'FX12345678') # row['Study Name'] = 'Study 1' # row['Project Name'] = 'Project 1' @@ -140,80 +140,91 @@ def scrna_config csv_tempfile.close! # Close and unlink the temporary file after the test. end - context 'with total number of samples in the submission' do - # Total number of samples in the submission must be between 5 and 96 (inclusive). - - let(:group_1_number_of_samples) { 15 } # Study 1, Project 1 - let(:group_2_number_of_samples) { 15 } - let(:group_3_number_of_samples) { 15 } - - let(:group_1_number_of_pools) { 1 } - let(:group_2_number_of_pools) { 1 } - let(:group_3_number_of_pools) { 1 } - - let(:group_1_donors) { Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } } - let(:group_2_donors) { Array.new(group_2_number_of_samples) { |index| "group_2_donor_#{index + 1}" } } - let(:group_3_donors) { Array.new(group_3_number_of_samples) { |index| "group_3_donor_#{index + 1}" } } - - let(:group_1_tubes) do - Array.new(group_1_number_of_samples) do |index| - create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_1_donors[index]) } - end + # Three study-project groups with different number of samples. + let(:group_1_number_of_samples) { 15 } # Study 1, Project 1 + let(:group_2_number_of_samples) { 15 } + let(:group_3_number_of_samples) { 15 } + + # The number of pools for each study-project group. + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + # Donor IDs for each group to set on the samples. + let(:group_1_donors) { Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } } + let(:group_2_donors) { Array.new(group_2_number_of_samples) { |index| "group_2_donor_#{index + 1}" } } + let(:group_3_donors) { Array.new(group_3_number_of_samples) { |index| "group_3_donor_#{index + 1}" } } + + # Tubes for each group with sample_metadata that contains the donor ID. + let(:group_1_tubes) do + Array.new(group_1_number_of_samples) do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_1_donors[index]) } end + end - let(:group_2_tubes) do - Array.new(group_2_number_of_samples) do |index| - create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_2_donors[index]) } - end + let(:group_2_tubes) do + Array.new(group_2_number_of_samples) do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_2_donors[index]) } end + end - let(:group_3_tubes) do - Array.new(group_3_number_of_samples) do |index| - create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_3_donors[index]) } - end + let(:group_3_tubes) do + Array.new(group_3_number_of_samples) do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_3_donors[index]) } end + end - let(:group_1_rows) do - Array.new(group_1_number_of_samples) do |index| - new_csv_row( - 'Study Name' => 'Study 1', - 'Project Name' => 'Project 1', - 'Barcode' => group_1_tubes[index].human_barcode, - 'Asset Group Name' => 'Asset Group 1', - 'scRNA Core Number of Pools' => group_1_number_of_pools - ) - end + # CSV rows for each group with the specified values. + let(:group_1_rows) do + Array.new(group_1_number_of_samples) do |index| + new_csv_row( + 'Study Name' => 'Study 1', + 'Project Name' => 'Project 1', + 'Barcode' => group_1_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 1', + 'scRNA Core Number of Pools' => group_1_number_of_pools + ) end + end - let(:group_2_rows) do - Array.new(group_2_number_of_samples) do |index| - new_csv_row( - 'Study Name' => 'Study 2', - 'Project Name' => 'Project 2', - 'Barcode' => group_2_tubes[index].human_barcode, - 'Asset Group Name' => 'Asset Group 2', - 'scRNA Core Number of Pools' => group_2_number_of_pools - ) - end + let(:group_2_rows) do + Array.new(group_2_number_of_samples) do |index| + new_csv_row( + 'Study Name' => 'Study 2', + 'Project Name' => 'Project 2', + 'Barcode' => group_2_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 2', + 'scRNA Core Number of Pools' => group_2_number_of_pools + ) end + end - let(:group_3_rows) do - Array.new(group_3_number_of_samples) do |index| - new_csv_row( - 'Study Name' => 'Study 3', - 'Project Name' => 'Project 3', - 'Barcode' => group_3_tubes[index].human_barcode, - 'Asset Group Name' => 'Asset Group 3', - 'scRNA Core Number of Pools' => group_3_number_of_pools - ) - end + let(:group_3_rows) do + Array.new(group_3_number_of_samples) do |index| + new_csv_row( + 'Study Name' => 'Study 3', + 'Project Name' => 'Project 3', + 'Barcode' => group_3_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 3', + 'scRNA Core Number of Pools' => group_3_number_of_pools + ) end + end - let(:csv_data) { group_1_rows + group_2_rows + group_3_rows } + # Combine the rows from all groups to create the CSV data. + let(:csv_data) { group_1_rows + group_2_rows + group_3_rows } - let(:i18n_scope) { described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR } + # The I18n scope for the error messages in the locale file. + let(:i18n_scope) { described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR } - let(:total_number_of_samples) { group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples } + # The total number of samples in the submission. + let(:total_number_of_samples) { group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples } + + # The total number of pools in the submission. + let(:total_number_of_pools) { group_1_number_of_pools + group_2_number_of_pools + group_3_number_of_pools } + + context '#validate_scrna_core_cdna_prep_total_number_of_samples' do + # Total number of samples in the submission must be between 5 and 96 (inclusive). context 'when the total number of samples is between minimum and maximum allowed' do it { is_expected.to be_valid } @@ -263,7 +274,7 @@ def scrna_config context 'when the total number of samples is greater than the maximum allowed' do let(:group_1_number_of_samples) { 32 } let(:group_2_number_of_samples) { 32 } - let(:group_3_number_of_samples) { 33 } # 97 samples + let(:group_3_number_of_samples) { 33 } let(:group_1_number_of_pools) { 3 } let(:group_2_number_of_pools) { 3 } @@ -285,4 +296,95 @@ def scrna_config # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations end end + + context '#validate_scrna_core_cdna_prep_total_number_of_pools' do + # Total (requested) number of pools must be between 1 and 8 (inclusive) + + context 'when the total number of pools is between minimum and maximum allowed' do + it { is_expected.to be_valid } + end + + context 'when the total number of pools is the minimum allowed' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 0 } + let(:group_3_number_of_samples) { 0 } + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 0 } + let(:group_3_number_of_pools) { 0 } + + it { is_expected.to be_valid } + end + + context 'when the total number of pools is the maximum allowed' do + let(:group_1_number_of_samples) { 31 } + let(:group_2_number_of_samples) { 32 } + let(:group_3_number_of_samples) { 33 } + + let(:group_1_number_of_pools) { 3 } + let(:group_2_number_of_pools) { 3 } + let(:group_3_number_of_pools) { 2 } + + it { is_expected.to be_valid } + end + + context 'when the total number of pools is less than the minimum allowed' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 0 } + let(:group_3_number_of_samples) { 0 } + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 0 } + let(:group_3_number_of_pools) { 0 } + + before do + # Because the allowed range for the total number of pools is configured + # as 1 to 8 (inclusive), testing it below the minimum value requires + # setting a zero number of pools, which causes a ZeroDivisionError + # before the feasibility validation. In order to test the error message, + # we will use a different range; 2 to 8 instead of 1 to 8. We will stub + # the scrna_config call on the Rails.application.config to return the + # modified scrna_config. + scrna_config_dup = Rails.application.config.scrna_config.dup + scrna_config_dup[:cdna_prep_minimum_total_number_of_pools] = 2 + allow(Rails.application.config).to receive(:scrna_config).and_return(scrna_config_dup) + end + + it 'sets the error message' do + error_message = + I18n.t( + 'errors.total_number_of_pools', + min: scrna_config[:cdna_prep_minimum_total_number_of_pools], + max: scrna_config[:cdna_prep_maximum_total_number_of_pools], + count: total_number_of_pools, + scope: i18n_scope + ) + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + end + + context 'when the total number of pools is greater than the maximum allowed' do + let(:group_1_number_of_samples) { 31 } + let(:group_2_number_of_samples) { 32 } + let(:group_3_number_of_samples) { 33 } + + let(:group_1_number_of_pools) { 3 } + let(:group_2_number_of_pools) { 3 } + let(:group_3_number_of_pools) { 3 } + + it 'sets the error message' do + error_message = + I18n.t( + 'errors.total_number_of_pools', + min: scrna_config[:cdna_prep_minimum_total_number_of_pools], + max: scrna_config[:cdna_prep_maximum_total_number_of_pools], + count: total_number_of_pools, + scope: i18n_scope + ) + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + end + end end From a2570d63cc072191505af49fe8d8be06374f561f Mon Sep 17 00:00:00 2001 From: yoldas Date: Fri, 20 Dec 2024 00:17:50 +0000 Subject: [PATCH 070/115] Avoid repeating the same error if the smallest and biggest pool sizes are equal because the remainder is zero --- .../scrna_core_cdna_prep_feasibility_validator.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 70cb964f07..59c6ea686b 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -87,9 +87,9 @@ def validate_scrna_core_cdna_prep_feasibility_by_samples def calculate_pool_size_types(number_of_samples, number_of_pools) quotient, remainder = number_of_samples.divmod(number_of_pools) - smallest = biggest = quotient - biggest += 1 if remainder.positive? - { 'smallest' => smallest, 'biggest' => biggest } + size_types = { 'smallest' => quotient } + size_types['biggest'] = quotient + 1 if remainder.positive? + size_types end # rubocop:disable Metrics/ParameterLists From c7050bc7e40d6d4870a5881d336872073a7f8ac0 Mon Sep 17 00:00:00 2001 From: yoldas Date: Fri, 20 Dec 2024 00:19:44 +0000 Subject: [PATCH 071/115] Add tests for the feasibility by samples and feasibility by donors validations --- ...re_cdna_prep_feasibility_validator_spec.rb | 392 ++++++++++++++---- 1 file changed, 303 insertions(+), 89 deletions(-) diff --git a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb index 7d16283d37..dfd78b3833 100644 --- a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb +++ b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb @@ -40,10 +40,82 @@ 'scRNA Core Cells per Chip Well' ] end + # Three study-project groups with different number of samples. + # Note that setting a positive number of samples for a study-project group + # will enable it for a test in a context; setting it to zero will disable it. + let(:group_1_number_of_samples) { 15 } # Study 1, Project 1 + let(:group_2_number_of_samples) { 15 } + let(:group_3_number_of_samples) { 15 } + # The number of pools for each study-project group. + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + # Donor IDs for each group to set on the samples. + let(:group_1_donors) { Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } } + let(:group_2_donors) { Array.new(group_2_number_of_samples) { |index| "group_2_donor_#{index + 1}" } } + let(:group_3_donors) { Array.new(group_3_number_of_samples) { |index| "group_3_donor_#{index + 1}" } } + # Tubes for each group with sample_metadata that contains the donor ID. + let(:group_1_tubes) do + Array.new(group_1_number_of_samples) do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_1_donors[index]) } + end + end + let(:group_2_tubes) do + Array.new(group_2_number_of_samples) do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_2_donors[index]) } + end + end + let(:group_3_tubes) do + Array.new(group_3_number_of_samples) do |index| + create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_3_donors[index]) } + end + end + # CSV rows for each group with the specified values. + let(:group_1_rows) do + Array.new(group_1_number_of_samples) do |index| + new_csv_row( + 'Study Name' => 'Study 1', + 'Project Name' => 'Project 1', + 'Barcode' => group_1_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 1', + 'scRNA Core Number of Pools' => group_1_number_of_pools + ) + end + end + let(:group_2_rows) do + Array.new(group_2_number_of_samples) do |index| + new_csv_row( + 'Study Name' => 'Study 2', + 'Project Name' => 'Project 2', + 'Barcode' => group_2_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 2', + 'scRNA Core Number of Pools' => group_2_number_of_pools + ) + end + end + let(:group_3_rows) do + Array.new(group_3_number_of_samples) do |index| + new_csv_row( + 'Study Name' => 'Study 3', + 'Project Name' => 'Project 3', + 'Barcode' => group_3_tubes[index].human_barcode, + 'Asset Group Name' => 'Asset Group 3', + 'scRNA Core Number of Pools' => group_3_number_of_pools + ) + end + end + # Combine the rows from all groups to create the CSV data. + let(:csv_data) { group_1_rows + group_2_rows + group_3_rows } + # The I18n scope for the error messages in the locale file. + let(:i18n_scope) { described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR } + # The total number of samples in the submission. + let(:total_number_of_samples) { group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples } + # The total number of pools in the submission. + let(:total_number_of_pools) { group_1_number_of_pools + group_2_number_of_pools + group_3_number_of_pools } # Defaults for the submission template and CSV data. let(:template_name) { 'Limber-Htp - scRNA Core cDNA Prep GEM-X 5p' } - let(:scrna_core_number_of_pools) { 2 } # study-project + let(:scrna_core_number_of_pools) { 1 } # study-project let(:scrna_core_cells_per_chip_well) { 90_000 } # The request types are used to create the submission template record. @@ -140,97 +212,19 @@ def scrna_config csv_tempfile.close! # Close and unlink the temporary file after the test. end - # Three study-project groups with different number of samples. - let(:group_1_number_of_samples) { 15 } # Study 1, Project 1 - let(:group_2_number_of_samples) { 15 } - let(:group_3_number_of_samples) { 15 } - - # The number of pools for each study-project group. - let(:group_1_number_of_pools) { 1 } - let(:group_2_number_of_pools) { 1 } - let(:group_3_number_of_pools) { 1 } - - # Donor IDs for each group to set on the samples. - let(:group_1_donors) { Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } } - let(:group_2_donors) { Array.new(group_2_number_of_samples) { |index| "group_2_donor_#{index + 1}" } } - let(:group_3_donors) { Array.new(group_3_number_of_samples) { |index| "group_3_donor_#{index + 1}" } } - - # Tubes for each group with sample_metadata that contains the donor ID. - let(:group_1_tubes) do - Array.new(group_1_number_of_samples) do |index| - create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_1_donors[index]) } - end - end - - let(:group_2_tubes) do - Array.new(group_2_number_of_samples) do |index| - create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_2_donors[index]) } - end - end - - let(:group_3_tubes) do - Array.new(group_3_number_of_samples) do |index| - create(:sample_tube).tap { |tube| tube.samples.first.sample_metadata.update!(donor_id: group_3_donors[index]) } - end - end - - # CSV rows for each group with the specified values. - let(:group_1_rows) do - Array.new(group_1_number_of_samples) do |index| - new_csv_row( - 'Study Name' => 'Study 1', - 'Project Name' => 'Project 1', - 'Barcode' => group_1_tubes[index].human_barcode, - 'Asset Group Name' => 'Asset Group 1', - 'scRNA Core Number of Pools' => group_1_number_of_pools - ) - end - end - - let(:group_2_rows) do - Array.new(group_2_number_of_samples) do |index| - new_csv_row( - 'Study Name' => 'Study 2', - 'Project Name' => 'Project 2', - 'Barcode' => group_2_tubes[index].human_barcode, - 'Asset Group Name' => 'Asset Group 2', - 'scRNA Core Number of Pools' => group_2_number_of_pools - ) - end - end - - let(:group_3_rows) do - Array.new(group_3_number_of_samples) do |index| - new_csv_row( - 'Study Name' => 'Study 3', - 'Project Name' => 'Project 3', - 'Barcode' => group_3_tubes[index].human_barcode, - 'Asset Group Name' => 'Asset Group 3', - 'scRNA Core Number of Pools' => group_3_number_of_pools - ) - end - end - - # Combine the rows from all groups to create the CSV data. - let(:csv_data) { group_1_rows + group_2_rows + group_3_rows } - - # The I18n scope for the error messages in the locale file. - let(:i18n_scope) { described_class::I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR } - - # The total number of samples in the submission. - let(:total_number_of_samples) { group_1_number_of_samples + group_2_number_of_samples + group_3_number_of_samples } - - # The total number of pools in the submission. - let(:total_number_of_pools) { group_1_number_of_pools + group_2_number_of_pools + group_3_number_of_pools } - - context '#validate_scrna_core_cdna_prep_total_number_of_samples' do + describe '#validate_scrna_core_cdna_prep_total_number_of_samples' do # Total number of samples in the submission must be between 5 and 96 (inclusive). - context 'when the total number of samples is between minimum and maximum allowed' do + context 'when the total number of samples is between the minimum and maximum allowed' do + let(:group_1_number_of_samples) { 15 } + let(:group_2_number_of_samples) { 16 } + let(:group_3_number_of_samples) { 17 } + it { is_expected.to be_valid } end context 'when the total number of samples is the minimum allowed' do + # This is possible with a single study-project group with 5 samples. let(:group_1_number_of_samples) { 5 } let(:group_2_number_of_samples) { 0 } let(:group_3_number_of_samples) { 0 } @@ -297,10 +291,17 @@ def scrna_config end end - context '#validate_scrna_core_cdna_prep_total_number_of_pools' do + describe '#validate_scrna_core_cdna_prep_total_number_of_pools' do # Total (requested) number of pools must be between 1 and 8 (inclusive) - - context 'when the total number of pools is between minimum and maximum allowed' do + # Calculating total number of pools + # Split the rows into study-project groups. + # The 'number of pools' applies at that level - grab one number for this + # column for each of those groups and add them up. + # e.g. Study A-Project A asks for 1 pool, Study A-Project B asks for 2 + # pools, Study C-Project C asks for 5 pools --> total pools is + # 1 + 2 + 5 = 8 --> passes + + context 'when the total number of pools is between the minimum and maximum allowed' do it { is_expected.to be_valid } end @@ -350,6 +351,7 @@ def scrna_config allow(Rails.application.config).to receive(:scrna_config).and_return(scrna_config_dup) end + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations it 'sets the error message' do error_message = I18n.t( @@ -362,6 +364,7 @@ def scrna_config expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) expect(bulk_submission.errors[:spreadsheet]).to include(error_message) end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations end context 'when the total number of pools is greater than the maximum allowed' do @@ -373,6 +376,7 @@ def scrna_config let(:group_2_number_of_pools) { 3 } let(:group_3_number_of_pools) { 3 } + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations it 'sets the error message' do error_message = I18n.t( @@ -385,6 +389,216 @@ def scrna_config expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) expect(bulk_submission.errors[:spreadsheet]).to include(error_message) end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations + end + end + + describe '#validate_scrna_core_cdna_prep_feasibility_by_samples' do + # The number of pools requested must be feasible given the number of samples. + # Checking if the number of pools is feasible, given number of samples + # For each study-project group: + # Smallest pool = Number of samples / number of pools (floor division) + # Biggest pool = Smallest pool + 1 (if remainder is positive) + # Check both smallest pool and biggest pool are between 5 and 25 (inclusive). + # Note that only the smallest pool is validated in the following cases: + # - if the remainder is zero; the smallest and biggest pool sizes are equal. + # - if the number of pools is one; there is only one pool. + context 'when the pool sizes are between the minimum and maximum allowed' do + let(:group_1_number_of_samples) { 10 } + let(:group_2_number_of_samples) { 30 } + let(:group_3_number_of_samples) { 56 } + + let(:group_1_number_of_pools) { 1 } # 10: (5 < 10 < 25) + let(:group_2_number_of_pools) { 2 } # 15, 15:(5 < 15 < 25) + let(:group_3_number_of_pools) { 3 } # 18, 19, 19: (5 < 18 < 25) and (5 < 19 < 25) + + it { is_expected.to be_valid } + end + + context 'when the smallest pool size is the minimum allowed' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 11 } + let(:group_3_number_of_samples) { 29 } + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 2 } + let(:group_3_number_of_pools) { 5 } + + it { is_expected.to be_valid } + end + + context 'when the biggest pool size is the maximum allowed' do + let(:group_1_number_of_samples) { 25 } + let(:group_2_number_of_samples) { 49 } + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 2 } + let(:group_3_number_of_pools) { 1 } + + it { is_expected.to be_valid } + end + + context 'when the smallest pool size is less than the minimum allowed' do + let(:group_1_number_of_samples) { 4 } # smallest = biggest < 5 + let(:group_2_number_of_samples) { 9 } # smallest < 5, biggest: OK + let(:group_3_number_of_samples) { 7 } # smallest < 5, biggest < 5 + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 2 } + let(:group_3_number_of_pools) { 2 } + + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations + it 'sets the error message' do + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + + # Parametrise the assertions for the error messages because multiple + # error messages are expected for different study-project groups and + # pool sizes (smallest and biggest). + + # study name, project name, pool size, size type + params = [ + ['Study 1', 'Project 1', 4, 'smallest'], + ['Study 2', 'Project 2', 4, 'smallest'], + ['Study 3', 'Project 3', 3, 'smallest'], + ['Study 3', 'Project 3', 4, 'biggest'] + ] + params.each do |study_name, project_name, pool_size, size_type| + error_message = + I18n.t( + 'errors.number_of_pools_by_samples', + study_name: study_name, + project_name: project_name, + min: scrna_config[:cdna_prep_minimum_number_of_samples_per_pool], + max: scrna_config[:cdna_prep_maximum_number_of_samples_per_pool], + count: pool_size, + size_type: size_type, # smallest or biggest + scope: i18n_scope + ) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations + end + + context 'when the biggest pool size is greater than the maximum allowed' do + let(:group_1_number_of_samples) { 51 } # smallest:OK, biggest > 25 + let(:group_2_number_of_samples) { 0 } + let(:group_3_number_of_samples) { 0 } + + let(:group_1_number_of_pools) { 2 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations + it 'sets the error message' do + error_message = + I18n.t( + 'errors.number_of_pools_by_samples', + study_name: 'Study 1', + project_name: 'Project 1', + min: scrna_config[:cdna_prep_minimum_number_of_samples_per_pool], + max: scrna_config[:cdna_prep_maximum_number_of_samples_per_pool], + count: 26, + size_type: 'biggest', + scope: i18n_scope + ) + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations + end + end + + describe '#validate_scrna_core_cdna_prep_feasibility_by_donors' do + # The number of pools requested must be feasible having checked for donor clash. + # Checking for donor clash + # For each study-project group: + # Group the samples by their donor id. + # Find the biggest group (will be 1 if all samples have unique donor ids) + # Check size of biggest group <= requested number of pools. + context 'when the number of samples with the same donor ID is less than ' \ + 'or equal to the requested number of pools for study and project' do + let(:group_1_number_of_samples) { 96 } + let(:group_2_number_of_samples) { 0 } # not included in the test + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 4 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + let(:group_1_donors) do + donors = Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } + donors[0..2] = [donors[0]] * 3 # samples with the same donor ID + donors + end + + # We can put the first 3 samples into separate pools to avoid donor clash, + # because we have 4 pools. + it { is_expected.to be_valid } + end + + context 'when the number of samples with the same donor ID is equal to ' \ + 'the requested number of pools for study and project' do + let(:group_1_number_of_samples) { 96 } + let(:group_2_number_of_samples) { 0 } # not included in the test + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 4 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + let(:group_1_donors) do + donors = Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } + donors[0..3] = [donors[0]] * 4 # samples with the same donor ID + donors + end + + # We can put the first 4 samples into separate pools to avoid donor clash, + # because we have 4 pools. + it { is_expected.to be_valid } + end + + context 'when the number of samples with the same donor ID is greater ' \ + 'than the requested number of pools for study and project' do + let(:group_1_number_of_samples) { 96 } + let(:group_2_number_of_samples) { 0 } # not included in the test + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 4 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + let(:group_1_donors) do + donors = Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } + donors[0..4] = [donors[0]] * 5 # samples with the same donor ID + donors + end + + # We cannot put the first 5 samples into separate pools to avoid donor + # clash, because we have 4 pools. + + # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations + it 'sets the error message' do + # Barcodes or well locations of the labware with the same donor ID are + # listed in the error message. This test uses tubes; hence the barcodes + # of the tubes will be listed to help the user identify the samples. + barcodes_or_well_locations = group_1_tubes[0..4].map(&:human_barcode).join(', ') + + error_message = + I18n.t( + 'errors.number_of_pools_by_donors', + study_name: 'Study 1', + project_name: 'Project 1', + count: 5, # biggest donor group size + number_of_pools: group_1_number_of_pools, # requested number of pools + barcodes_or_well_locations: barcodes_or_well_locations, + scope: i18n_scope + ) + expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) + expect(bulk_submission.errors[:spreadsheet]).to include(error_message) + end + # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations end end end From dd9fa1fbfe9d0d527b6c837d6ae0fc3b83c7ec28 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sat, 21 Dec 2024 16:10:36 +0000 Subject: [PATCH 072/115] Added the full allowance warning message to the locale file --- config/locales/en.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index 3b3a65c8aa..f7897b849e 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -226,6 +226,16 @@ en: The number of samples with the same donor ID must be less than or equal to the requested number of pools. However, %{count} samples with the same donor ID were found, while the number of pools requested was %{number_of_pools}. The samples with the same donor ID follow: %{barcodes_or_well_locations}. + warnings: + # Validation: There is not enough material for the "full allowance" (2 full runs on the chip) + # study_name and project_name are the values of respective fields from the spreadsheet. + # number_of_samples_in_smallest_pool is the number of samples in the smallest pool. + # final_resuspension_volume is the final resuspension volume given the number of samples in the smallest pool for the smallest pool. + # full_allowance is the full allowance given the number of cells per chip well for the study and project. + full_allowance: > + There is not enough material for the "full allowance" (2 full runs on the chip) for the study %{study_name} and the project %{project_name}. + The final resuspension volume for the smallest pool should be greater than or equal to the full allowance. + However, for the smallest pool of size %{number_of_samples_in_smallest_pool}, the final resuspension volume is %{final_resuspension_volume} and the full allowance is %{full_allowance}. cherrypick: picking_by_row: "This cherrypick may take longer as it is picking by rows, rather than columns." From 6c8762a633543c6924409909fc358b9018b1af9f Mon Sep 17 00:00:00 2001 From: yoldas Date: Sat, 21 Dec 2024 16:12:35 +0000 Subject: [PATCH 073/115] Add cDNA Prep feasibility calculator module for full allowance calculations --- ...a_core_cdna_prep_feasibility_calculator.rb | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb new file mode 100644 index 0000000000..6fe11254fe --- /dev/null +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Submission::ScrnaCoreCdnaPrepFeasibilityCalculator + + def calculate_full_allowance(number_of_cells_per_chip_well) + # "Full allowance" = ( "Chip loading volume" * 2) + 25 + # 2 is because this is for 2 runs + # 25 is 2 lots of 10ul for cell counting, and 5ul for wastage when transferring between labware + chip_loading_volume = calculate_chip_loading_volume(number_of_cells_per_chip_well) + (chip_loading_volume * scrna_config[:desired_number_of_runs]) + + (scrna_config[:desired_number_of_runs] * scrna_config[:volume_taken_for_cell_counting]) + + scrna_config[:wastage_volume] + end + + + def calculate_chip_loading_volume(number_of_cells_per_chip_well) + # "Chip loading volume" = "Number of cells per chip well" / "Chip loading concentration" + number_of_cells_per_chip_well / scrna_config[:desired_chip_loading_concentration] + end + + def calculate_resuspension_volume(count_of_samples_in_pool) + total_cells_in_300ul = calculate_total_cells_in_300ul(count_of_samples_in_pool) + total_cells_in_300ul / scrna_config[:desired_chip_loading_concentration] + end + + def calculate_total_cells_in_300ul(count_of_samples_in_pool) + (count_of_samples_in_pool * scrna_config[:required_number_of_cells_per_sample_in_pool]) * + scrna_config[:wastage_factor] + end + + private + + def scrna_config + Rails.application.config.scrna_config + end +end From dac8945a593c898bf1e60b46b06325bbeba082a0 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sat, 21 Dec 2024 16:13:21 +0000 Subject: [PATCH 074/115] Add cDNA Prep full allowance validation --- ...na_core_cdna_prep_feasibility_validator.rb | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 59c6ea686b..60ab60d3e3 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -2,20 +2,24 @@ # rubocop:disable Metrics/ModuleLength module Submission::ScrnaCoreCdnaPrepFeasibilityValidator + + include Submission::ScrnaCoreCdnaPrepFeasibilityCalculator + HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) HEADER_PLATE_WELL = 'plate well' unless defined?(HEADER_PLATE_WELL) HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' unless defined?(HEADER_NUMBER_OF_POOLS) + HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' unless defined?(HEADER_CELLS_PER_CHIP_WELL) I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR = 'submissions.scrna_core_cdna_prep_feasibility_validator' def validate_scrna_core_cdna_prep_feasibility - required = [HEADER_BARCODE, HEADER_PLATE_WELL, HEADER_NUMBER_OF_POOLS] + required = [HEADER_BARCODE, HEADER_PLATE_WELL, HEADER_NUMBER_OF_POOLS, HEADER_CELLS_PER_CHIP_WELL] return unless required.all? { |header| headers.include?(header) } validate_scrna_core_cdna_prep_total_number_of_samples validate_scrna_core_cdna_prep_total_number_of_pools validate_scrna_core_cdna_prep_feasibility_by_samples validate_scrna_core_cdna_prep_feasibility_by_donors - validate_scrna_core_cdna_prep_full_allowance + validate_scrna_core_cdna_prep_full_allowance if errors.empty? end private @@ -158,6 +162,32 @@ def list_barcodes_or_well_locations_to_check_for_donors(rows) end def validate_scrna_core_cdna_prep_full_allowance + group_rows_by_study_and_project.each do |(study_name, project_name), rows| + number_of_samples_in_smallest_pool = calculate_number_of_samples_in_smallest_pool(rows) + number_of_cells_per_chip_well = rows.first[headers.index(HEADER_CELLS_PER_CHIP_WELL)].to_i + final_resuspension_volume = calculate_resuspension_volume(number_of_samples_in_smallest_pool) + full_allowance = calculate_full_allowance(number_of_cells_per_chip_well) + + return if final_resuspension_volume >= full_allowance + + warnings.add( + :spreadsheet, + I18n.t( + 'warnings.full_allowance', + study_name: study_name, + project_name: project_name, + number_of_samples_in_smallest_pool: number_of_samples_in_smallest_pool, + final_resuspension_volume: final_resuspension_volume, + full_allowance: full_allowance, + end + end + + def calculate_number_of_samples_in_smallest_pool(rows) + barcodes, well_locations = extract_barcodes_and_well_locations(rows) + number_of_samples = calculate_total_number_of_samples(barcodes, well_locations) + number_of_pools = rows.first[headers.index(HEADER_NUMBER_OF_POOLS)].to_i + pool_sizes = calculate_pool_size_types(number_of_samples, number_of_pools) + pool_sizes['smallest'] end def extract_barcodes_and_well_locations(rows) From 40e690c0077414cbf1b35957bc76ff735832d13c Mon Sep 17 00:00:00 2001 From: yoldas Date: Sat, 21 Dec 2024 16:14:16 +0000 Subject: [PATCH 075/115] Add yml document start marker --- config/locales/en.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index f7897b849e..4514a1c108 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1,3 +1,4 @@ +--- en: date: formats: From ecd8b3fc3270d622f0dfd2071b924ab51de2c2cc Mon Sep 17 00:00:00 2001 From: yoldas Date: Sat, 21 Dec 2024 17:12:06 +0000 Subject: [PATCH 076/115] Add warnings collectiont to bulk submission --- app/models/bulk_submission.rb | 7 +++++++ .../scrna_core_cdna_prep_feasibility_validator.rb | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index f399c063e4..37b9faa9a7 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -54,6 +54,13 @@ def initialize(attrs = {}) self.encoding = attrs.fetch(:encoding, DEFAULT_ENCODING) end + # Returns the warnings collection for the BulkSubmission object. + # Initialises the warnings collectin if it does not exist yet. + # @return [ActiveModel::Errors] the warnings collection + def warnings + @warnings ||= ActiveModel::Errors.new(self) + end + include ManifestUtil # rubocop:todo Metrics/MethodLength diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 60ab60d3e3..046c331448 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -178,7 +178,8 @@ def validate_scrna_core_cdna_prep_full_allowance project_name: project_name, number_of_samples_in_smallest_pool: number_of_samples_in_smallest_pool, final_resuspension_volume: final_resuspension_volume, - full_allowance: full_allowance, + full_allowance: full_allowance + ) end end From 66db3ff3c134eb58b752c7e8c931f5c24090e093 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sat, 21 Dec 2024 17:13:52 +0000 Subject: [PATCH 077/115] Fix syntax error in warnings.add call --- .../submission/scrna_core_cdna_prep_feasibility_validator.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 046c331448..9af9b4409b 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -180,6 +180,7 @@ def validate_scrna_core_cdna_prep_full_allowance final_resuspension_volume: final_resuspension_volume, full_allowance: full_allowance ) + ) end end From 372b2a76b8a82437f680a62ff2c7b112e7135b25 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 00:33:18 +0000 Subject: [PATCH 078/115] Use the pooling request type and assign donors to samples in the bulk submission additional validations test --- spec/models/bulk_submission_spec.rb | 30 ++++++++--------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/spec/models/bulk_submission_spec.rb b/spec/models/bulk_submission_spec.rb index c00a8f84de..830ec4f157 100644 --- a/spec/models/bulk_submission_spec.rb +++ b/spec/models/bulk_submission_spec.rb @@ -25,15 +25,7 @@ } end - before do - SubmissionSerializer.construct!(submission_template_hash) - tubes.each_with_index.map do |tube, i| - create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) - end - tubes.each_with_index do |tube, i| - tube.receptacle.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1}") - end - end + before { SubmissionSerializer.construct!(submission_template_hash) } it 'is invalid' do expect { subject.process }.to raise_error(ActiveRecord::RecordInvalid) @@ -381,11 +373,14 @@ end context 'when an scRNA Bulk Submission for tubes' do - let(:request_types) { create_list(:sequencing_request_type, 2) } + let!(:request_types) { [create(:pbmc_pooling_customer_request_type)] } # Create a list of tubes with samples let!(:tubes) do - create_list(:phi_x_stock_tube, 6) do |tube, i| - tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{i + 1}") + Array.new(6) do |index| + create(:sample_tube).tap do |tube| + tube.barcodes << Barcode.new(format: :sanger_ean13, barcode: "NT#{index + 1}") + tube.samples.first.sample_metadata.update!(donor_id: "donor_#{index}") + end end end let!(:study) { create(:study, name: 'Test Study') } @@ -405,16 +400,7 @@ } end - before do - SubmissionSerializer.construct!(submission_template_hash) - tubes.each_with_index.map do |tube, i| - create(:asset_group, name: "ag#{i + 1}", study: study, assets: [tube.receptacle]) - end - - tubes.each_with_index do |tube, i| - tube.receptacle.aliquots.first.sample.sample_metadata.update!(donor_id: "donor_#{i + 1}") - end - end + before { SubmissionSerializer.construct!(submission_template_hash) } it 'is valid' do expect(subject).to be_valid From ce071ee504d2b454845a06b5269379645986d23a Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 00:40:17 +0000 Subject: [PATCH 079/115] Rubocop --- ...a_core_cdna_prep_feasibility_calculator.rb | 2 - ...na_core_cdna_prep_feasibility_validator.rb | 41 +++++++++++++------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb index 6fe11254fe..ee0b535160 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true module Submission::ScrnaCoreCdnaPrepFeasibilityCalculator - def calculate_full_allowance(number_of_cells_per_chip_well) # "Full allowance" = ( "Chip loading volume" * 2) + 25 # 2 is because this is for 2 runs @@ -12,7 +11,6 @@ def calculate_full_allowance(number_of_cells_per_chip_well) scrna_config[:wastage_volume] end - def calculate_chip_loading_volume(number_of_cells_per_chip_well) # "Chip loading volume" = "Number of cells per chip well" / "Chip loading concentration" number_of_cells_per_chip_well / scrna_config[:desired_chip_loading_concentration] diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 9af9b4409b..5b816c91e7 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -2,7 +2,6 @@ # rubocop:disable Metrics/ModuleLength module Submission::ScrnaCoreCdnaPrepFeasibilityValidator - include Submission::ScrnaCoreCdnaPrepFeasibilityCalculator HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) @@ -168,22 +167,38 @@ def validate_scrna_core_cdna_prep_full_allowance final_resuspension_volume = calculate_resuspension_volume(number_of_samples_in_smallest_pool) full_allowance = calculate_full_allowance(number_of_cells_per_chip_well) - return if final_resuspension_volume >= full_allowance - - warnings.add( - :spreadsheet, - I18n.t( - 'warnings.full_allowance', - study_name: study_name, - project_name: project_name, - number_of_samples_in_smallest_pool: number_of_samples_in_smallest_pool, - final_resuspension_volume: final_resuspension_volume, - full_allowance: full_allowance - ) + next if final_resuspension_volume >= full_allowance + + add_warning_scrna_core_cdna_prep_full_allowance( + study_name, + project_name, + number_of_samples_in_smallest_pool, + final_resuspension_volume, + full_allowance ) end end + def add_warning_scrna_core_cdna_prep_full_allowance( + study_name, + project_name, + number_of_samples_in_smallest_pool, + final_resuspension_volume, + full_allowance + ) + warnings.add( + :spreadsheet, + I18n.t( + 'warnings.full_allowance', + study_name:, + project_name:, + number_of_samples_in_smallest_pool:, + final_resuspension_volume:, + full_allowance: + ) + ) + end + def calculate_number_of_samples_in_smallest_pool(rows) barcodes, well_locations = extract_barcodes_and_well_locations(rows) number_of_samples = calculate_total_number_of_samples(barcodes, well_locations) From bbfaa52d7ea4c88bbd393ad7528c9b743e35d979 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 01:30:23 +0000 Subject: [PATCH 080/115] Add partial to render if there are bulk submission warnings --- app/views/bulk_submissions/_warnings.erb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/views/bulk_submissions/_warnings.erb diff --git a/app/views/bulk_submissions/_warnings.erb b/app/views/bulk_submissions/_warnings.erb new file mode 100644 index 0000000000..ca9fd20162 --- /dev/null +++ b/app/views/bulk_submissions/_warnings.erb @@ -0,0 +1,7 @@ +<% alert(:warning) do %> +
    Warning + <% @presenter.warnings.full_messages.each do |message| %> +
  • <%= message %>
    • + <% end %> +
+<% end %> \ No newline at end of file From af6ebe8de3c7e2886cf034fe37feb71da8da95f0 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 01:32:03 +0000 Subject: [PATCH 081/115] Add info about number of cells per chip well adjustment in the full allowance warning --- config/locales/en.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index 4514a1c108..806c7752e4 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -237,6 +237,8 @@ en: There is not enough material for the "full allowance" (2 full runs on the chip) for the study %{study_name} and the project %{project_name}. The final resuspension volume for the smallest pool should be greater than or equal to the full allowance. However, for the smallest pool of size %{number_of_samples_in_smallest_pool}, the final resuspension volume is %{final_resuspension_volume} and the full allowance is %{full_allowance}. + The number of cells per chip well will be adjusted according to the number of samples while pooling. + cherrypick: picking_by_row: "This cherrypick may take longer as it is picking by rows, rather than columns." From 73aa40991cd6e3455b2dc962a73b063a24e7b790 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 01:34:09 +0000 Subject: [PATCH 082/115] Render partial in the create view if there are bulk submission warnings --- app/views/bulk_submissions/create.html.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/views/bulk_submissions/create.html.erb b/app/views/bulk_submissions/create.html.erb index 1599a6991b..8017c8826a 100644 --- a/app/views/bulk_submissions/create.html.erb +++ b/app/views/bulk_submissions/create.html.erb @@ -3,6 +3,10 @@

Your bulk submission has been processed.

+<% if @bulk_submission.warnings.present? %> + <% render partial: 'bulk_submissions/warnings' locals: {presenter: @bulk_submission} %> +<% end %> + <% @these_subs.each do |submission| %> <%= render partial: 'submissions/warnings', locals: { presenter: submission } %> <% end %> From ffb743bf49654786bb82fe2cc5fd4243cd827081 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 02:38:00 +0000 Subject: [PATCH 083/115] Fix the bulk_submissions warnings partial --- app/views/bulk_submissions/_warnings.erb | 12 ++++++------ app/views/bulk_submissions/create.html.erb | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/views/bulk_submissions/_warnings.erb b/app/views/bulk_submissions/_warnings.erb index ca9fd20162..dd53783765 100644 --- a/app/views/bulk_submissions/_warnings.erb +++ b/app/views/bulk_submissions/_warnings.erb @@ -1,7 +1,7 @@ <% alert(:warning) do %> -
    Warning - <% @presenter.warnings.full_messages.each do |message| %> -
  • <%= message %>
    • - <% end %> -
-<% end %> \ No newline at end of file +
    Warning + <% @presenter.warnings.full_messages.each do |message| %> +
  • <%= message %>
    • + <% end %> +
+<% end %> diff --git a/app/views/bulk_submissions/create.html.erb b/app/views/bulk_submissions/create.html.erb index 8017c8826a..19f87c2539 100644 --- a/app/views/bulk_submissions/create.html.erb +++ b/app/views/bulk_submissions/create.html.erb @@ -4,7 +4,7 @@

Your bulk submission has been processed.

<% if @bulk_submission.warnings.present? %> - <% render partial: 'bulk_submissions/warnings' locals: {presenter: @bulk_submission} %> + <% render partial: 'bulk_submissions/warnings', locals: {presenter: @bulk_submission} %> <% end %> <% @these_subs.each do |submission| %> From 8aa16cb09def20ce352e5297f1a2eba6d17c9708 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 03:20:28 +0000 Subject: [PATCH 084/115] Update comment about where warnings are used --- app/models/bulk_submission.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/bulk_submission.rb b/app/models/bulk_submission.rb index 37b9faa9a7..f4a982da4d 100644 --- a/app/models/bulk_submission.rb +++ b/app/models/bulk_submission.rb @@ -55,7 +55,9 @@ def initialize(attrs = {}) end # Returns the warnings collection for the BulkSubmission object. - # Initialises the warnings collectin if it does not exist yet. + # Initialises the warnings collection if it does not exist yet. The collection + # is used for showing informative warning messages to the user after the bulk + # submission has been processed successfully. # @return [ActiveModel::Errors] the warnings collection def warnings @warnings ||= ActiveModel::Errors.new(self) From 4c2263131135f93502dcc7876b9aefca9d0ef5f6 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 21:37:44 +0000 Subject: [PATCH 085/115] Round volumes to 1 decimal place --- ...crna_core_cdna_prep_feasibility_validator.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 5b816c91e7..3d4c052375 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -160,6 +160,7 @@ def list_barcodes_or_well_locations_to_check_for_donors(rows) tube?(barcodes, well_locations) ? barcodes.join(', ') : well_locations.join(', ') end + # rubocop:disable Metrics/MethodLength def validate_scrna_core_cdna_prep_full_allowance group_rows_by_study_and_project.each do |(study_name, project_name), rows| number_of_samples_in_smallest_pool = calculate_number_of_samples_in_smallest_pool(rows) @@ -173,11 +174,12 @@ def validate_scrna_core_cdna_prep_full_allowance study_name, project_name, number_of_samples_in_smallest_pool, - final_resuspension_volume, - full_allowance + final_resuspension_volume.round(1), # round to 1 decimal place + full_allowance.round(1) # round to 1 decimal place ) end end + # rubocop:enable Metrics/MethodLength def add_warning_scrna_core_cdna_prep_full_allowance( study_name, @@ -190,11 +192,12 @@ def add_warning_scrna_core_cdna_prep_full_allowance( :spreadsheet, I18n.t( 'warnings.full_allowance', - study_name:, - project_name:, - number_of_samples_in_smallest_pool:, - final_resuspension_volume:, - full_allowance: + study_name: study_name, + project_name: project_name, + number_of_samples_in_smallest_pool: number_of_samples_in_smallest_pool, + final_resuspension_volume: final_resuspension_volume, + full_allowance: full_allowance, + scope: I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR ) ) end From dc8060eb4993df486862707ee66af4fcc2b9a5a2 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 21:39:30 +0000 Subject: [PATCH 086/115] Fix adding warning to the page --- app/views/bulk_submissions/_warnings.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/bulk_submissions/_warnings.erb b/app/views/bulk_submissions/_warnings.erb index dd53783765..8021629b3e 100644 --- a/app/views/bulk_submissions/_warnings.erb +++ b/app/views/bulk_submissions/_warnings.erb @@ -1,6 +1,6 @@ -<% alert(:warning) do %> -
    Warning - <% @presenter.warnings.full_messages.each do |message| %> +<%= alert(:warning) do %> +
      + <% presenter.warnings.full_messages.each do |message| %>
    • <%= message %>
      • <% end %>
    From 31cffcaf4996fdfd42437100ce0212a91b41e74c Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 21:40:13 +0000 Subject: [PATCH 087/115] Fix rendering partial for warnings --- app/views/bulk_submissions/create.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/bulk_submissions/create.html.erb b/app/views/bulk_submissions/create.html.erb index 19f87c2539..6155a3c092 100644 --- a/app/views/bulk_submissions/create.html.erb +++ b/app/views/bulk_submissions/create.html.erb @@ -4,7 +4,7 @@

    Your bulk submission has been processed.

    <% if @bulk_submission.warnings.present? %> - <% render partial: 'bulk_submissions/warnings', locals: {presenter: @bulk_submission} %> + <%= render partial: 'bulk_submissions/warnings', locals: {presenter: @bulk_submission} %> <% end %> <% @these_subs.each do |submission| %> From a34173bc00bd0e85c1ba803028d5f9e05119e6a4 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 21:41:56 +0000 Subject: [PATCH 088/115] Prettier --- config/locales/en.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 806c7752e4..52f70832df 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -239,7 +239,6 @@ en: However, for the smallest pool of size %{number_of_samples_in_smallest_pool}, the final resuspension volume is %{final_resuspension_volume} and the full allowance is %{full_allowance}. The number of cells per chip well will be adjusted according to the number of samples while pooling. - cherrypick: picking_by_row: "This cherrypick may take longer as it is picking by rows, rather than columns." From 4d4b0a6a4749f19de8f79c12884a1bdb7a4914f5 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 21:42:54 +0000 Subject: [PATCH 089/115] Add tests for cDNA Prep full allowance validation --- ...re_cdna_prep_feasibility_validator_spec.rb | 128 ++++++++++++++++-- 1 file changed, 118 insertions(+), 10 deletions(-) diff --git a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb index dfd78b3833..24f22ad055 100644 --- a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb +++ b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb @@ -50,6 +50,10 @@ let(:group_1_number_of_pools) { 1 } let(:group_2_number_of_pools) { 1 } let(:group_3_number_of_pools) { 1 } + # The cells per chip well for each study-project group. + let(:group_1_cells_per_chip_well) { 90_000 } + let(:group_2_cells_per_chip_well) { 90_000 } + let(:group_3_cells_per_chip_well) { 90_000 } # Donor IDs for each group to set on the samples. let(:group_1_donors) { Array.new(group_1_number_of_samples) { |index| "group_1_donor_#{index + 1}" } } let(:group_2_donors) { Array.new(group_2_number_of_samples) { |index| "group_2_donor_#{index + 1}" } } @@ -78,7 +82,8 @@ 'Project Name' => 'Project 1', 'Barcode' => group_1_tubes[index].human_barcode, 'Asset Group Name' => 'Asset Group 1', - 'scRNA Core Number of Pools' => group_1_number_of_pools + 'scRNA Core Number of Pools' => group_1_number_of_pools, + 'scRNA Core Cells per Chip Well' => group_1_cells_per_chip_well ) end end @@ -89,7 +94,8 @@ 'Project Name' => 'Project 2', 'Barcode' => group_2_tubes[index].human_barcode, 'Asset Group Name' => 'Asset Group 2', - 'scRNA Core Number of Pools' => group_2_number_of_pools + 'scRNA Core Number of Pools' => group_2_number_of_pools, + 'scRNA Core Cells per Chip Well' => group_2_cells_per_chip_well ) end end @@ -100,7 +106,8 @@ 'Project Name' => 'Project 3', 'Barcode' => group_3_tubes[index].human_barcode, 'Asset Group Name' => 'Asset Group 3', - 'scRNA Core Number of Pools' => group_3_number_of_pools + 'scRNA Core Number of Pools' => group_3_number_of_pools, + 'scRNA Core Cells per Chip Well' => group_3_cells_per_chip_well ) end end @@ -250,7 +257,7 @@ def scrna_config let(:group_3_number_of_samples) { 0 } # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do error_message = I18n.t( 'errors.total_number_of_samples', @@ -275,7 +282,7 @@ def scrna_config let(:group_3_number_of_pools) { 2 } # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do error_message = I18n.t( 'errors.total_number_of_samples', @@ -352,7 +359,7 @@ def scrna_config end # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do error_message = I18n.t( 'errors.total_number_of_pools', @@ -377,7 +384,7 @@ def scrna_config let(:group_3_number_of_pools) { 3 } # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do error_message = I18n.t( 'errors.total_number_of_pools', @@ -449,7 +456,7 @@ def scrna_config let(:group_3_number_of_pools) { 2 } # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do expect { bulk_submission.process }.to raise_error(ActiveRecord::RecordInvalid) # Parametrise the assertions for the error messages because multiple @@ -491,7 +498,7 @@ def scrna_config let(:group_3_number_of_pools) { 1 } # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do error_message = I18n.t( 'errors.number_of_pools_by_samples', @@ -579,7 +586,7 @@ def scrna_config # clash, because we have 4 pools. # rubocop:disable RSpec/ExampleLength, RSpec/MultipleExpectations - it 'sets the error message' do + it 'adds the error message' do # Barcodes or well locations of the labware with the same donor ID are # listed in the error message. This test uses tubes; hence the barcodes # of the tubes will be listed to help the user identify the samples. @@ -601,4 +608,105 @@ def scrna_config # rubocop:enable RSpec/ExampleLength, RSpec/MultipleExpectations end end + + describe '#validate_scrna_core_cdna_prep_full_allowance' do + # There is not enough material for the "full allowance" (2 full runs on the + # chip) for the smallest pool size for a study-project group. + context 'when final_resuspension_volume is greater than the full allowance' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 0 } # not included in the test + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + # rubocop:disable RSpec/MultipleExpectations + it 'adds the warning message' do + warning_message = + I18n.t( + 'warnings.full_allowance', + study_name: 'Study 1', + project_name: 'Project 1', + number_of_samples_in_smallest_pool: 5, + final_resuspension_volume: '59.4', + full_allowance: '99.0', + scope: i18n_scope + ) + + expect(bulk_submission).to be_valid + expect(bulk_submission.warnings[:spreadsheet]).to include(warning_message) + end + # rubocop:enable RSpec/MultipleExpectations + end + + context 'when final_resuspension_volume is equal to the full allowance' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 0 } # not included in the test + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + let(:group_1_cells_per_chip_well) { 41_250 } + + # rubocop:disable RSpec/MultipleExpectations + it 'adds the warning message' do + expect(bulk_submission).to be_valid + expect(bulk_submission.warnings).to be_empty + end + # rubocop:enable RSpec/MultipleExpectations + end + + context 'when final resuspension volume is less than the full allowance' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 0 } # not included in the test + let(:group_3_number_of_samples) { 0 } # not included in the test + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + let(:group_1_cells_per_chip_well) { 30_000 } + + # rubocop:disable RSpec/MultipleExpectations + it 'does not add the warning message' do + expect(bulk_submission).to be_valid + expect(bulk_submission.warnings).to be_empty + end + # rubocop:enable RSpec/MultipleExpectations + end + + context 'when there are multiple full allowance warnings' do + let(:group_1_number_of_samples) { 5 } + let(:group_2_number_of_samples) { 5 } + let(:group_3_number_of_samples) { 5 } + + let(:group_1_number_of_pools) { 1 } + let(:group_2_number_of_pools) { 1 } + let(:group_3_number_of_pools) { 1 } + + # rubocop:disable RSpec/MultipleExpectations, RSpec/ExampleLength + it 'adds the warning message for each study-project' do + expect(bulk_submission).to be_valid + + params = [['Study 1', 'Project 1'], ['Study 2', 'Project 2'], ['Study 3', 'Project 3']] + params.each do |study_name, project_name| + warning_message = + I18n.t( + 'warnings.full_allowance', + study_name: study_name, + project_name: project_name, + number_of_samples_in_smallest_pool: 5, + final_resuspension_volume: '59.4', + full_allowance: '99.0', + scope: i18n_scope + ) + expect(bulk_submission.warnings[:spreadsheet]).to include(warning_message) + end + end + # rubocop:enable RSpec/MultipleExpectations, RSpec/ExampleLength + end + end end From e7f1b868d81994f02f38274c3c6d073431f210cf Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 23:16:44 +0000 Subject: [PATCH 090/115] Add feature flag for enabling the feasibility validations for cDNA Prep Bulk Submission --- .../submission/validations_by_template_name.rb | 6 +++++- config/feature_flags.yml | 1 + ...na_core_cdna_prep_feasibility_validator_spec.rb | 14 ++++++++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/app/models/submission/validations_by_template_name.rb b/app/models/submission/validations_by_template_name.rb index a9fd8d6283..d9acd721ec 100644 --- a/app/models/submission/validations_by_template_name.rb +++ b/app/models/submission/validations_by_template_name.rb @@ -30,6 +30,7 @@ module Submission::ValidationsByTemplateName # errors [ActiveModel::Errors] The errors object to which validation errors are added. # # @return [void] + # rubocop:disable Metrics/MethodLength def apply_additional_validations_by_template_name # depending on the submission template type, call additional validations # NB. assumption that all rows in the csv have the same submission template name @@ -42,9 +43,12 @@ def apply_additional_validations_by_template_name validate_consistent_column_value(HEADER_NUMBER_OF_POOLS) validate_consistent_column_value(HEADER_CELLS_PER_CHIP_WELL) validate_samples_per_pool_for_labware - validate_scrna_core_cdna_prep_feasibility + if Flipper.enabled?(:y24_429_enable_check_feasibility_of_cdna_prep_submission) + validate_scrna_core_cdna_prep_feasibility + end end end + # rubocop:enable Metrics/MethodLength def apply_number_of_samples_per_pool_validation # Creates groups of rows based on the study and project name (pool_number, study-project) combinations diff --git a/config/feature_flags.yml b/config/feature_flags.yml index 719af38eed..298050e612 100644 --- a/config/feature_flags.yml +++ b/config/feature_flags.yml @@ -4,3 +4,4 @@ dpl_395_2_enable_advanced_search_tab: Shows the Search tab in the top navigation bar # Below is flagged off until some existing data that would violate the rule is fixed. Should be re-enabled in Y24-058. y24_052_enable_data_release_timing_validation: Enables server-side validation that enforces a relationship between the values of two study metadata fields +y24_429_enable_check_feasibility_of_cdna_prep_submission: Enables the feasibility validations for cDNA Prep Bulk Submission diff --git a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb index 24f22ad055..8da85b6546 100644 --- a/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb +++ b/spec/models/scrna_core_cdna_prep_feasibility_validator_spec.rb @@ -7,9 +7,15 @@ # module. CSV files are created for bulk submissions and the validations # provided by the module are tested. RSpec.describe BulkSubmission, with: :uploader do + # Enable the feature flag for the feasibility validations. # The test subject is initialised with the uploaded file. subject(:bulk_submission) { described_class.new(spreadsheet: submission_file) } + before do + Flipper.enable(:y24_429_enable_check_feasibility_of_cdna_prep_submission) + SubmissionSerializer.construct!(submission_template_hash) # Create the template. + end + # The CSV headers are used to create the CSV content; copied from headings in # config/bulk_submission_excel/columns.yml let(:csv_headers) do @@ -211,10 +217,6 @@ def scrna_config Rails.application.config.scrna_config end - before do - SubmissionSerializer.construct!(submission_template_hash) # Create the template. - end - after do csv_tempfile.close! # Close and unlink the temporary file after the test. end @@ -621,7 +623,7 @@ def scrna_config let(:group_2_number_of_pools) { 1 } let(:group_3_number_of_pools) { 1 } - # rubocop:disable RSpec/MultipleExpectations + # rubocop:disable RSpec/MultipleExpectations, RSpec/ExampleLength it 'adds the warning message' do warning_message = I18n.t( @@ -637,7 +639,7 @@ def scrna_config expect(bulk_submission).to be_valid expect(bulk_submission.warnings[:spreadsheet]).to include(warning_message) end - # rubocop:enable RSpec/MultipleExpectations + # rubocop:enable RSpec/MultipleExpectations, RSpec/ExampleLength end context 'when final_resuspension_volume is equal to the full allowance' do From f3113b8f6fc4016270f381687bb0eea1c59ee363 Mon Sep 17 00:00:00 2001 From: yoldas Date: Sun, 22 Dec 2024 23:51:41 +0000 Subject: [PATCH 091/115] Add code documentation for the feasibility calculator module --- ...a_core_cdna_prep_feasibility_calculator.rb | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb index ee0b535160..56b1d3553a 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb @@ -1,6 +1,17 @@ # frozen_string_literal: true +# This module provides methods for calculating the full allowance volume and the +# final resuspension volume for scRNA core cDNA prep feasibility validations. module Submission::ScrnaCoreCdnaPrepFeasibilityCalculator + # This method calculates the full allowance volume (in microlitres) for the + # specified number of cells per chip well, which is typically specified in + # in a bulk submission per study and project. It uses the pooling settings + # from the scRNA config. It first calculates the chip loading volume for the + # given number of cells per chip well, and then the full allowance for that + # chip loading volume. + # + # @param number_of_cells_per_chip_well [Integer] the number of cells per chip well from the bulk submission + # @return [Float] the full allowance volume def calculate_full_allowance(number_of_cells_per_chip_well) # "Full allowance" = ( "Chip loading volume" * 2) + 25 # 2 is because this is for 2 runs @@ -11,16 +22,39 @@ def calculate_full_allowance(number_of_cells_per_chip_well) scrna_config[:wastage_volume] end + # This method calculates the chip loading volume (in microlitres) for the + # specified number of cells per chip well, which is typically specified in + # in a bulk submission per study and project. It uses the pooling settings + # from the scRNA config. + # + # @param num_cells_per_chip_well [Integer] the number of cells per chip well from the bulk submission + # @return [Float] the chip loading volume def calculate_chip_loading_volume(number_of_cells_per_chip_well) # "Chip loading volume" = "Number of cells per chip well" / "Chip loading concentration" number_of_cells_per_chip_well / scrna_config[:desired_chip_loading_concentration] end + # This method calculates the resuspension volume (in microlitres) for the + # specified number of samples in a pool, which is typically taken as the + # number of samples in the smallest pool for a study and project. It uses the + # pooling settings from the scRNA config. It first calculates the total cells + # in 300ul for the given number of samples in the pool, and then the + # resuspension volume for that total cell count. + # + # @param count_of_samples_in_pool [Integer] the number of samples in the pool + # @return [Float] the resuspension volume def calculate_resuspension_volume(count_of_samples_in_pool) total_cells_in_300ul = calculate_total_cells_in_300ul(count_of_samples_in_pool) total_cells_in_300ul / scrna_config[:desired_chip_loading_concentration] end + # This method calculates the total cells in 300ul for the specified number of + # samples in a pool, which is typically taken as the number of samples in the + # smallest pool for a study and project. It uses the pooling settings from the + # scRNA config. + # + # @param count_of_samples_in_pool [Integer] the number of samples in the pool + # @return [Integer] the total cells in 300ul def calculate_total_cells_in_300ul(count_of_samples_in_pool) (count_of_samples_in_pool * scrna_config[:required_number_of_cells_per_sample_in_pool]) * scrna_config[:wastage_factor] @@ -28,6 +62,8 @@ def calculate_total_cells_in_300ul(count_of_samples_in_pool) private + # This method returns the scRNA config from the Rails application config. + # @return [Hash] the scRNA config def scrna_config Rails.application.config.scrna_config end From 48c8af88173c017bdce5297b17284bd63682bb68 Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 23 Dec 2024 02:17:30 +0000 Subject: [PATCH 092/115] Add code documentation for the feasibility validator module --- ...na_core_cdna_prep_feasibility_validator.rb | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index 3d4c052375..e8a4ffa33f 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -1,15 +1,45 @@ # frozen_string_literal: true +# This module provides methods for additional validations for bulk submissions. +# It checks if the pooling strategy given in an scRNA core cDNA prep submission +# is feasible so that it can be adjusted earlier at the submission stage rather +# than at the pooling stage. It generates error messages for the following +# criteria: +# - Total number of samples in the submission must be between 5 and 96 (inclusive). +# - Total (requested) number of pools must be between 1 and 8 (inclusive). +# - The number of pools requested must be feasible given the number of samples, +# and having checked for donor clash +# It also generates a warning message, only if there are no errors, for the +# following condition: +# - There is not enough material for the "full allowance" (2 full runs on the chip) +# +# The messages are generated using the strings in the locale file. +# +# This module is feature-flagged with the key +# y24_429_enable_check_feasibility_of_cdna_prep_submission . If enabled, the +# validate_scrna_core_cdna_prep_feasibility method is called by the +# apply_additional_validations_by_template_name method of the +# Submission::ValidationsByTemplateName module. +# # rubocop:disable Metrics/ModuleLength module Submission::ScrnaCoreCdnaPrepFeasibilityValidator + # Add the methods from the calculator module for volume calculations. include Submission::ScrnaCoreCdnaPrepFeasibilityCalculator + # Constants for the headers in the bulk submission for scRNA core cDNA prep. HEADER_BARCODE = 'barcode' unless defined?(HEADER_BARCODE) HEADER_PLATE_WELL = 'plate well' unless defined?(HEADER_PLATE_WELL) HEADER_NUMBER_OF_POOLS = 'scrna core number of pools' unless defined?(HEADER_NUMBER_OF_POOLS) HEADER_CELLS_PER_CHIP_WELL = 'scrna core cells per chip well' unless defined?(HEADER_CELLS_PER_CHIP_WELL) + # I18n scope for the error messages in this module; where to find the translations in the locale file. I18N_SCOPE_SCRNA_CORE_CDNA_PREP_FEASIBILITY_VALIDATOR = 'submissions.scrna_core_cdna_prep_feasibility_validator' + # This method checks the feasibility of scRNA Core cDNA Prep bulk submission. + # If the submission spreadsheet does not contain the necessary headers, the + # method returns early. Otherwise, it performs a series of validations and + # adds errors and warnings to the bulk submission if necessary. + # + # @return [void] def validate_scrna_core_cdna_prep_feasibility required = [HEADER_BARCODE, HEADER_PLATE_WELL, HEADER_NUMBER_OF_POOLS, HEADER_CELLS_PER_CHIP_WELL] return unless required.all? { |header| headers.include?(header) } @@ -23,6 +53,13 @@ def validate_scrna_core_cdna_prep_feasibility private + # Validates the total number of samples for scRNA Core cDNA Prep submission. + # This method extracts barcodes and well locations from the CSV data rows, + # calculates the total number of samples, and checks if the count is within + # the allowed range. If the count is outside the allowed range, it adds an + # error message. The allowed range is defined in the scRNA config. + # + # @return [void] def validate_scrna_core_cdna_prep_total_number_of_samples barcodes, well_locations = extract_barcodes_and_well_locations(csv_data_rows) count = calculate_total_number_of_samples(barcodes, well_locations) @@ -34,6 +71,13 @@ def validate_scrna_core_cdna_prep_total_number_of_samples add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) end + # Adds an error message for the total number of samples in the submission. + # + # @param min [Integer] the minimum total number of samples allowed + # @param max [Integer] the maximum total number of samples allowed + # @param count [Integer] the actual total number of samples in the submission + # + # @return [void] def add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) errors.add( :spreadsheet, @@ -47,6 +91,21 @@ def add_error_scrna_core_cdna_prep_total_number_of_samples(min, max, count) ) end + # Validates the total number of pools for scRNA Core cDNA Prep submission. + # This method groups rows by study and project, extracts the number of pools + # from the first row of each group, calculates the total number of pools, and + # checks if the count is within the allowed range. If the count is outside the + # allowed range, it adds an error message. The allowed range is defined in the + # scRNA config. + # + # @example Calculating total number of pools + # Study A-Project A asks for 1 pool. + # Study A-Project B asks for 2 pools. + # Study C-Project C asks for 5 pools. + # total number of pools is 1 + 2 + 5 = 8 . + # For the allowed range of 1 to 8 pools, this is valid. + # + # @return [void] def validate_scrna_core_cdna_prep_total_number_of_pools first_rows = group_rows_by_study_and_project.map { |_study_project, rows| rows.first } count = first_rows.sum { |row| row[headers.index(HEADER_NUMBER_OF_POOLS)].to_i } @@ -58,6 +117,13 @@ def validate_scrna_core_cdna_prep_total_number_of_pools add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) end + # Adds an error message for the total number of pools in the submission. + # + # @param min [Integer] the minimum total number of pools allowed + # @param max [Integer] the maximum total number of pools allowed + # @param count [Integer] the actual total number of pools in the submission + # + # @return [void] def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) errors.add( :spreadsheet, @@ -71,6 +137,21 @@ def add_error_scrna_core_cdna_prep_total_number_of_pools(min, max, count) ) end + # Validates the feasibility of scRNA Core cDNA preparation by samples. + # This method groups rows by study and project, extracts barcodes and well + # locations, calculates the number of samples, and checks if the smallest and + # biggest pool sizes are within the allowed range. If any pool size is outside + # the allowed range, it adds an error message. If the number of pools is one + # the smallest pool size and the biggest pool size are the same, only one + # pools size, the smallest, is checked. The allowed range is defined in the + # scRNA config. + # + # @example Checking if the number of pools is feasible, given number of samples + # Study A-Project A has 21 samples and asks for 2 pools. + # The smallest pool size is 10 and the biggest pool size is 11. + # For the allowed range of 5 to 25 samples, this is valid. + # + # @return [void] # rubocop:disable Metrics/AbcSize, Metrics/MethodLength def validate_scrna_core_cdna_prep_feasibility_by_samples group_rows_by_study_and_project.each do |(study_name, project_name), rows| @@ -88,6 +169,26 @@ def validate_scrna_core_cdna_prep_feasibility_by_samples end # rubocop:enable Metrics/AbcSize, Metrics/MethodLength + # Calculates the pool size types as smallest and biggest for the feasibility + # by samples validation. The smallest pool size is the quotient of the number + # of samples divided by the number of pools. If the remainder of the division + # is positive the biggest pool size is the quotient plus one. If the smallest + # and biggest pool sizes are the same, only the smallest pool size is returned. + # + # @example Calculating pool size types + # calculate_pool_size_types(21, 2) + # # => { 'smallest' => 10, 'biggest' => 11 } + # + # calculate_pool_size_types(20, 2) + # # => { 'smallest' => 10 } + # + # calculate_pool_size_types(10, 1) + # # => { 'smallest' => 10 } + # + # @param number_of_samples [Integer] the number of samples for study and project + # @param number_of_pools [Integer] the number of pools for study and project + # @return [Hash] A hash with the pool size types ('smallest' + # and 'biggest') and their corresponding sizes. def calculate_pool_size_types(number_of_samples, number_of_pools) quotient, remainder = number_of_samples.divmod(number_of_pools) size_types = { 'smallest' => quotient } @@ -95,6 +196,16 @@ def calculate_pool_size_types(number_of_samples, number_of_pools) size_types end + # Adds an error message for the feasibility of scRNA Core cDNA Prep by samples. + # + # @param study_name [String] the name of the study + # @param project_name [String] the name of the project + # @param min [Integer] the minimum pool size allowed + # @param max [Integer] the maximum pool size allowed + # @param count [Integer] the actual pool size + # @param size_type [String] the type of pool size ('smallest' or 'biggest') + # + # @return [void] # rubocop:disable Metrics/ParameterLists def add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_name, min, max, count, size_type) errors.add( @@ -113,6 +224,21 @@ def add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_na end # rubocop:enable Metrics/ParameterLists + # Validates the feasibility of scRNA Core cDNA preparation by donor IDs. + # This method groups rows by study and project, extracts the number of pools + # from the first row of each group, groups rows by donor ID, and checks if + # the largest group of samples with the same donor ID is within the allowed + # range. If the largest group size exceeds the number of pools, it adds an + # error message. + # + # @example Checking if the number of pools is feasible, given donor IDs + # Study A-Project A has 21 samples and asks for 2 pools. + # Three of the samples are from donor the same donors. Therefore, + # the largest group of samples from the same donor has 3 samples. + # It is not possible to separate 3 samples into different pools to avoid + # donor clash, so this is invalid. + # + # @return [void] # rubocop:disable Metrics/MethodLength def validate_scrna_core_cdna_prep_feasibility_by_donors group_rows_by_study_and_project.each do |(study_name, project_name), rows| @@ -134,6 +260,15 @@ def validate_scrna_core_cdna_prep_feasibility_by_donors end # rubocop:enable Metrics/MethodLength + # Adds an error message for the feasibility of scRNA Core cDNA Prep by donor IDs. + # + # @param study_name [String] the name of the study + # @param project_name [String] the name of the project + # @param count [Integer] the actual number of samples in the largest group of samples with the same donor ID + # @param number_of_pools [Integer] the number of pools requested for the study and project + # @param barcodes_or_well_locations [String] the barcodes or well locations of the samples in the largest group + # + # @return [void] def add_error_scrna_core_cdna_prep_feasibility_by_donors( study_name, project_name, @@ -155,11 +290,28 @@ def add_error_scrna_core_cdna_prep_feasibility_by_donors( ) end + # Lists the barcodes or well locations to check for donors to help user + # identify the samples that are involved in the donor clash. + # + # @param rows [Array>] the rows of the CSV data for a study and + # project that are involved in the donor clash + # @return [String] comma separated list of barcodes or well locations to check + # for donors def list_barcodes_or_well_locations_to_check_for_donors(rows) barcodes, well_locations = extract_barcodes_and_well_locations(rows) tube?(barcodes, well_locations) ? barcodes.join(', ') : well_locations.join(', ') end + # Validates the full allowance for scRNA Core cDNA preparation. + # This method groups rows by study and project, calculates the number of + # samples in the smallest pool, the number of cells per chip well, the final + # resuspension volume, and the full allowance. If the final resuspension + # volume is less than the full allowance, it adds a warning message. This + # validation is only performed if there are no errors in the submission. + # The warnings are kept separate from the errors and they are displayed to + # the user the submission created page. + # + # @return [void] # rubocop:disable Metrics/MethodLength def validate_scrna_core_cdna_prep_full_allowance group_rows_by_study_and_project.each do |(study_name, project_name), rows| @@ -181,6 +333,15 @@ def validate_scrna_core_cdna_prep_full_allowance end # rubocop:enable Metrics/MethodLength + # Adds a warning message for the full allowance in the submission. + # + # @param study_name [String] the name of the study + # @param project_name [String] the name of the project + # @param number_of_samples_in_smallest_pool [Integer] the number of samples in the smallest pool + # @param final_resuspension_volume [Float] the final resuspension volume + # @param full_allowance [Float] the full allowance volume + # + # @return [void] def add_warning_scrna_core_cdna_prep_full_allowance( study_name, project_name, @@ -202,6 +363,13 @@ def add_warning_scrna_core_cdna_prep_full_allowance( ) end + # Calculates the number of samples in the smallest pool for the full allowance + # validation. This method extracts barcodes and well locations from the CSV + # rows of the study and project, finds the number of samples and the number + # of pools, and calculates the number of samples in the smallest pool. + # + # @param rows [Array>] the rows of the CSV data for a study and project + # @return [Integer] the number of samples in the smallest pool def calculate_number_of_samples_in_smallest_pool(rows) barcodes, well_locations = extract_barcodes_and_well_locations(rows) number_of_samples = calculate_total_number_of_samples(barcodes, well_locations) @@ -210,17 +378,34 @@ def calculate_number_of_samples_in_smallest_pool(rows) pool_sizes['smallest'] end + # Extracts barcodes and well locations from the specified CSV data rows. + # + # @param rows [Array>] the rows of the CSV data + # @return [Array>] the extracted barcodes and well locations def extract_barcodes_and_well_locations(rows) barcodes = rows.pluck(headers.index(HEADER_BARCODE)) well_locations = rows.pluck(headers.index(HEADER_PLATE_WELL)) [barcodes, well_locations] end + # Calculates the total number of samples for the specified barcodes and well + # locations. The samples are counted from the tube receptacles or plate wells. def calculate_total_number_of_samples(barcodes, well_locations) receptacles = find_receptacles(barcodes, well_locations) receptacles.map(&:samples).flatten.count.to_i end + # Finds the receptacles for the specified barcodes and well locations. It + # finds if the submission contains tubes or a plate using the barcodes and + # well locations. If the submission contains tubes, it returns the tube + # receptacles. If the submission contains a plate, it returns the plate wells. + # The receptacles are sorted by rows; they are returned in the same order as + # the barcodes or well locations in the submission. + # + # @param barcodes [Array] the barcodes of the labware (tubes or plate) + # @param well_locations [Array] the well locations if the labware is a plate + # + # @return [Array] the receptacles sorted by rows def find_receptacles(barcodes, well_locations) if tube?(barcodes, well_locations) receptacles = find_tubes(barcodes) @@ -232,6 +417,13 @@ def find_receptacles(barcodes, well_locations) end end + # Sorts the tube receptacles by rows. The tube receptacles are sorted in the + # same order as the barcodes in the submission. + # + # @param receptacles [Array] the tube receptacles + # @param barcodes [Array] the barcodes of the tubes + # + # @return [Array] the tube receptacles sorted by rows def sort_tube_receptacles_by_rows(receptacles, barcodes) receptacle_map = {} receptacles.each do |receptacle| @@ -240,12 +432,26 @@ def sort_tube_receptacles_by_rows(receptacles, barcodes) barcodes.map { |barcode| receptacle_map[barcode] } end + # Sorts the plate wells by rows. The plate wells are sorted in the same order + # as the well locations in the submission. + # + # @param receptacles [Array] the plate wells + # @param well_locations [Array] the well locations of the plate wells + # + # @return [Array] the plate wells sorted by rows def sort_plate_wells_by_rows(receptacles, well_locations) receptacle_map = {} receptacles.each { |receptacle| receptacle_map[receptacle.map_description] = receptacle } well_locations.map { |well_location| receptacle_map[well_location] } end + # Groups the specified rows by donor ID. It extracts barcodes and well + # locations from the rows, finds the receptacles, groups the receptacles by + # donor ID, and returns a mapping between donor IDs and the corresponding rows. + # + # @param rows [Array>] the CSV rows of a study and project + + # @return [Hash>] the mapping between donor IDs and the corresponding rows # rubocop:disable Metrics/AbcSize def group_rows_by_donor_id(rows) barcodes, well_locations = extract_barcodes_and_well_locations(rows) @@ -261,6 +467,8 @@ def group_rows_by_donor_id(rows) end # rubocop:enable Metrics/AbcSize + # This method returns the scRNA config from the Rails application config. + # @return [Hash] the scRNA config def scrna_config Rails.application.config.scrna_config end From 583ef9b751d09940e0b335716aa3990452cdd716 Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 23 Dec 2024 02:27:19 +0000 Subject: [PATCH 093/115] Add descriptions to the allowed ranges for cDNA Prep submission in scRNA Config --- config/initializers/scrna_config.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/initializers/scrna_config.rb b/config/initializers/scrna_config.rb index daf1b63422..14a4ca2ed6 100644 --- a/config/initializers/scrna_config.rb +++ b/config/initializers/scrna_config.rb @@ -39,11 +39,13 @@ total_cell_count_default_threshold: 50_000, # Key for the number of cells per chip well metadata stored on pool wells (in poly_metadata) number_of_cells_per_chip_well_key: 'scrna_core_pbmc_donor_pooling_number_of_cells_per_chip_well', - # + # Minimum and maximum total number of samples allowed for cDNA Prep submission cdna_prep_minimum_total_number_of_samples: 5, cdna_prep_maximum_total_number_of_samples: 96, + # Minimum and maximum total number of pools allowed for cDNA Prep submission cdna_prep_minimum_total_number_of_pools: 1, cdna_prep_maximum_total_number_of_pools: 8, + # Minimum and maximum number of samples allowed in a pool for cDNA Prep Submission cdna_prep_minimum_number_of_samples_per_pool: 5, cdna_prep_maximum_number_of_samples_per_pool: 25 }.freeze From 6877931e8b6da0873f355a90f4cb7c075c655367 Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 23 Dec 2024 02:38:28 +0000 Subject: [PATCH 094/115] Yard-Junk --- .../submission/scrna_core_cdna_prep_feasibility_calculator.rb | 2 +- .../submission/scrna_core_cdna_prep_feasibility_validator.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb index 56b1d3553a..b50c0b7674 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_calculator.rb @@ -27,7 +27,7 @@ def calculate_full_allowance(number_of_cells_per_chip_well) # in a bulk submission per study and project. It uses the pooling settings # from the scRNA config. # - # @param num_cells_per_chip_well [Integer] the number of cells per chip well from the bulk submission + # @param number_of_cells_per_chip_well [Integer] the number of cells per chip well from the bulk submission # @return [Float] the chip loading volume def calculate_chip_loading_volume(number_of_cells_per_chip_well) # "Chip loading volume" = "Number of cells per chip well" / "Chip loading concentration" diff --git a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb index e8a4ffa33f..d56fd4ea11 100644 --- a/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb +++ b/app/models/submission/scrna_core_cdna_prep_feasibility_validator.rb @@ -233,7 +233,7 @@ def add_error_scrna_core_cdna_prep_feasibility_by_samples(study_name, project_na # # @example Checking if the number of pools is feasible, given donor IDs # Study A-Project A has 21 samples and asks for 2 pools. - # Three of the samples are from donor the same donors. Therefore, + # Three of the samples are from the same donor. Therefore, # the largest group of samples from the same donor has 3 samples. # It is not possible to separate 3 samples into different pools to avoid # donor clash, so this is invalid. From 70afbdf1f291db25f4cc2ddf681ba3a404b1e830 Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:04:46 +0000 Subject: [PATCH 095/115] Add migration to rename the column number_of_samples_per_pool to number_of_pools in request_metadata --- ...n_number_of_samples_per_pool_to_number_of_pools.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb diff --git a/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb b/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb new file mode 100644 index 0000000000..7786c627da --- /dev/null +++ b/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# This migration renames the column number_of_samples_per_pool to +# number_of_pools in the request_metadata table for the scRNA cDNA Prep +# submissions. The column will be used for storing the number of pools +# allocated for a study-project. +class RenameColumnNumberOfSamplesPerPoolToNumberOfPools < ActiveRecord::Migration[7.0] + def change + rename_column :request_metadata, :number_of_samples_per_pool, :number_of_pools + end +end From e28e423a9f9cc44006bf8c64014fd2e282ea0803 Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:09:13 +0000 Subject: [PATCH 096/115] Apply migration to rename the column number_of_samples_per_pool to number_of_pools in request_metadata --- db/schema.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 11da36d9d8..424fb4d93f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_11_06_103710) do +ActiveRecord::Schema[7.0].define(version: 2025_01_02_154835) do create_table "aliquot_indices", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_ci", options: "ENGINE=InnoDB ROW_FORMAT=DYNAMIC", force: :cascade do |t| t.integer "aliquot_id", null: false t.integer "lane_id", null: false @@ -1174,7 +1174,7 @@ t.string "data_type" t.integer "primer_panel_id" t.string "requested_flowcell_type" - t.integer "number_of_samples_per_pool" + t.integer "number_of_pools" t.integer "cells_per_chip_well" t.index ["request_id"], name: "index_request_metadata_on_request_id" end From 0f266b719de3700f63759c18ae6e30ed9c99ecee Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:12:39 +0000 Subject: [PATCH 097/115] Remove the rake task to populate the number of samples per pool for a given submission --- .../populate_number_of_samples_per_pool.rake | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 lib/tasks/populate_number_of_samples_per_pool.rake diff --git a/lib/tasks/populate_number_of_samples_per_pool.rake b/lib/tasks/populate_number_of_samples_per_pool.rake deleted file mode 100644 index 9e779feaee..0000000000 --- a/lib/tasks/populate_number_of_samples_per_pool.rake +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -# Run the rake task with the following command: -# bundle exec rake number_of_samples_per_pool:populate[20,1] -# The rake task will populate the number of samples per pool column in the request_metadata table -# for a given submission ID. -namespace :number_of_samples_per_pool do - desc 'Populate number of samples per pool column in request_metadata table for a given submission ID' - - task :populate, %i[samples_per_pool submission_id] => :environment do |_, args| - args.with_defaults(samples_per_pool: nil, submission_id: nil) - - raise StandardError, 'Number of samples per pool is missing' if args[:samples_per_pool].nil? - raise StandardError, 'Submission ID is missing' if args[:submission_id].nil? - - puts "Populating number of samples per pool column with #{args[:samples_per_pool]} - in request_metadata table for submission: #{args[:submission_id]}..." - - ActiveRecord::Base.transaction do - saved_count = 0 - Request::Metadata - .joins(:request) - .where(requests: { submission_id: args[:submission_id] }) - .find_each(batch_size: 50) do |request_metadata| - puts "Processing request_metadata #{request_metadata.id}..." - saved_count = process_request_metadata(request_metadata, saved_count, args[:samples_per_pool]) - end - end - end - - def process_request_metadata(request_metadata, saved_count, samples_per_pool) - request_metadata.number_of_samples_per_pool = samples_per_pool - begin - request_metadata.save! - saved_count += 1 - rescue ActiveRecord::ActiveRecordError, StandardError => e - puts "Error processing request_metadata #{request_metadata.id}: #{e.message}" - raise e - end - saved_count - end -end From a1f8c05697db51bd7cf0f29db919ec6d57402c36 Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:13:41 +0000 Subject: [PATCH 098/115] Remove test for the rake task to populate the number of samples per pool --- ...populate_numer_of_samples_per_pool_spec.rb | 64 ------------------- 1 file changed, 64 deletions(-) delete mode 100644 spec/lib/populate_numer_of_samples_per_pool_spec.rb diff --git a/spec/lib/populate_numer_of_samples_per_pool_spec.rb b/spec/lib/populate_numer_of_samples_per_pool_spec.rb deleted file mode 100644 index 3686ce8d97..0000000000 --- a/spec/lib/populate_numer_of_samples_per_pool_spec.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -# rubocop:disable RSpec/DescribeClass -require 'rails_helper' -require 'rake' - -RSpec.describe 'number_of_samples_per_pool:populate' do - def run_rake_task_with_args(task_name, *args) - Rake::Task[task_name].reenable - Rake.application.invoke_task("#{task_name}[#{args.join(',')}]") - end - - context 'when number of samples per pool rake task is invoked' do - let(:samples_per_pool) { 20 } - - before do - Rake.application.rake_require 'tasks/populate_number_of_samples_per_pool' - Rake::Task.define_task(:environment) - end - - it 'populating number of samples per pool' do - submission = create(:submission) - tube = create(:tube) - request = create(:well_request, asset: tube, submission: submission) - - # Execute - run_rake_task_with_args('number_of_samples_per_pool:populate', samples_per_pool, submission.reload.id) - - # Verify - expect(request.reload.request_metadata.number_of_samples_per_pool).to eq(samples_per_pool) - end - - it 'does not populate number of samples per pool when submission_id is nil' do - error_message = nil - - # Execute - begin - run_rake_task_with_args('number_of_samples_per_pool:populate', samples_per_pool, nil) - rescue StandardError => e - error_message = e.message - end - - # Verify - expect(error_message).to eq('Submission ID is missing') - end - - it 'does not populate number of samples per pool when samples_per_pool is nil' do - submission = create(:submission) - create(:tube) - error_message = nil - - # Execute - begin - run_rake_task_with_args('number_of_samples_per_pool:populate', nil, submission.reload.id) - rescue StandardError => e - error_message = e.message - end - - # Verify - expect(error_message).to eq('Number of samples per pool is missing') - end - end -end -# rubocop:enable RSpec/DescribeClass From f22de0acaa0a39004cb060efeb0bc582ac91ff92 Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:24:19 +0000 Subject: [PATCH 099/115] Rename the form field number_of_samples_per_pool to number_of_pools in the tube submission UAT action --- app/uat_actions/uat_actions/tube_submission.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/uat_actions/uat_actions/tube_submission.rb b/app/uat_actions/uat_actions/tube_submission.rb index db35dce31c..1b1d7c5194 100644 --- a/app/uat_actions/uat_actions/tube_submission.rb +++ b/app/uat_actions/uat_actions/tube_submission.rb @@ -31,11 +31,15 @@ class UatActions::TubeSubmission < UatActions include_blank: 'Using default library type...' } - form_field :number_of_samples_per_pool, + # The number_of_pools and cells_per_chip_well options are applied to a single + # study-project and set in request metadata. For multiple study and projects, + # a bulk submission is required. + + form_field :number_of_pools, :number_field, - label: 'Number of samples per pool', + label: 'Number of pools', help: - 'Optional field to set the number_of_samples_per_pool field on the ' \ + 'Optional field to set the number_of_pools field on the ' \ 'submission request. Leave blank if not required.', options: { minimum: 0 From 293f0092d17eaec69b87798b58cba20ca4e9487b Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:37:40 +0000 Subject: [PATCH 100/115] Rename number_of_samples_per_pool key to number_of_pools of report and request options in the tube submission UAT action --- app/uat_actions/uat_actions/tube_submission.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/uat_actions/uat_actions/tube_submission.rb b/app/uat_actions/uat_actions/tube_submission.rb index 1b1d7c5194..5ec6643bb9 100644 --- a/app/uat_actions/uat_actions/tube_submission.rb +++ b/app/uat_actions/uat_actions/tube_submission.rb @@ -103,8 +103,8 @@ def fill_report(order) report['tube_barcodes'] = assets.map(&:human_barcode) report['submission_id'] = order.submission.id report['library_type'] = order.request_options[:library_type] if order.request_options[:library_type].present? - report['number_of_samples_per_pool'] = order.request_options[:number_of_samples_per_pool] if order.request_options[ - :number_of_samples_per_pool + report['number_of_pools'] = order.request_options[:number_of_pools] if order.request_options[ + :number_of_pools ].present? report['cells_per_chip_well'] = order.request_options[:cells_per_chip_well] if order.request_options[ :cells_per_chip_well @@ -164,7 +164,7 @@ def default_request_options def custom_request_options options = {} options[:library_type] = library_type_name if library_type_name.present? - options[:number_of_samples_per_pool] = number_of_samples_per_pool.presence + options[:number_of_pools] = number_of_pools.presence options[:cells_per_chip_well] = cells_per_chip_well.presence options end From 9277b8cd2cf4710dd61ed9c1d2f365a58004a5f3 Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 16:40:25 +0000 Subject: [PATCH 101/115] Update the tube submission UAT action test with number_of_pools --- spec/uat_actions/tube_submission_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/uat_actions/tube_submission_spec.rb b/spec/uat_actions/tube_submission_spec.rb index b66d74b390..a48deb5efc 100644 --- a/spec/uat_actions/tube_submission_spec.rb +++ b/spec/uat_actions/tube_submission_spec.rb @@ -38,13 +38,13 @@ end end - context 'with optional number of samples per pool supplied' do - let(:num_samples) { 15 } + context 'with optional number of pools supplied' do + let(:number_of_pools) { 2 } let(:parameters) do { submission_template_name: submission_template.name, tube_barcodes: tube_barcode, - number_of_samples_per_pool: num_samples + number_of_pools: number_of_pools } end @@ -52,7 +52,7 @@ expect(uat_action.perform).to be true expect(uat_action.report['tube_barcodes']).to eq report['tube_barcodes'] expect(uat_action.report['submission_id']).to be_a Integer - expect(uat_action.report['number_of_samples_per_pool']).to eq num_samples + expect(uat_action.report['number_of_pools']).to eq number_of_pools end end From 8ca780676a1d169f0eb3421683125d47d4cacd90 Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 17:21:41 +0000 Subject: [PATCH 102/115] Rename the attribute number_of_samples_per_pool to number_of_pools in request_metadata_resource --- app/resources/api/v2/request_metadata_resource.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/resources/api/v2/request_metadata_resource.rb b/app/resources/api/v2/request_metadata_resource.rb index dc765f21a2..6e7b717d36 100644 --- a/app/resources/api/v2/request_metadata_resource.rb +++ b/app/resources/api/v2/request_metadata_resource.rb @@ -28,9 +28,9 @@ class RequestMetadataResource < BaseResource # Attributes ### - # @!attribute [r] number_of_samples_per_pool - # @return [Int] the number_of_samples_per_pool. - attribute :number_of_samples_per_pool, write_once: true + # @!attribute [r] number_of_pools + # @return [Int] the number_of_pools. + attribute :number_of_pools, write_once: true # @!attribute [r] cells_per_chip_well # @return [Int] the cells_per_chip_well. From 65e08634f5d1e026b60a7c0625ec393dc7c24cdc Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 2 Jan 2025 23:25:47 +0000 Subject: [PATCH 103/115] Update comment to use the word requested instead of allocated --- ...name_column_number_of_samples_per_pool_to_number_of_pools.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb b/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb index 7786c627da..81ce66e9c3 100644 --- a/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb +++ b/db/migrate/20250102154835_rename_column_number_of_samples_per_pool_to_number_of_pools.rb @@ -3,7 +3,7 @@ # This migration renames the column number_of_samples_per_pool to # number_of_pools in the request_metadata table for the scRNA cDNA Prep # submissions. The column will be used for storing the number of pools -# allocated for a study-project. +# requested for a study-project. class RenameColumnNumberOfSamplesPerPoolToNumberOfPools < ActiveRecord::Migration[7.0] def change rename_column :request_metadata, :number_of_samples_per_pool, :number_of_pools From 39b9b0648e275bd89315f7d4eeb4ac9a34b94b9c Mon Sep 17 00:00:00 2001 From: yoldas Date: Fri, 3 Jan 2025 09:18:27 +0000 Subject: [PATCH 104/115] Add more detail to request metadata resource docs as suggested in PR --- app/resources/api/v2/request_metadata_resource.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/resources/api/v2/request_metadata_resource.rb b/app/resources/api/v2/request_metadata_resource.rb index 6e7b717d36..504c8df145 100644 --- a/app/resources/api/v2/request_metadata_resource.rb +++ b/app/resources/api/v2/request_metadata_resource.rb @@ -29,11 +29,18 @@ class RequestMetadataResource < BaseResource ### # @!attribute [r] number_of_pools - # @return [Int] the number_of_pools. + # @return [Int] the number_of_pools requested in the Submission. As used + # in the scRNA Core pipeline, it is specified at the Study-Project + # level: it will have the same value for all Requests that share the + # same Study and Project. It is used in the pooling algorithm. attribute :number_of_pools, write_once: true # @!attribute [r] cells_per_chip_well - # @return [Int] the cells_per_chip_well. + # @return [Int] the cells_per_chip_well requested in the Submission. As + # used in the scRNA Core pipeline, it is specified at the Study-Project + # level: it will have the same value for all Requests that share the + # same Study and Project. It is used for volume calculations for + # pooling. attribute :cells_per_chip_well, write_once: true # Filters From 420eef3a26c533ec05fe2739714617785a2dfcfc Mon Sep 17 00:00:00 2001 From: yoldas Date: Fri, 3 Jan 2025 14:02:47 +0000 Subject: [PATCH 105/115] Add comment to specify that the config was copied from Limber and modified with additional entries --- config/initializers/scrna_config.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/initializers/scrna_config.rb b/config/initializers/scrna_config.rb index 14a4ca2ed6..522a9c398e 100644 --- a/config/initializers/scrna_config.rb +++ b/config/initializers/scrna_config.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This configuration was copied from Limber and modified with additional entries +# for the scRNA Core cDNA Prep submission. # Stores constants used in pooling and chip loading calculations for samples in the scRNA Core pipeline. Rails.application.config.scrna_config = { # Maximum volume to take into the pools plate for each sample (in microlitres) From 86a6c10d934ed3e0cca91d33cfe8993c3d3b6128 Mon Sep 17 00:00:00 2001 From: yoldas Date: Mon, 6 Jan 2025 10:01:31 +0000 Subject: [PATCH 106/115] Override the autogenerated poly_metadata method to pick the type correctly --- app/models/well.rb | 13 +++++++++++++ app/resources/api/v2/well_resource.rb | 2 ++ 2 files changed, 15 insertions(+) diff --git a/app/models/well.rb b/app/models/well.rb index d40ad694d6..398fb8bde5 100644 --- a/app/models/well.rb +++ b/app/models/well.rb @@ -48,6 +48,19 @@ def attach(wells) has_many :target_well_links, -> { stock }, class_name: 'Well::Link', foreign_key: :source_well_id has_many :target_wells, through: :target_well_links, source: :target_well + # Can have many key value pairs of metadata + has_many :poly_metadata, as: :metadatable, dependent: :destroy + + # Returns a collection of PolyMetadatum records associated with the Well. + # This method overrides the autogenerated poly_metadata method to pick the + # correct metadatable_type instead of the parent type. Without this override, + # the generated SQL uses the wrong class name, e.g. Receptacle instead of Well. + # + # @return [ActiveRecord::Relation] a collection of PolyMetadatum records + def poly_metadata + PolyMetadatum.where(metadatable_id: id, metadatable_type: self.class.name) + end + belongs_to :plate, foreign_key: :labware_id has_one :well_attribute, inverse_of: :well diff --git a/app/resources/api/v2/well_resource.rb b/app/resources/api/v2/well_resource.rb index 21574734df..273d24aec1 100644 --- a/app/resources/api/v2/well_resource.rb +++ b/app/resources/api/v2/well_resource.rb @@ -21,6 +21,8 @@ class WellResource < BaseResource # Attributes attribute :position, readonly: true + has_many :poly_metadata, as: :metadatable, class_name: 'PolyMetadatum' + def position { 'name' => _model.map_description } end From fdc2e1ae33a337194727f9ceaa94ff5d6e819de4 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 8 Jan 2025 11:34:52 +0000 Subject: [PATCH 107/115] feat(study): updates study edit dropdown options to display old options if present --- app/models/study.rb | 39 +++++++++++++++++-- .../shared/metadata/edit/_study.html.erb | 10 ++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index 92ea477ce2..2fd7aadf7c 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -83,6 +83,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength DATA_RELEASE_TIMING_DELAYED ].freeze + OLD_DATA_RELEASE_PREVENTION_REASONS = ['data validity', 'legal', 'replication of data subset'].freeze DATA_RELEASE_PREVENTION_REASONS = [ 'Pilot or validation studies - DAC approval not required', 'Collaborators will share data in a research repository - DAC approval not required', @@ -91,6 +92,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength 'Other (please specify)' ].freeze + OLD_DATA_RELEASE_DELAY_REASONS = ['other', 'phd study'].freeze DATA_RELEASE_DELAY_FOR_OTHER = 'Other (with free text box)' DATA_RELEASE_DELAY_REASONS_STANDARD = [ 'PhD study', @@ -99,7 +101,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength 'Additional time to make data FAIR', DATA_RELEASE_DELAY_FOR_OTHER ].freeze - DATA_RELEASE_DELAY_REASONS_ASSAY = ['assay of no other use', *DATA_RELEASE_DELAY_REASONS_STANDARD].freeze + DATA_RELEASE_DELAY_REASONS_ASSAY = ['assay of no other use'].freeze DATA_RELEASE_DELAY_PERIODS = ['3 months', '6 months', '9 months', '12 months', '18 months'].freeze @@ -227,7 +229,7 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute( :data_release_delay_reason, required: true, - in: DATA_RELEASE_DELAY_REASONS_ASSAY, + in: [*DATA_RELEASE_DELAY_REASONS_STANDARD, *DATA_RELEASE_DELAY_REASONS_ASSAY, *OLD_DATA_RELEASE_DELAY_REASONS], if: :delayed_release? ) @@ -246,7 +248,11 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute(:array_express_accession_number) with_options(if: :never_release?) do - custom_attribute(:data_release_prevention_reason, in: DATA_RELEASE_PREVENTION_REASONS, required: true) + custom_attribute( + :data_release_prevention_reason, + in: [*DATA_RELEASE_PREVENTION_REASONS, *OLD_DATA_RELEASE_PREVENTION_REASONS], + required: true + ) custom_attribute(:data_release_prevention_reason_comment, required: true) custom_attribute(:data_release_prevention_approval) end @@ -577,6 +583,33 @@ def poly_metadatum_by_key(key) poly_metadata.find { |pm| pm.key == key.to_s } end + # Helper method for edit dropdowns to support backwards compatibility with old options. + # + # @return [Array] the list of options for the data release prevention reason dropdown + def data_release_prevention_options + additional_options = [] + if OLD_DATA_RELEASE_PREVENTION_REASONS.include? study_metadata.data_release_prevention_reason + additional_options << study_metadata.data_release_prevention_reason + end + + additional_options + DATA_RELEASE_PREVENTION_REASONS + end + + # Helper method for edit dropdowns to support backwards compatibility with old options. + # + # @param [Boolean] assay_option - whether to include assay-specific options + # @return [Array] the list of options for the data release delay reason dropdown + def data_release_delay_options(assay_option: false) + # If the current value is an old one, then we need to include it in the list of options + additional_options = [] + if OLD_DATA_RELEASE_DELAY_REASONS.include? study_metadata.data_release_delay_reason + additional_options << study_metadata.data_release_delay_reason + end + + additional_options << DATA_RELEASE_DELAY_REASONS_ASSAY if assay_option + additional_options + DATA_RELEASE_DELAY_REASONS_STANDARD + end + private def valid_ethically_approved? diff --git a/app/views/shared/metadata/edit/_study.html.erb b/app/views/shared/metadata/edit/_study.html.erb index d754aaddb4..3012fbb9e2 100644 --- a/app/views/shared/metadata/edit/_study.html.erb +++ b/app/views/shared/metadata/edit/_study.html.erb @@ -59,22 +59,22 @@ %> <% metadata_fields.related_fields(to: :data_release_strategy, when: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> - <%= group.select(:data_release_prevention_reason, [''] + Study::DATA_RELEASE_PREVENTION_REASONS) %> + <%= group.select(:data_release_prevention_reason, study.data_release_prevention_options) %> <%= group.text_area(:data_release_prevention_approval) %> <%= group.text_area(:data_release_prevention_reason_comment) %> <% end %> <% metadata_fields.related_fields(to: :data_release_strategy, in: Study::DATA_RELEASE_STRATEGIES, not: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> <% metadata_fields.related_fields(to: :data_release_timing, when: Study::DATA_RELEASE_TIMING_DELAYED) do %> - <%= group.select(:data_release_delay_reason, [''] + Study::DATA_RELEASE_DELAY_REASONS_STANDARD) %> + <%= group.select(:data_release_delay_reason, study.data_release_delay_options) %> <% group.change_select_options_for(:data_release_delay_reason, when: :data_release_study_type_id, values: { - DataReleaseStudyType.assay_types.map(&:id) => [ '' ] + Study::DATA_RELEASE_DELAY_REASONS_ASSAY, - DataReleaseStudyType.non_assay_types.map(&:id) => [ '' ] + Study::DATA_RELEASE_DELAY_REASONS_STANDARD + DataReleaseStudyType.assay_types.map(&:id) => [ '' ] + study.data_release_delay_options(assay_option: true), + DataReleaseStudyType.non_assay_types.map(&:id) => [ '' ] + study.data_release_delay_options }) %> <%= group.select(:data_release_delay_period, Study::DATA_RELEASE_DELAY_PERIODS) %> - <% metadata_fields.related_fields(to: :data_release_delay_reason, when: Study::DATA_RELEASE_DELAY_FOR_OTHER) do %> + <% metadata_fields.related_fields(to: :data_release_delay_reason, in: [Study::DATA_RELEASE_DELAY_FOR_OTHER, 'other']) do %> <%= group.text_area(:data_release_delay_other_comment) %> <% metadata_fields.related_fields(to: :data_release_delay_period, in: Study::DATA_RELEASE_DELAY_PERIODS) do %> <%= group.text_area(:data_release_delay_reason_comment) %> From c400f4e949594a23dc27c4624ddc177605dd14f9 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 8 Jan 2025 13:37:27 +0000 Subject: [PATCH 108/115] tests(study): adds tests for dropdown option generation functions --- app/models/study.rb | 10 ++++---- spec/models/study_spec.rb | 52 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index 2fd7aadf7c..5b6e30bda0 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -584,7 +584,7 @@ def poly_metadatum_by_key(key) end # Helper method for edit dropdowns to support backwards compatibility with old options. - # + # # @return [Array] the list of options for the data release prevention reason dropdown def data_release_prevention_options additional_options = [] @@ -592,7 +592,7 @@ def data_release_prevention_options additional_options << study_metadata.data_release_prevention_reason end - additional_options + DATA_RELEASE_PREVENTION_REASONS + DATA_RELEASE_PREVENTION_REASONS + additional_options end # Helper method for edit dropdowns to support backwards compatibility with old options. @@ -605,9 +605,9 @@ def data_release_delay_options(assay_option: false) if OLD_DATA_RELEASE_DELAY_REASONS.include? study_metadata.data_release_delay_reason additional_options << study_metadata.data_release_delay_reason end - - additional_options << DATA_RELEASE_DELAY_REASONS_ASSAY if assay_option - additional_options + DATA_RELEASE_DELAY_REASONS_STANDARD + + additional_options.concat(DATA_RELEASE_DELAY_REASONS_ASSAY) if assay_option + DATA_RELEASE_DELAY_REASONS_STANDARD + additional_options end private diff --git a/spec/models/study_spec.rb b/spec/models/study_spec.rb index 696170a97f..5f63d476fa 100644 --- a/spec/models/study_spec.rb +++ b/spec/models/study_spec.rb @@ -426,6 +426,58 @@ end end + describe '#data_release_prevention_options' do + subject { study.data_release_prevention_options } + + let(:study) { create(:study) } + + context 'when there is no existing data release prevention reason' do + it { is_expected.to eq Study::DATA_RELEASE_PREVENTION_REASONS } + end + + context 'when there is an existing data release prevention reason' do + before { study.study_metadata.data_release_prevention_reason = 'Protecting IP - DAC approval required' } + + it { is_expected.to eq Study::DATA_RELEASE_PREVENTION_REASONS } + end + + context 'when there is an existing legacy data release prevention reason' do + before { study.study_metadata.data_release_prevention_reason = 'data validity' } + + it { is_expected.to eq [*Study::DATA_RELEASE_PREVENTION_REASONS, 'data validity'] } + end + end + + describe '#data_release_delay_options' do + subject { study.data_release_delay_options } + + let(:study) { create(:study) } + + context 'when there is no existing data release delay reason' do + it { is_expected.to eq Study::DATA_RELEASE_DELAY_REASONS_STANDARD } + end + + context 'when there is an existing data release delay reason' do + before { study.study_metadata.data_release_delay_reason = 'Capacity building' } + + it { is_expected.to eq Study::DATA_RELEASE_DELAY_REASONS_STANDARD } + end + + context 'when there is an existing legacy data release delay reason' do + before { study.study_metadata.data_release_delay_reason = 'phd study' } + + it { is_expected.to eq [*Study::DATA_RELEASE_DELAY_REASONS_STANDARD, 'phd study'] } + end + + context 'when the data release delay options include assays' do + it 'includes the assay options' do + expect(study.data_release_delay_options(assay_option: true)).to eq( + [*Study::DATA_RELEASE_DELAY_REASONS_STANDARD, *Study::DATA_RELEASE_DELAY_REASONS_ASSAY] + ) + end + end + end + describe 'metadata' do let(:metadata) do { From 108220151cecc236a0e96c0ba4debf52988252ab Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 8 Jan 2025 13:48:42 +0000 Subject: [PATCH 109/115] fix(study): fixes old other option not firing mandatory validation --- app/models/study.rb | 5 +++-- app/views/shared/metadata/edit/_study.html.erb | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/models/study.rb b/app/models/study.rb index 5b6e30bda0..bc00c799da 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -92,8 +92,9 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength 'Other (please specify)' ].freeze - OLD_DATA_RELEASE_DELAY_REASONS = ['other', 'phd study'].freeze + OLD_DATA_RELEASE_DELAY_FOR_OTHER = 'other' DATA_RELEASE_DELAY_FOR_OTHER = 'Other (with free text box)' + OLD_DATA_RELEASE_DELAY_REASONS = ['other', 'phd study'].freeze DATA_RELEASE_DELAY_REASONS_STANDARD = [ 'PhD study', 'Capacity building', @@ -644,7 +645,7 @@ def never_release? end def delayed_for_other_reasons? - data_release_delay_reason == DATA_RELEASE_DELAY_FOR_OTHER + [DATA_RELEASE_DELAY_FOR_OTHER, OLD_DATA_RELEASE_DELAY_FOR_OTHER].include?(data_release_delay_reason) end def delayed_for_long_time? diff --git a/app/views/shared/metadata/edit/_study.html.erb b/app/views/shared/metadata/edit/_study.html.erb index 3012fbb9e2..b174425fd9 100644 --- a/app/views/shared/metadata/edit/_study.html.erb +++ b/app/views/shared/metadata/edit/_study.html.erb @@ -74,7 +74,7 @@ %> <%= group.select(:data_release_delay_period, Study::DATA_RELEASE_DELAY_PERIODS) %> - <% metadata_fields.related_fields(to: :data_release_delay_reason, in: [Study::DATA_RELEASE_DELAY_FOR_OTHER, 'other']) do %> + <% metadata_fields.related_fields(to: :data_release_delay_reason, in: [Study::DATA_RELEASE_DELAY_FOR_OTHER, Study::OLD_DATA_RELEASE_DELAY_FOR_OTHER]) do %> <%= group.text_area(:data_release_delay_other_comment) %> <% metadata_fields.related_fields(to: :data_release_delay_period, in: Study::DATA_RELEASE_DELAY_PERIODS) do %> <%= group.text_area(:data_release_delay_reason_comment) %> From ec2447376518bed8198e46075eefd1d784acfffe Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Tue, 14 Jan 2025 15:34:56 +0000 Subject: [PATCH 110/115] feat(study): adds data_release_prevention_other_comment to study metadata, removes redundant fields --- app/models/study.rb | 18 +++++-- .../shared/metadata/edit/_study.html.erb | 6 +-- .../shared/metadata/show/_study.html.erb | 1 + config/locales/metadata/en.yml | 4 +- ...vention_other_comment_to_study_metadata.rb | 6 +++ db/schema.rb | 3 +- .../8447221_data_release_help_text.feature | 6 +-- features/studies/data_release_timings.feature | 52 +++++-------------- .../support/step_definitions/study_steps.rb | 5 +- spec/models/study_spec.rb | 11 +++- 10 files changed, 56 insertions(+), 56 deletions(-) create mode 100644 db/migrate/20250114135342_add_data_release_prevention_other_comment_to_study_metadata.rb diff --git a/app/models/study.rb b/app/models/study.rb index bc00c799da..f893ad111b 100644 --- a/app/models/study.rb +++ b/app/models/study.rb @@ -84,16 +84,17 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength ].freeze OLD_DATA_RELEASE_PREVENTION_REASONS = ['data validity', 'legal', 'replication of data subset'].freeze + DATA_RELEASE_PREVENTION_REASON_OTHER = 'Other (please specify)' DATA_RELEASE_PREVENTION_REASONS = [ 'Pilot or validation studies - DAC approval not required', 'Collaborators will share data in a research repository - DAC approval not required', 'Prevent harm (e.g sensitive studies or biosecurity) - DAC approval required', 'Protecting IP - DAC approval required', - 'Other (please specify)' + DATA_RELEASE_PREVENTION_REASON_OTHER ].freeze OLD_DATA_RELEASE_DELAY_FOR_OTHER = 'other' - DATA_RELEASE_DELAY_FOR_OTHER = 'Other (with free text box)' + DATA_RELEASE_DELAY_FOR_OTHER = 'Other (please specify below)' OLD_DATA_RELEASE_DELAY_REASONS = ['other', 'phd study'].freeze DATA_RELEASE_DELAY_REASONS_STANDARD = [ 'PhD study', @@ -237,8 +238,8 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength custom_attribute(:data_release_delay_period, required: true, in: DATA_RELEASE_DELAY_PERIODS, if: :delayed_release?) custom_attribute(:bam, default: true) - with_options(required: true, if: :delayed_for_other_reasons?) do - custom_attribute(:data_release_delay_other_comment) + with_options(if: :delayed_for_other_reasons?) do + custom_attribute(:data_release_delay_other_comment, required: true) custom_attribute(:data_release_delay_reason_comment) end @@ -254,6 +255,11 @@ class Study < ApplicationRecord # rubocop:todo Metrics/ClassLength in: [*DATA_RELEASE_PREVENTION_REASONS, *OLD_DATA_RELEASE_PREVENTION_REASONS], required: true ) + custom_attribute( + :data_release_prevention_other_comment, + required: true, + if: :data_release_prevention_reason_other? + ) custom_attribute(:data_release_prevention_reason_comment, required: true) custom_attribute(:data_release_prevention_approval) end @@ -648,6 +654,10 @@ def delayed_for_other_reasons? [DATA_RELEASE_DELAY_FOR_OTHER, OLD_DATA_RELEASE_DELAY_FOR_OTHER].include?(data_release_delay_reason) end + def data_release_prevention_reason_other? + data_release_prevention_reason == DATA_RELEASE_PREVENTION_REASON_OTHER + end + def delayed_for_long_time? DATA_RELEASE_DELAY_PERIODS.include?(data_release_delay_period) end diff --git a/app/views/shared/metadata/edit/_study.html.erb b/app/views/shared/metadata/edit/_study.html.erb index b174425fd9..8741a9f4aa 100644 --- a/app/views/shared/metadata/edit/_study.html.erb +++ b/app/views/shared/metadata/edit/_study.html.erb @@ -60,6 +60,9 @@ <% metadata_fields.related_fields(to: :data_release_strategy, when: Study::DATA_RELEASE_STRATEGY_NOT_APPLICABLE) do %> <%= group.select(:data_release_prevention_reason, study.data_release_prevention_options) %> + <% metadata_fields.related_fields(to: :data_release_prevention_reason, in: [Study::DATA_RELEASE_PREVENTION_REASON_OTHER]) do %> + <%= group.text_area(:data_release_prevention_other_comment) %> + <% end %> <%= group.text_area(:data_release_prevention_approval) %> <%= group.text_area(:data_release_prevention_reason_comment) %> <% end %> @@ -76,9 +79,6 @@ <%= group.select(:data_release_delay_period, Study::DATA_RELEASE_DELAY_PERIODS) %> <% metadata_fields.related_fields(to: :data_release_delay_reason, in: [Study::DATA_RELEASE_DELAY_FOR_OTHER, Study::OLD_DATA_RELEASE_DELAY_FOR_OTHER]) do %> <%= group.text_area(:data_release_delay_other_comment) %> - <% metadata_fields.related_fields(to: :data_release_delay_period, in: Study::DATA_RELEASE_DELAY_PERIODS) do %> - <%= group.text_area(:data_release_delay_reason_comment) %> - <% end %> <% end %> <% end %> <% end %> diff --git a/app/views/shared/metadata/show/_study.html.erb b/app/views/shared/metadata/show/_study.html.erb index d6ebe29a7c..1030ceeb5d 100644 --- a/app/views/shared/metadata/show/_study.html.erb +++ b/app/views/shared/metadata/show/_study.html.erb @@ -57,6 +57,7 @@ <%= group.plain_value(:data_release_prevention_reason) %> <%= group.plain_value(:data_release_prevention_approval) %> <%= group.plain_value(:data_release_prevention_reason_comment) %> + <%= group.plain_value(:data_release_prevention_other_comment) %> <% end %> <%= metadata_fields.plain_value(:dac_policy_title) %> <%= metadata_fields.plain_value(:dac_policy) %> diff --git a/config/locales/metadata/en.yml b/config/locales/metadata/en.yml index 6e0890bb9d..ae7b3a5084 100644 --- a/config/locales/metadata/en.yml +++ b/config/locales/metadata/en.yml @@ -441,7 +441,9 @@ en: data_release_prevention_approval: label: If reason for exemption requires DAC approval, what is the approval number? - help: "If this is for data validity reasons: approval from the sponsor is required
    If this is for legal reasons: approval from the Data Sharing Committee is required (please contact sd4)
    " + + data_release_prevention_other_comment: + label: Please explain the reason for preventing data release data_release_prevention_reason_comment: label: Comment regarding prevention of data release and approval diff --git a/db/migrate/20250114135342_add_data_release_prevention_other_comment_to_study_metadata.rb b/db/migrate/20250114135342_add_data_release_prevention_other_comment_to_study_metadata.rb new file mode 100644 index 0000000000..1caaa449b2 --- /dev/null +++ b/db/migrate/20250114135342_add_data_release_prevention_other_comment_to_study_metadata.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +class AddDataReleasePreventionOtherCommentToStudyMetadata < ActiveRecord::Migration[7.0] + def change + add_column :study_metadata, :data_release_prevention_other_comment, :string, default: nil + end +end diff --git a/db/schema.rb b/db/schema.rb index 187e3c4e3e..0174acc85c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_12_11_143636) do +ActiveRecord::Schema[7.0].define(version: 2025_01_14_135342) do create_table "aliquot_indices", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_ci", options: "ENGINE=InnoDB ROW_FORMAT=DYNAMIC", force: :cascade do |t| t.integer "aliquot_id", null: false t.integer "lane_id", null: false @@ -1548,6 +1548,7 @@ t.string "s3_email_list" t.string "data_deletion_period" t.string "contaminated_human_data_access_group" + t.string "data_release_prevention_other_comment" t.index ["faculty_sponsor_id"], name: "index_study_metadata_on_faculty_sponsor_id" t.index ["study_id"], name: "index_study_metadata_on_study_id" end diff --git a/features/studies/8447221_data_release_help_text.feature b/features/studies/8447221_data_release_help_text.feature index dd79263d3e..e5ab02f06a 100644 --- a/features/studies/8447221_data_release_help_text.feature +++ b/features/studies/8447221_data_release_help_text.feature @@ -22,7 +22,7 @@ Feature: Update the data release fields for creating a study Scenario Outline: Add help text opposite delay drop down (4044305) When I choose "" from "What is the data release strategy for this study?" When I select "delayed" from "How is the data release to be timed?" - When I select "Other (with free text box)" from "Reason for delaying release" + When I select "Other (please specify below)" from "Reason for delaying release" Then I should exactly see "Reason for delaying release" Examples: @@ -41,15 +41,13 @@ Feature: Update the data release fields for creating a study Scenario Outline: Delaying for 3 months should have the same questions as all other delays (4044273) When I select "delayed" from "How is the data release to be timed?" - And I select "Other (with free text box)" from "Reason for delaying release" + And I select "Other (please specify below)" from "Reason for delaying release" And I select "" from "Delay for" - And I should exactly see "Comment regarding data release timing and approval" When I fill in the following: | Study name | new study | | Study description | writing cukes | | Please explain the reason for delaying release | some comment | - | Comment regarding data release timing and approval | another comment | And I select "Jack Sponsor" from "Faculty Sponsor" And I choose "Yes" from "Do any of the samples in this study contain human DNA?" diff --git a/features/studies/data_release_timings.feature b/features/studies/data_release_timings.feature index cc6562c638..38458bf41a 100644 --- a/features/studies/data_release_timings.feature +++ b/features/studies/data_release_timings.feature @@ -29,57 +29,33 @@ Feature: Studies have timings for release of their data Scenario: When the data release is delayed for PhD study Given I select "delayed" from "How is the data release to be timed?" And I select "PhD study" from "Reason for delaying release" - Then the "Comment regarding data release timing and approval" field is hidden When I select "6 months" from "Delay for" And I press "Create" Then I should be on the study information page for "Testing data release strategies" And I should see "Your study has been created" - Scenario Outline: When the data release is delayed but no reasons are provided - Given I select "delayed" from "How is the data release to be timed?" - And I select "Other (with free text box)" from "Reason for delaying release" - And I fill in "Please explain the reason for delaying release" with "Some reason" - And I select "" from "Delay for" + Scenario: When the data release is never but the prevention other comment is not supplied + When I choose "Not Applicable" from "What is the data release strategy for this study?" + And I select "never" from "How is the data release to be timed?" + And I select "Other (please specify)" from "What is the reason for preventing data release?" + And I fill in "If reason for exemption requires DAC approval, what is the approval number?" with "12345" When I press "Create" Then I should be on the studies page - # Ideally this should be without the study metadata qualification - And I should see "Study metadata data release delay reason comment can't be blank" - - Examples: - | period | - | 3 months | - | 6 months | - | 9 months | - | 12 months | - - Scenario Outline: When the data release is delayed and the reasons are provided - Given I select "delayed" from "How is the data release to be timed?" - And I select "Other (with free text box)" from "Reason for delaying release" - And I fill in "Please explain the reason for delaying release" with "Some reason" - And I select "" from "Delay for" - And I fill in "Comment regarding data release timing and approval" with "Because it is ok?" - When I press "Create" - Then I should be on the study information page for "Testing data release strategies" - And I should see "Your study has been created" - - Examples: - | period | - | 3 months | - | 6 months | - | 9 months | - | 12 months | + # Again, ideally without study metadata + And I should see "Study metadata data release prevention other comment can't be blank" - Scenario: When the data release is never but the comment is not supplied + Scenario: When the data release is never and the prevention other comment is supplied When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" - And I select "Protecting IP - DAC approval required" from "What is the reason for preventing data release?" + And I select "Other (please specify)" from "What is the reason for preventing data release?" + And I fill in "Please explain the reason for preventing data release" with "Some reason" + And I fill in "Comment regarding prevention of data release and approval" with "Some reason" And I fill in "If reason for exemption requires DAC approval, what is the approval number?" with "12345" When I press "Create" - Then I should be on the studies page - # Again, ideally without study metadata - And I should see "Study metadata data release prevention reason comment can't be blank" + Then I should be on the study information page for "Testing data release strategies" + And I should see "Your study has been created" - Scenario: When the data release is never and the comment is supplied + Scenario: When the data release is never and the prevention comment is supplied When I choose "Not Applicable" from "What is the data release strategy for this study?" And I select "never" from "How is the data release to be timed?" And I select "Protecting IP - DAC approval required" from "What is the reason for preventing data release?" diff --git a/features/support/step_definitions/study_steps.rb b/features/support/step_definitions/study_steps.rb index 86f220fab3..05798b7dfb 100644 --- a/features/support/step_definitions/study_steps.rb +++ b/features/support/step_definitions/study_steps.rb @@ -159,10 +159,9 @@ def given_study_metadata(attribute, regexp) study.update!( study_metadata_attributes: { data_release_timing: 'delayed', - data_release_delay_reason: 'Other (with free text box)', + data_release_delay_reason: 'Other (please specify below)', data_release_delay_other_comment: reason, - data_release_delay_period: "#{period} months", - data_release_delay_reason_comment: reason + data_release_delay_period: "#{period} months" } ) end diff --git a/spec/models/study_spec.rb b/spec/models/study_spec.rb index 5f63d476fa..0b26b961e3 100644 --- a/spec/models/study_spec.rb +++ b/spec/models/study_spec.rb @@ -508,9 +508,10 @@ ega_policy_accession_number: 'POL123456', array_express_accession_number: 'ARR123456', data_release_delay_approval: 'Yes', - data_release_prevention_reason: 'Protecting IP - DAC approval required', + data_release_prevention_reason: 'Other (please specify)', data_release_prevention_approval: 'Yes', data_release_prevention_reason_comment: 'Data Release prevention reason comment', + data_release_prevention_other_comment: 'Data Release prevention other comment', data_access_group: 'something', snp_study_id: 123_456, snp_parent_study_id: 123_456, @@ -767,7 +768,7 @@ study_metadata: create( :study_metadata, - metadata.merge(data_release_timing: 'delayed', data_release_delay_reason: 'Other (with free text box)') + metadata.merge(data_release_timing: 'delayed', data_release_delay_reason: 'Other (please specify below)') ) ) end @@ -818,6 +819,12 @@ ) end + it 'will have a data_release_prevention_other_comment' do + expect(study.study_metadata.data_release_prevention_other_comment).to eq( + metadata[:data_release_prevention_other_comment] + ) + end + context 'when the data_release_timing validation is switched on' do before { Flipper.enable :y24_052_enable_data_release_timing_validation } From befe5d4fdf08b4e1fe242eb48e9a8300fa325686 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Tue, 14 Jan 2025 15:54:09 +0000 Subject: [PATCH 111/115] tests(cucumber): fixes test looking for removed text --- features/studies/8447221_data_release_help_text.feature | 9 --------- 1 file changed, 9 deletions(-) diff --git a/features/studies/8447221_data_release_help_text.feature b/features/studies/8447221_data_release_help_text.feature index e5ab02f06a..bfda315bda 100644 --- a/features/studies/8447221_data_release_help_text.feature +++ b/features/studies/8447221_data_release_help_text.feature @@ -30,15 +30,6 @@ Feature: Update the data release fields for creating a study | Managed (EGA) | | Open (ENA) | - Scenario: Add help text to has this been approved for never release (4044343) - When I choose "Not Applicable" from "What is the data release strategy for this study?" - When I select "never" from "How is the data release to be timed?" - Then the help text for "If reason for exemption requires DAC approval, what is the approval number?" should contain: - """ - If this is for data validity reasons: approval from the sponsor is required - If this is for legal reasons: approval from the Data Sharing Committee is required (please contact sd4) - """ - Scenario Outline: Delaying for 3 months should have the same questions as all other delays (4044273) When I select "delayed" from "How is the data release to be timed?" And I select "Other (please specify below)" from "Reason for delaying release" From 7b18b96bb2774649c35287257ea84e21f95408bf Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Tue, 14 Jan 2025 16:19:50 +0000 Subject: [PATCH 112/115] tests(cucumber): fixes study xml api response --- ...5391_study_xml_needs_to_be_reverted_to_old_version.feature | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature b/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature index 8057e24450..83ce342e94 100644 --- a/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature +++ b/features/studies/4295391_study_xml_needs_to_be_reverted_to_old_version.feature @@ -174,6 +174,10 @@ Feature: The XML for the sequencescape API Please explain the reason for delaying release + + Please explain the reason for preventing data release + + SNP parent study ID From d0f93bbf435caeba8b0bf4b7e7922a7d1c88845e Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 22:35:32 +0000 Subject: [PATCH 113/115] Update ruby-units to version 4.1.0 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1fdb646f0b..0b5e34510e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -456,7 +456,7 @@ GEM rexml ruby-prof (1.7.1) ruby-progressbar (1.13.0) - ruby-units (4.0.3) + ruby-units (4.1.0) ruby-vips (2.2.1) ffi (~> 1.12) ruby2_keywords (0.0.5) From b37c5a0c56898eaca0a454be8951270fa8282460 Mon Sep 17 00:00:00 2001 From: Ben Topping Date: Wed, 15 Jan 2025 10:02:11 +0000 Subject: [PATCH 114/115] misc(study): remove helper text for study description --- config/locales/metadata/en.yml | 1 - .../studies/8447221_data_release_help_text.feature | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/config/locales/metadata/en.yml b/config/locales/metadata/en.yml index ae7b3a5084..0a3610d094 100644 --- a/config/locales/metadata/en.yml +++ b/config/locales/metadata/en.yml @@ -351,7 +351,6 @@ en: study_description: label: Study description edit_info: "ENA requirement" - help: Please choose one of the following 2 standard statements to be included with your data submissions (one or the other, depending on the study). If you use the second statement, replace [doi or ref] by a reference or doi for your publication:

    This data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria), please see http://www.sanger.ac.uk/datasharing/

    OR

    This data has been described in the following article [doi or ref] and its further analysis can be freely submitted for publication. For information on the proper use of data shared by the Wellcome Trust Sanger Institute (including information on acknowledgement), please see http://www.sanger.ac.uk/datasharing/

    If applicable, include a brief description of any restrictions on data usage, e.g. 'For AIDS-related research only'

    contaminated_human_dna: label: Does this study contain samples that are contaminated with human DNA which must be removed prior to analysis? diff --git a/features/studies/8447221_data_release_help_text.feature b/features/studies/8447221_data_release_help_text.feature index bfda315bda..673c5cd658 100644 --- a/features/studies/8447221_data_release_help_text.feature +++ b/features/studies/8447221_data_release_help_text.feature @@ -6,19 +6,6 @@ Feature: Update the data release fields for creating a study Given a faculty sponsor called "Jack Sponsor" exists And I am on the new study page - Scenario: Add help text to study description (8348119) - Then the help text for "Study description" should contain: - """ - Please choose one of the following 2 standard statements to be included with your data submissions (one or the other, depending on the study). If you use the second statement, replace [doi or ref] by a reference or doi for your publication: - - This data is part of a pre-publication release. For information on the proper use of pre-publication data shared by the Wellcome Trust Sanger Institute (including details of any publication moratoria), please see http://www.sanger.ac.uk/datasharing/ - - OR - - This data has been described in the following article [doi or ref] and its further analysis can be freely submitted for publication. For information on the proper use of data shared by the Wellcome Trust Sanger Institute (including information on acknowledgement), please see http://www.sanger.ac.uk/datasharing/ - If applicable, include a brief description of any restrictions on data usage, e.g. 'For AIDS-related research only' - """ - Scenario Outline: Add help text opposite delay drop down (4044305) When I choose "" from "What is the data release strategy for this study?" When I select "delayed" from "How is the data release to be timed?" From 21ad54bddd353064f2660eba4005b40b0b62c2cc Mon Sep 17 00:00:00 2001 From: yoldas Date: Thu, 16 Jan 2025 01:44:01 +0000 Subject: [PATCH 115/115] Fix LCMT DNA Adp Lig purpose record --- .../013_lcm_triomics_plate_purposes.wip.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/config/default_records/plate_purposes/013_lcm_triomics_plate_purposes.wip.yml b/config/default_records/plate_purposes/013_lcm_triomics_plate_purposes.wip.yml index 097cf34ca4..2a21d32552 100644 --- a/config/default_records/plate_purposes/013_lcm_triomics_plate_purposes.wip.yml +++ b/config/default_records/plate_purposes/013_lcm_triomics_plate_purposes.wip.yml @@ -1,10 +1,21 @@ # Plate purposes for LCM Triomics WGS and EMSeq --- +# The starting point for the 'LCM Triomics' pipeline is the LCMT Lysate plate. +# It is defined here so it can be added to the acceptable purposes for the +# 'LCM Triomics' manual submission. It is an input plate and so it is 'passed' +# when all non-empty wells have requests out of them, i.e., when the submission +# is built. LCMT Lysate: type: PlatePurpose::Input stock_plate: true cherrypickable_target: false +# Five plates of the LCM Triomics pipeline all go from 'pending' to 'started' +# at 'Bravo LCMT EMSeq Verify Initial Setup' and then to 'passed' one by one in +# either single bed verification or manual transfer. None of them can be input +# plates. The LCMT DNA Adp Lig purpose is defined here so it can be added to the +# acceptable purposes for the 'LCM Triomics WGS' automated submission, which +# is enabled when LCMT DNA Adp Lig is 'passed'. LCMT DNA Adp Lig: - type: PlatePurpose::Input + type: PlatePurpose stock_plate: false cherrypickable_target: false