From 95432808f7b119cf5788b7095290171ad4a28e52 Mon Sep 17 00:00:00 2001 From: Jacob Penner Date: Fri, 6 Sep 2024 17:16:56 -0400 Subject: [PATCH] more changes to make VFF forms work --- .../simple_forms_api/v1/uploads_controller.rb | 11 +- .../s3_service/archive_submission_to_pdf.rb | 66 ++++---- .../s3_service/submission_archive_handler.rb | 10 +- .../submission_archive_handler_job.rb | 4 +- .../user_submission_archive_handler.rb | 2 +- .../simple_forms_api/s3_service/utils.rb | 2 +- .../archive_submission_to_pdf_spec.rb | 142 ++++++++++++++++++ 7 files changed, 191 insertions(+), 46 deletions(-) create mode 100644 modules/simple_forms_api/spec/services/s3_service/archive_submission_to_pdf_spec.rb diff --git a/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb b/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb index ef8d1115565..a6a85ed9f87 100644 --- a/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb +++ b/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb @@ -169,9 +169,14 @@ def get_file_paths_and_metadata(parsed_form_data) end def upload_pdf(file_path, metadata, form) - location, uuid = prepare_for_upload(form) + location, uuid, submission_attempt = prepare_for_upload(form) log_upload_details(location, uuid) response = perform_pdf_upload(location, file_path, metadata) + SimpleFormsApi::S3Service::SubmissionArchiveHandlerJob.perform_async( + submission_ids: [submission_attempt.form_submission.id], + metadata:, + file_path: + ) [response.status, uuid] end @@ -181,9 +186,9 @@ def prepare_for_upload(form) form_id: get_form_id) location, uuid = lighthouse_service.request_upload stamp_pdf_with_uuid(form, uuid) - create_form_submission_attempt(uuid) + submission_attempt = create_form_submission_attempt(uuid) - [location, uuid] + [location, uuid, submission_attempt] end def stamp_pdf_with_uuid(form, uuid) diff --git a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/archive_submission_to_pdf.rb b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/archive_submission_to_pdf.rb index d2d9994df8a..ef594e461e2 100644 --- a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/archive_submission_to_pdf.rb +++ b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/archive_submission_to_pdf.rb @@ -10,7 +10,7 @@ # 3. search for dsva-vetsgov-prod-reports # 4. search for your parent_dir name, e.g. 526archive_aug_21st_2024 # -# If you do not provide a parent_dir, the script defaults to a folder called wipn8923-test +# If you do not provide a parent_dir, the script defaults to a folder called vff-simple-forms # # OPTION 1: Run the script with user groupings # - requires SubmissionDuplicateReport object @@ -22,7 +22,7 @@ module SimpleFormsApi module S3Service class ArchiveSubmissionToPdf < SimpleFormsApi::S3Service::Utils - attr_reader :failures, :form_id, :include_json_archive, :include_text_archive, + attr_reader :failures, :include_json_archive, :include_text_archive, :metadata, :parent_dir, :quiet_pdf_failures, :quiet_upload_failures, :run_quiet, :submission @@ -31,11 +31,10 @@ class ArchiveSubmissionToPdf < SimpleFormsApi::S3Service::Utils 21-4138 21-4142 21P-0847 26-4555 40-0247 40-10007 ].freeze - def initialize(form_id: nil, submission_id: nil, submission: nil, **options) + def initialize(submission_id: nil, submission: nil, **options) defaults = default_options.merge(options) @failures = [] - @form_id = form_id @submission = submission || FormSubmission.find(submission_id) assign_instance_variables(defaults) @@ -46,19 +45,22 @@ def run process_submission_files output_directory_path rescue => e - handle_error("Failed submission: #{submission.id}", e, submission_id: submission.id) + handle_error("Failed submission: #{submission.id}", e, { submission_id: submission.id }) end private def default_options { + file_path: nil, # file path for the PDF file to be archived include_json_archive: true, # include the form data as a JSON object include_text_archive: true, # include the form data as a text file - parent_dir: 'wipn8923-test', + metadata: {}, + parent_dir: 'vff-simple-forms', quiet_pdf_failures: true, # skip PDF generation silently quiet_upload_failures: true, # skip problematic uploads silently - run_quiet: true # silence but record errors, logged at the end + run_quiet: true, # silence but record errors, logged at the end + uploads_path: ['uploadedFile'] # hierarchy where the attachments can be found } end @@ -72,15 +74,24 @@ def process_submission_files def write_pdf encoded_pdf = generate_pdf_content - save_file_to_s3("#{output_directory_path}/form.pdf", Base64.decode64(encoded_pdf)) + pdf = save_file_to_s3( + "#{output_directory_path}/form_#{submission.form_data['form_number']}.pdf", + Base64.decode64(encoded_pdf) + ) + sign_s3_file_url(pdf) rescue => e quiet_pdf_failures ? write_pdf_error(e) : raise(e) end - # TODO: update this method to support configurable pdf generation logic def generate_pdf_content - service = EVSS::DisabilityCompensationForm::NonBreakeredService.new(submission.auth_headers) - service.get_form(form_json.to_json).body['pdf'] + raise 'Missing PDF file to upload' unless file_path + + Faraday::UploadIO.new(file_path, Mime[:pdf].to_s, File.basename(file_path)) + end + + def sign_s3_file_url(pdf) + signed_url = pdf.presigned_url(:get, expires_in: 1.year.to_i) + submission.form_submission_attempts&.last&.update(signed_url:) end def write_pdf_error(error) @@ -135,41 +146,22 @@ def write_failure_report end def save_file_to_s3(path, content) - s3_resource.bucket(target_bucket).object(path).put(body: content) + s3_resource.bucket(target_bucket).object(path).tap do |obj| + obj.put(body: content) + end end def form_json - @form_json ||= JSON.parse(submission.form_json)[form_id] + @form_json ||= JSON.parse(submission.form_data) end def form_text_archive - submission.form.tap do |form| - form[form_id]['claimDate'] ||= submission.created_at.iso8601 - end - end - - def metadata - return {} unless submission.auth_headers.present? && submission.form[form_id].present? - - extract_metadata_from_submission - end - - # TODO: update this method to support configurable metadata - def extract_metadata_from_submission - address = submission.form.dig(form_id, 'veteran', 'currentMailingAddress') - zip = [address['zipFirstFive'], address['zipLastFour']].join('-') if address.present? - pii = JSON.parse(submission.auth_headers['va_eauth_authorization'])['authorizationResponse'] - pii.merge({ - fileNumber: pii['va_eauth_pnid'], - zipCode: zip || '00000', - claimDate: submission.created_at.iso8601, - formsIncluded: map_form_inclusion - }) + submission.form_data['claimDate'] ||= submission.created_at.iso8601 end # TODO: update this method to check against configured form list def map_form_inclusion - VALID_VFF_FORMS.select { |type| submission.form[type].present? } + VALID_VFF_FORMS.select { |type| submission.form_number == type } end def output_directory_path @@ -177,7 +169,7 @@ def output_directory_path end def user_uploads - @user_uploads ||= submission.form['form_uploads'] + @user_uploads ||= submission.fetch(*uploads_path, nil) end def user_upload_failures diff --git a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler.rb b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler.rb index 765f970686a..7df98891fd1 100644 --- a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler.rb +++ b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler.rb @@ -26,7 +26,9 @@ def run def default_options { bundle_by_user: true, - parent_dir: 'wipn8923-test', + file_path: nil, # file path for the PDF file to be archived + metadata: nil, + parent_dir: 'vff-simple-forms', quiet_pdf_failures: false, # granular control over how pdf processing raises errors quiet_upload_failures: false, # granular control over how upload processing raises errors run_quiet: true # silence but record errors until the end @@ -72,10 +74,12 @@ def process_user_submissions(uuid, submission_ids) def process_submission(submission_id) ArchiveSubmissionToPdf.new( - submission_id:, + file_path:, + metadata:, parent_dir:, quiet_pdf_failures:, - quiet_upload_failures: + quiet_upload_failures:, + submission_id: ).run rescue => e handle_error("Submission archiver failure: #{submission_id}", e, submission_id:) diff --git a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler_job.rb b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler_job.rb index 1ea5d043316..60433f9b5ef 100644 --- a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler_job.rb +++ b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/submission_archive_handler_job.rb @@ -22,7 +22,9 @@ def perform(submission_ids:, **options) def default_options { bundle_by_user: true, - parent_dir: 'wipn8923-test', + file_path: nil, # file path for the PDF file to be archived + metadata: nil, + parent_dir: 'vff-simple-forms', quiet_pdf_failures: false, quiet_upload_failures: false, run_quiet: true, diff --git a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/user_submission_archive_handler.rb b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/user_submission_archive_handler.rb index 1b463a5e65a..99256ee8f5f 100644 --- a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/user_submission_archive_handler.rb +++ b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/user_submission_archive_handler.rb @@ -5,7 +5,7 @@ module S3Service class UserSubmissionArchiveHandler < SimpleFormsApi::S3Service::Utils attr_reader :uuid, :user_dir, :submission_ids - def initialize(uuid:, submission_ids:, parent_dir: 'wipn8923-test') + def initialize(uuid:, submission_ids:, parent_dir: 'vff-simple-forms') @submission_ids = submission_ids @uuid = uuid @user_dir = build_user_directory(parent_dir) diff --git a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/utils.rb b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/utils.rb index fb08ca68e8c..471e4b02add 100644 --- a/modules/simple_forms_api/app/services/simple_forms_api/s3_service/utils.rb +++ b/modules/simple_forms_api/app/services/simple_forms_api/s3_service/utils.rb @@ -22,7 +22,7 @@ def log_error(message, error, **details) def handle_error(message, error, context) raise error unless run_quiet - log_error(message, error, context) + log_error(message, error, **context) failures << { message:, error:, **context } end diff --git a/modules/simple_forms_api/spec/services/s3_service/archive_submission_to_pdf_spec.rb b/modules/simple_forms_api/spec/services/s3_service/archive_submission_to_pdf_spec.rb new file mode 100644 index 00000000000..d01ae14e114 --- /dev/null +++ b/modules/simple_forms_api/spec/services/s3_service/archive_submission_to_pdf_spec.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +require 'rails_helper' +require SimpleFormsApi::Engine.root.join('spec', 'spec_helper.rb') + +RSpec.describe SimpleFormsApi::S3Service::ArchiveSubmissionToPdf, type: :model do + let(:submission_id) { 1 } + let(:form_id) { '21-10210' } + let(:form_data) { File.read('modules/simple_forms_api/spec/fixtures/form_json/vba_21_10210.json') } + let(:submission) { create(:form_submission, :pending, form_type: form_id, form_data:) } + let(:options) do + { + include_json_archive: true, + include_text_archive: true, + parent_dir: 'test-dir', + quiet_pdf_failures: true, + quiet_upload_failures: true, + run_quiet: true + } + end + let(:archive_submission) { described_class.new(submission_id:, **options) } + + before do + allow(FormSubmission).to receive(:find).and_return(submission) + end + + describe '#initialize' do + it 'sets default values for instance variables' do + expect(archive_submission.submission).to eq(submission) + expect(archive_submission.parent_dir).to eq('test-dir') + expect(archive_submission.include_json_archive).to be(true) + expect(archive_submission.include_text_archive).to be(true) + expect(archive_submission.quiet_pdf_failures).to be(true) + expect(archive_submission.quiet_upload_failures).to be(true) + end + end + + describe '#run' do + before do + allow(archive_submission).to receive(:process_submission_files) + allow(archive_submission).to receive(:output_directory_path).and_return('/some/path') + allow(archive_submission).to receive(:log_info) + end + + it 'logs the processing of the submission and calls process_submission_files' do + expect(archive_submission).to receive(:log_info).with("Processing submission ID: #{submission.id}") + expect(archive_submission).to receive(:process_submission_files) + archive_submission.run + end + + context 'when an error occurs' do + before do + allow(archive_submission).to receive(:process_submission_files).and_raise(StandardError, 'Processing error') + end + + xit 'handles errors and logs them' do + expect(archive_submission).to( + receive(:handle_error).with( + "Failed submission: #{submission.id}", + instance_of(StandardError), submission_id: submission.id + ) + ) + expect { archive_submission.run }.not_to raise_error + end + end + end + + describe '#write_pdf' do + before do + allow(archive_submission).to receive(:generate_pdf_content).and_return(Base64.encode64('pdf content')) + allow(archive_submission).to receive(:save_file_to_s3) + end + + xit 'writes the PDF to S3' do + expect(archive_submission).to receive(:save_file_to_s3).with(/form.pdf/, 'pdf content') + archive_submission.run + end + + context 'when an error occurs' do + before do + allow(archive_submission).to receive(:generate_pdf_content).and_raise(StandardError, 'PDF generation error') + end + + it 'handles pdf generation errors based on quiet_pdf_failures' do + expect(archive_submission).to receive(:write_pdf_error).with(instance_of(StandardError)) + expect { archive_submission.run }.not_to raise_error + end + end + end + + describe '#write_as_json_archive' do + before do + allow(archive_submission).to receive(:save_file_to_s3) + allow(archive_submission).to receive(:form_json).and_return({ key: 'value' }) + end + + it 'writes the JSON archive to S3' do + expect(archive_submission).to receive(:save_file_to_s3).with(/form_text_archive.json/, + JSON.pretty_generate({ key: 'value' })) + archive_submission.run + end + end + + describe '#write_as_text_archive' do + before do + allow(archive_submission).to receive(:save_file_to_s3) + allow(archive_submission).to receive(:form_text_archive).and_return({ key: 'value' }) + end + + it 'writes the text archive to S3' do + expect(archive_submission).to receive(:save_file_to_s3).with(/form_text_archive.txt/, { key: 'value' }.to_json) + archive_submission.run + end + end + + describe '#write_metadata' do + before do + allow(archive_submission).to receive(:save_file_to_s3) + allow(archive_submission).to receive(:metadata).and_return({ key: 'value' }) + end + + xit 'writes metadata to S3' do + expect(archive_submission).to receive(:save_file_to_s3).with(/metadata.json/, { key: 'value' }.to_json) + archive_submission.run + end + end + + describe '#handle_error' do + before do + allow(archive_submission).to receive(:process_submission_files).and_return(error) + end + + let(:error) { StandardError.new('some error') } + + xit 'logs the error and re-raises it' do + expect(archive_submission).to receive(:log_error).with( + "Failed submission: #{submission.id}", error, submission_id: submission.id + ) + expect { archive_submission.run }.to raise_error(error) + end + end +end