[CST] - Create cron job to send emails for failed uploads (updated pr)

.github/CODEOWNERS

@@ -631,7 +631,6 @@ app/sidekiq/evss/delete_old_claims.rb @department-of-veterans-affairs/mbs-core-t
 app/sidekiq/evss/dependents_application_job.rb @department-of-veterans-affairs/benefits-dependents-management @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/evss/disability_compensation_form @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/evss/document_upload.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/backend-review-group
-app/sidekiq/evss/failure_notification.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/benefits-management-tools-be
 app/sidekiq/evss/disability_compensation_form/form526_document_upload_failure_email.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/evss/retrieve_claims_from_remote_job.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/evss/request_decision.rb @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -655,10 +654,10 @@ app/sidekiq/income_limits @department-of-veterans-affairs/vfs-public-websites-fr
 app/sidekiq/in_progress_form_cleaner.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/kms_key_rotation @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/lighthouse @department-of-veterans-affairs/backend-review-group
+app/sidekiq/lighthouse/benefits_documents/failure_notification_email_job.rb @department-of-veterans-affairs/benefits-management-tools-be, @department-of-veterans-affairs/backend-review-group and @department-of-veterans-affairs/va-api-engineers
 app/sidekiq/lighthouse/submit_career_counseling_job.rb @department-of-veterans-affairs/my-education-benefits @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/lighthouse/create_intent_to_file_job.rb @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/backend-review-group
 app/sidekiq/lighthouse/income_and_assets_intake_job.rb @department-of-veterans-affairs/pension-and-burials @department-of-veterans-affairs/backend-review-group
-app/sidekiq/lighthouse/failure_notification.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/benefits-management-tools-be
 app/sidekiq/load_average_days_for_claim_completion_job.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/mhv/account_creator_job.rb @department-of-veterans-affairs/octo-identity
@@ -1219,6 +1218,7 @@ spec/factories/gi_bill_status_response.rb @department-of-veterans-affairs/govcio
 spec/factories/gids_response.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/factories/hca_attachment.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/factories/health_care_application.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/factories/lighthouse/benefits_documents/evidence_submission.rb @department-of-veterans-affairs/benefits-management-tools-be, @department-of-veterans-affairs/backend-review-group and @department-of-veterans-affairs/va-api-engineers
 spec/factories/iam_introspection_responses.rb @department-of-veterans-affairs/octo-identity
 spec/factories/iam_user_identities.rb @department-of-veterans-affairs/octo-identity
 spec/factories/iam_users.rb @department-of-veterans-affairs/octo-identity
@@ -1386,6 +1386,7 @@ spec/sidekiq/income_limits @department-of-veterans-affairs/vfs-public-websites-f
 spec/sidekiq/in_progress_form_cleaner_spec.rb @department-of-veterans-affairs/vfs-authenticated-experience-backend @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/kms_key_rotation @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/lighthouse @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/lighthouse/benefits_documents/document_upload_spec.rb @department-of-veterans-affairs/benefits-management-tools-be, @department-of-veterans-affairs/backend-review-group and @department-of-veterans-affairs/va-api-engineers 
 spec/sidekiq/lighthouse/failure_notification_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/benefits-management-tools-be
 spec/sidekiq/load_average_days_for_claim_completion_job_spec.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group

app/models/evidence_submission.rb

@@ -1,7 +1,18 @@
 # frozen_string_literal: true
 
+require 'lighthouse/benefits_documents/constants'
+
 class EvidenceSubmission < ApplicationRecord
   belongs_to :user_account
   has_kms_key
   has_encrypted :template_metadata, key: :kms_key, **lockbox_options
+
+  scope :va_notify_email_queued, lambda {
+    where(upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED])
+      .where.not(va_notify_date: nil)
+      .where.not(va_notify_id: nil)
+  }
+  scope :va_notify_email_not_queued, lambda {
+    where(upload_status: BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED], va_notify_id: nil, va_notify_date: nil)
+  }
 end

app/sidekiq/evss/document_upload.rb

@@ -3,21 +3,15 @@
 require 'ddtrace'
 require 'timeout'
 require 'logging/third_party_transaction'
-require 'evss/failure_notification'
+require 'lighthouse/benefits_documents/constants'
 
 class EVSS::DocumentUpload
   include Sidekiq::Job
-  extend SentryLogging
   extend Logging::ThirdPartyTransaction::MethodWrapper
 
   FILENAME_EXTENSION_MATCHER = /\.\w*$/
   OBFUSCATED_CHARACTER_MATCHER = /[a-zA-Z\d]/
 
-  DD_ZSF_TAGS = ['service:claim-status', 'function: evidence upload to EVSS'].freeze
-
-  NOTIFY_SETTINGS = Settings.vanotify.services.benefits_management_tools
-  MAILER_TEMPLATE_ID = NOTIFY_SETTINGS.template_id.evidence_submission_failure_email
-
   attr_accessor :auth_headers, :user_uuid, :document_hash
 
   wrap_with_logging(
@@ -40,28 +34,32 @@ class EVSS::DocumentUpload
   end
 
   sidekiq_retries_exhausted do |msg, _ex|
-    # There should be 3 values in msg['args']:
-    # 1) Auth headers needed to authenticate with EVSS
-    # 2) The uuid of the record in the UserAccount table
-    # 3) Document metadata
-
-    next unless Flipper.enabled?('cst_send_evidence_failure_emails')
-
-    icn = UserAccount.find(msg['args'][1]).icn
-    first_name = msg['args'].first['va_eauth_firstName'].titleize
+    job_id = msg['jid']
+    job_class = 'EVSS::DocumentUpload'
+    first_name = msg['args'][0]['va_eauth_firstName'].titleize unless msg['args'][0]['va_eauth_firstName'].nil?
+    claim_id = msg['args'][2]['evss_claim_id']
+    tracked_item_id = msg['args'][2]['tracked_item_id']
     filename = obscured_filename(msg['args'][2]['file_name'])
     date_submitted = format_issue_instant_for_mailers(msg['created_at'])
     date_failed = format_issue_instant_for_mailers(msg['failed_at'])
-
-    EVSS::FailureNotification.perform_async(icn, first_name, filename, date_submitted, date_failed)
-
-    ::Rails.logger.info('EVSS::DocumentUpload exhaustion handler email queued')
-    StatsD.increment('silent_failure_avoided_no_confirmation', tags: DD_ZSF_TAGS)
+    upload_status = BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED]
+    uuid = msg['args'][2]['uuid']
+    user_account = UserAccount.find_or_create_by(id: uuid)
+    personalisation = { first_name:, filename:, date_submitted:, date_failed: }
+
+    EvidenceSubmission.create(
+      job_id:,
+      job_class:,
+      claim_id:,
+      tracked_item_id:,
+      upload_status:,
+      user_account:,
+      template_metadata_ciphertext: { personalisation: }.to_json
+    )
   rescue => e
-    ::Rails.logger.error('EVSS::DocumentUpload exhaustion handler email error',
-                         { message: e.message })
-    StatsD.increment('silent_failure', tags: DD_ZSF_TAGS)
-    log_exception_to_sentry(e)
+    error_message = "#{job_class} failed to create EvidenceSubmission"
+    ::Rails.logger.info(error_message, { messsage: e.message })
+    StatsD.increment('silent_failure', tags: ['service:claim-status', "function: #{error_message}"])
   end
 
   def perform(auth_headers, user_uuid, document_hash)

app/sidekiq/lighthouse/document_upload.rb → app/sidekiq/lighthouse/benefits_documents/document_upload.rb

@@ -4,22 +4,15 @@
 require 'timeout'
 require 'lighthouse/benefits_documents/worker_service'
 require 'lighthouse/benefits_documents/constants'
-require 'lighthouse/failure_notification'
 
-class Lighthouse::DocumentUpload
+class Lighthouse::BenefitsDocuments::DocumentUpload
   include Sidekiq::Job
-  extend SentryLogging
 
   attr_accessor :user_icn, :document_hash
 
   FILENAME_EXTENSION_MATCHER = /\.\w*$/
   OBFUSCATED_CHARACTER_MATCHER = /[a-zA-Z\d]/
 
-  DD_ZSF_TAGS = ['service:claim-status', 'function: evidence upload to Lighthouse'].freeze
-
-  NOTIFY_SETTINGS = Settings.vanotify.services.benefits_management_tools
-  MAILER_TEMPLATE_ID = NOTIFY_SETTINGS.template_id.evidence_submission_failure_email
-
   # retry for  2d 1h 47m 12s
   # https://github.com/sidekiq/sidekiq/wiki/Error-Handling
   sidekiq_options retry: 16, queue: 'low'
@@ -29,27 +22,32 @@ class Lighthouse::DocumentUpload
   end
 
   sidekiq_retries_exhausted do |msg, _ex|
-    # There should be 2 values in msg['args']:
-    # 1) The ICN of the user
-    # 2) Document metadata
-
-    next unless Flipper.enabled?('cst_send_evidence_failure_emails')
-
-    icn = msg['args'].first
-    first_name = msg['args'][1]['first_name'].titleize
+    job_id = msg['jid']
+    job_class = 'Lighthouse::BenefitsDocuments::DocumentUpload'
+    first_name = msg['args'][1]['first_name'].titleize unless msg['args'][1]['first_name'].nil?
+    claim_id = msg['args'][1]['claim_id']
+    tracked_item_id = msg['args'][1]['tracked_item_id']
     filename = obscured_filename(msg['args'][1]['file_name'])
     date_submitted = format_issue_instant_for_mailers(msg['created_at'])
     date_failed = format_issue_instant_for_mailers(msg['failed_at'])
-
-    Lighthouse::FailureNotification.perform_async(icn, first_name, filename, date_submitted, date_failed)
-
-    ::Rails.logger.info('Lighthouse::DocumentUpload exhaustion handler email queued')
-    StatsD.increment('silent_failure_avoided_no_confirmation', tags: DD_ZSF_TAGS)
+    upload_status = BenefitsDocuments::Constants::UPLOAD_STATUS[:FAILED]
+    uuid = msg['args'][1]['uuid']
+    user_account = UserAccount.find_or_create_by(id: uuid)
+    personalisation = { first_name:, filename:, date_submitted:, date_failed: }
+
+    EvidenceSubmission.create(
+      job_id:,
+      job_class:,
+      claim_id:,
+      tracked_item_id:,
+      upload_status:,
+      user_account:,
+      template_metadata_ciphertext: { personalisation: }.to_json
+    )
   rescue => e
-    ::Rails.logger.error('Lighthouse::DocumentUpload exhaustion handler email error',
-                         { message: e.message })
-    StatsD.increment('silent_failure', tags: DD_ZSF_TAGS)
-    log_exception_to_sentry(e)
+    error_message = "#{job_class} failed to create EvidenceSubmission"
+    ::Rails.logger.info(error_message, { messsage: e.message })
+    StatsD.increment('silent_failure', tags: ['service:claim-status', "function: #{error_message}"])
   end
 
   def self.obscured_filename(original_filename)
@@ -75,10 +73,6 @@ def self.format_issue_instant_for_mailers(issue_instant)
     timestamp.strftime('%B %-d, %Y %-l:%M %P %Z').sub(/([ap])m/, '\1.m.')
   end
 
-  def self.notify_client
-    VaNotify::Service.new(NOTIFY_SETTINGS.api_key)
-  end
-
   def perform(user_icn, document_hash, user_account_uuid, claim_id, tracked_item_id)
     @user_icn = user_icn
     @document_hash = document_hash

app/sidekiq/lighthouse/benefits_documents/failure_notification_email_job.rb

@@ -0,0 +1,67 @@
+require 'sidekiq'
+require 'lighthouse/benefits_documents/constants'
+
+module BenefitsDocuments
+  class FailureNotificationEmailJob
+    include Sidekiq::Job
+    include SentryLogging
+
+    sidekiq_options retry: false, unique_for: 30.minutes
+    NOTIFY_SETTINGS = Settings.vanotify.services.benefits_management_tools
+    MAILER_TEMPLATE_ID = NOTIFY_SETTINGS.template_id.evidence_submission_failure_email
+    # TODO: need to add statsd logic
+    # STATSD_KEY_PREFIX = ''
+
+    def perform
+      return unless should_perform?
+
+      send_failed_evidence_submissions
+
+      nil
+    end
+
+    def should_perform?
+      failed_uploads.present?
+    end
+
+    # Fetches FAILED evidence submission records for BenefitsDocuments that dont have a va_notify_date
+    def failed_uploads
+      @failed_uploads ||= EvidenceSubmission.va_notify_email_not_queued
+    end
+
+    def notify_client
+      VaNotify::Service.new(NOTIFY_SETTINGS.api_key)
+    end
+
+    def send_failed_evidence_submissions
+      failed_uploads.each do |upload|
+        personalisation = JSON.parse(upload.template_metadata_ciphertext)['personalisation']
+        response = notify_client.send_email(
+          recipient_identifier: { id_value: upload.user_account.icn, id_type: 'ICN' },
+          template_id: MAILER_TEMPLATE_ID,
+          personalisation:
+        )
+        record_email_send_success(upload, response)
+      rescue => e
+        record_email_send_failure(upload, e)
+      end
+
+      nil
+    end
+
+    def record_email_send_success(upload, response)
+      upload.update(va_notify_id: response.id, va_notify_date: DateTime.now)
+      message = "#{upload.job_class} va notify failure email queued"
+      ::Rails.logger.info(message)
+      StatsD.increment('silent_failure_avoided_no_confirmation',
+                       tags: ['service:claim-status', "function: #{message}"])
+    end
+
+    def record_email_send_failure(upload, error)
+      error_message = "#{upload.job_class} va notify failure email errored"
+      ::Rails.logger.error(error_message, { message: error.message })
+      StatsD.increment('silent_failure', tags: ['service:claim-status', "function: #{error_message}"])
+      log_exception_to_sentry(error)
+    end
+  end
+end

db/migrate/20240925160219_create_evidence_submissions.rb → db/migrate/20241217210622_create_evidence_submissions.rb

@@ -1,4 +1,4 @@
-class CreateEvidenceSubmissions < ActiveRecord::Migration[7.1]
+class CreateEvidenceSubmissions < ActiveRecord::Migration[7.2]
   def change
     create_table :evidence_submissions do |t|
       t.string :job_id
@@ -12,6 +12,7 @@ def change
       t.string :va_notify_id
       t.string :va_notify_status
       t.date :delete_date
+      t.datetime :va_notify_date
       t.string :tracked_item_id
 
       t.timestamps