From 885726da4302e948d78eebde90a3b89ee1254ea8 Mon Sep 17 00:00:00 2001 From: Vanson Samuel <37637+binq@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:01:18 -0500 Subject: [PATCH 1/2] BDN import --- modules/vye/app/models/vye/address_change.rb | 24 ++++--- modules/vye/app/models/vye/user_info.rb | 31 ++++++--- modules/vye/app/models/vye/user_profile.rb | 10 ++- .../config/bdn_line_extraction_config.yaml | 61 +++++++++++++++++ .../lib/vye/batch_transfer/ingress_files.rb | 14 +++- .../ingress_files/bdn_line_extraction.rb | 68 +++++++++++++++++++ modules/vye/lib/vye/engine.rb | 1 + .../vye/spec/factories/vye/address_changes.rb | 1 + modules/vye/spec/fixtures/bdn_sample/WAVE.txt | 1 + .../vye/batch_transfer/ingress_files_spec.rb | 9 ++- 10 files changed, 198 insertions(+), 22 deletions(-) create mode 100644 modules/vye/config/bdn_line_extraction_config.yaml create mode 100644 modules/vye/lib/vye/batch_transfer/ingress_files/bdn_line_extraction.rb create mode 100755 modules/vye/spec/fixtures/bdn_sample/WAVE.txt diff --git a/modules/vye/app/models/vye/address_change.rb b/modules/vye/app/models/vye/address_change.rb index 1f55b44bcb3..5394939ed19 100644 --- a/modules/vye/app/models/vye/address_change.rb +++ b/modules/vye/app/models/vye/address_change.rb @@ -4,18 +4,24 @@ module Vye class Vye::AddressChange < ApplicationRecord belongs_to :user_info - ENCRYPTED_ATTRIBUTES = %i[ - veteran_name address1 address2 address3 address4 city state zip_code - ].freeze - has_kms_key - has_encrypted(*ENCRYPTED_ATTRIBUTES, key: :kms_key, **lockbox_options) - REQUIRED_ATTRIBUTES = %i[ - veteran_name address1 city state - ].freeze + has_encrypted( + :veteran_name, + :address1, :address2, :address3, :address4, :address5, + :city, :state, :zip_code, + key: :kms_key, **lockbox_options + ) + + validates( + :veteran_name, :address1, :city, :state, + presence: true, if: -> { origin == 'frontend' } + ) - validates(*REQUIRED_ATTRIBUTES, presence: true) + validates( + :veteran_name, :address1, + presence: true, if: -> { origin == 'backend' } + ) enum origin: { frontend: 'f', backend: 'b' } diff --git a/modules/vye/app/models/vye/user_info.rb b/modules/vye/app/models/vye/user_info.rb index 59c30346162..3d94440ca00 100644 --- a/modules/vye/app/models/vye/user_info.rb +++ b/modules/vye/app/models/vye/user_info.rb @@ -4,11 +4,16 @@ module Vye class Vye::UserInfo < ApplicationRecord INCLUDES = %i[address_changes awards pending_documents verifications].freeze - self.ignored_columns += %i[ - address_line2_ciphertext address_line3_ciphertext address_line4_ciphertext - address_line5_ciphertext address_line6_ciphertext - full_name_ciphertext icn ssn_digest suffix zip_ciphertext - ] + self.ignored_columns += + [ + :ssn_digest, :icn, # moved to UserProfile + + :suffix, # not needed + + :address_line2_ciphertext, :address_line3_ciphertext, # moved to AddressChange + :address_line4_ciphertext, :address_line5_ciphertext, # moved to AddressChange + :address_line6_ciphertext, :full_name_ciphertext, :zip_ciphertext # moved to AddressChange + ] belongs_to :user_profile @@ -28,12 +33,12 @@ class Vye::UserInfo < ApplicationRecord delegate :icn, to: :user_profile, allow_nil: true delegate :pending_documents, to: :user_profile, allow_nil: true - %i[dob file_number ssn stub_nm].freeze.tap do |attributes| - has_kms_key - has_encrypted(*attributes, key: :kms_key, **lockbox_options) + has_kms_key + has_encrypted(:file_number, :ssn, :dob, :stub_nm, key: :kms_key, **lockbox_options) - validates(*attributes, presence: true) - end + validates :dob, :stub_nm, presence: true + + validate :ssn_or_file_number_present validates( :cert_issue_date, :date_last_certified, :del_date, :fac_code, :indicator, @@ -45,6 +50,12 @@ def verification_required verifications.empty? end + def ssn_or_file_number_present + return true if ssn.present? || file_number.present? + + errors.add(:base, 'Either SSN or file number must be present.') + end + scope :with_assos, -> { includes(:address_changes, :awards, user_profile: :pending_documents) } end end diff --git a/modules/vye/app/models/vye/user_profile.rb b/modules/vye/app/models/vye/user_profile.rb index 810cd822721..8f930ccc93d 100644 --- a/modules/vye/app/models/vye/user_profile.rb +++ b/modules/vye/app/models/vye/user_profile.rb @@ -10,7 +10,7 @@ class Vye::UserProfile < ApplicationRecord digest_attribute :ssn digest_attribute :file_number - validates :ssn_digest, :file_number_digest, presence: true + validate :ssn_or_file_number_present scope :with_assos, -> { includes(:pending_documents, :user_infos) } @@ -25,4 +25,12 @@ def self.find_and_update_icn(user:) result&.update!(icn: user.icn) end end + + private + + def ssn_or_file_number_present + return true if ssn_digest.present? || file_number_digest.present? + + errors.add(:base, 'Either SSN or file number must be present.') + end end diff --git a/modules/vye/config/bdn_line_extraction_config.yaml b/modules/vye/config/bdn_line_extraction_config.yaml new file mode 100644 index 00000000000..c174983ce7f --- /dev/null +++ b/modules/vye/config/bdn_line_extraction_config.yaml @@ -0,0 +1,61 @@ +--- +:main_line: + :ssn: 9 + :file_number: 9 + :suffix: 2 + :dob: 8 + :mr_status: 1 + :rem_ent: 7 + :cert_issue_date: 8 + :del_date: 8 + :date_last_certified: 8 + :veteran_name: 20 + :address1: 20 + :address2: 20 + :address3: 20 + :address4: 20 + :address5: 20 + :zip_code: 9 + :stub_nm: 7 + :rpo_code: 3 + :fac_code: 8 + :payment_amt: 7 +:award_line: + :award_begin_date: 8 + :award_end_date: 8 + :training_time: 1 + :payment_date: 8 + :monthly_rate: 7 + :begin_rsn: 2 + :end_rsn: 2 + :type_training: 1 + :number_hours: 2 + :type_hours: 1 + :cur_award_ind: 1 +:mappings: + :profile: + - :ssn + - :file_number + :info: + - :ssn + - :file_number + - :dob + - :mr_status + - :rem_ent + - :cert_issue_date + - :del_date + - :date_last_certified + - :stub_nm + - :rpo_code + - :fac_code + - :payment_amt + - :indicator + :address: + - :veteran_name + - :address1 + - :address2 + - :address3 + - :address4 + - :address5 + - :zip_code +:indicator: true diff --git a/modules/vye/lib/vye/batch_transfer/ingress_files.rb b/modules/vye/lib/vye/batch_transfer/ingress_files.rb index aab2dc7a7c0..2c0ffa6fc0b 100644 --- a/modules/vye/lib/vye/batch_transfer/ingress_files.rb +++ b/modules/vye/lib/vye/batch_transfer/ingress_files.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module VYE +module Vye module BatchTransfer module IngressFiles module_function @@ -10,6 +10,18 @@ module IngressFiles def bdn_feed_filename = BDN_FEED_FILENAME def tims_feed_filename = TIMS_FEED_FILENAME + + def bdn_import(data) + data.each_line do |line| + parsed = BdnLineExtraction.new(line: line.chomp, result: {}, award_lines: [], awards: []) + + profile = Vye::UserProfile.build(parsed.attributes[:profile]) + info = profile.user_infos.build(parsed.attributes[:info]) + info.address_changes.build({ origin: 'backend' }.merge(parsed.attributes[:address])) + + profile.save! + end + end end end end diff --git a/modules/vye/lib/vye/batch_transfer/ingress_files/bdn_line_extraction.rb b/modules/vye/lib/vye/batch_transfer/ingress_files/bdn_line_extraction.rb new file mode 100644 index 00000000000..c8469d538ac --- /dev/null +++ b/modules/vye/lib/vye/batch_transfer/ingress_files/bdn_line_extraction.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Vye + module BatchTransfer + module IngressFiles + BdnLineExtraction = Struct.new(:line, :result, :award_lines, :awards) do + def initialize(line:, result:, award_lines:, awards:) + super + + extract_main + extract_award_lines + extract_indicator + extract_awards + end + + def config + @config ||= YAML.load_file Vye::Engine.root / 'config/bdn_line_extraction_config.yaml' + end + + def extract_main + config[:main_line] + .each do |field, length| + extracted = line.slice!(0...length).strip + result.update(field => extracted) + end + end + + def extract_award_lines + (0...4) + .each do |_i| + dead = line[0...8].strip == '' + extracted = line.slice!(0...41) + award_lines << extracted unless dead + end + end + + def extract_awards + # iterate over the fields for each award line and extract the data + # think list comprehensions in python, haskell, or erlang + award_lines + .each_with_index.to_a + .product(config[:award_line].each_pair.to_a) + .each do |(award_line, i), (field, length)| + extracted = award_line.slice!(0...length).strip + awards[i] ||= {} + awards[i].update(field => extracted) + end + end + + def extract_indicator + return unless config[:indicator] + + result.update(indicator: line.slice!(0...1).strip) + end + + def attributes + raise 'incomplete extraction' unless line.blank? && award_lines.all?(&:blank?) + + profile = result.slice(*config[:mappings][:profile]) + info = result.slice(*config[:mappings][:info]) + address = result.slice(*config[:mappings][:address]) + + { profile:, info:, address:, awards: } + end + end + end + end +end diff --git a/modules/vye/lib/vye/engine.rb b/modules/vye/lib/vye/engine.rb index dd0bc342d93..2310151db33 100644 --- a/modules/vye/lib/vye/engine.rb +++ b/modules/vye/lib/vye/engine.rb @@ -6,6 +6,7 @@ module Vye class Engine < Rails::Engine isolate_namespace Vye config.generators.api_only = true + config.autoload_paths << (root / 'lib') initializer 'model_core.factories', after: 'factory_bot.set_factory_paths' do FactoryBot.definition_file_paths << File.expand_path('../../spec/factories', __dir__) if defined?(FactoryBot) diff --git a/modules/vye/spec/factories/vye/address_changes.rb b/modules/vye/spec/factories/vye/address_changes.rb index 4614d1e8958..40bc66818a8 100644 --- a/modules/vye/spec/factories/vye/address_changes.rb +++ b/modules/vye/spec/factories/vye/address_changes.rb @@ -7,5 +7,6 @@ city { Faker::Address.city } state { Faker::Address.state_abbr } zip_code { Faker::Address.zip_code } + origin { Vye::AddressChange.origins['frontend'] } end end diff --git a/modules/vye/spec/fixtures/bdn_sample/WAVE.txt b/modules/vye/spec/fixtures/bdn_sample/WAVE.txt new file mode 100755 index 00000000000..a6e7d15c57e --- /dev/null +++ b/modules/vye/spec/fixtures/bdn_sample/WAVE.txt @@ -0,0 +1 @@ +123456789 19800101E3600000198603281996020519860328JOHN APPLESEED 1 Mockingbird Ln APT 1 Houston TX 77401 JAPPLES316119071110011550 00000000198603281198603280003500 66 00 C A diff --git a/modules/vye/spec/lib/vye/batch_transfer/ingress_files_spec.rb b/modules/vye/spec/lib/vye/batch_transfer/ingress_files_spec.rb index bd396a66e54..735cea37c76 100644 --- a/modules/vye/spec/lib/vye/batch_transfer/ingress_files_spec.rb +++ b/modules/vye/spec/lib/vye/batch_transfer/ingress_files_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' require 'vye/batch_transfer/ingress_files' -RSpec.describe VYE::BatchTransfer::IngressFiles do +RSpec.describe Vye::BatchTransfer::IngressFiles do describe '#bdn_feed_filename' do it 'returns a string' do expect(described_class.bdn_feed_filename).to be_a(String) @@ -15,4 +15,11 @@ expect(described_class.tims_feed_filename).to be_a(String) end end + + it 'imports lines from BDN extract' do + data = Vye::Engine.root / 'spec/fixtures/bdn_sample/WAVE.txt' + expect do + described_class.bdn_import(data) + end.not_to raise_error + end end From efbd47e7d151da17a4107e0188bad01dd8de6e7e Mon Sep 17 00:00:00 2001 From: Vanson Samuel <37637+binq@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:50:21 -0500 Subject: [PATCH 2/2] ingress awards also --- modules/vye/app/models/vye/award.rb | 13 +++++++------ modules/vye/lib/vye/batch_transfer/ingress_files.rb | 4 +++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/vye/app/models/vye/award.rb b/modules/vye/app/models/vye/award.rb index 09682f9cc11..f37db8962a3 100644 --- a/modules/vye/app/models/vye/award.rb +++ b/modules/vye/app/models/vye/award.rb @@ -6,11 +6,12 @@ class Vye::Award < ApplicationRecord enum cur_award_ind: { current: 'C', future: 'F', past: 'P' } - REQUIRED_ATTRIBUTES = %i[ - award_begin_date award_end_date begin_rsn cur_award_ind end_rsn - monthly_rate number_hours payment_date training_time type_hours type_training - ].freeze - - validates(*REQUIRED_ATTRIBUTES, presence: true) + validates( + *%i[ + award_end_date cur_award_ind end_rsn + monthly_rate number_hours payment_date training_time + ].freeze, + presence: true + ) end end diff --git a/modules/vye/lib/vye/batch_transfer/ingress_files.rb b/modules/vye/lib/vye/batch_transfer/ingress_files.rb index 2c0ffa6fc0b..a744822b31f 100644 --- a/modules/vye/lib/vye/batch_transfer/ingress_files.rb +++ b/modules/vye/lib/vye/batch_transfer/ingress_files.rb @@ -18,7 +18,9 @@ def bdn_import(data) profile = Vye::UserProfile.build(parsed.attributes[:profile]) info = profile.user_infos.build(parsed.attributes[:info]) info.address_changes.build({ origin: 'backend' }.merge(parsed.attributes[:address])) - + parsed.attributes[:awards].each do |award| + info.awards.build(award) + end profile.save! end end