Bdn import (#16079)

* BDN import

* ingress awards also

modules/vye/app/models/vye/address_change.rb

@@ -4,18 +4,24 @@ module Vye
   class Vye::AddressChange < ApplicationRecord
     belongs_to :user_info
 
-    ENCRYPTED_ATTRIBUTES = %i[
-      veteran_name address1 address2 address3 address4 city state zip_code
-    ].freeze
-
     has_kms_key
-    has_encrypted(*ENCRYPTED_ATTRIBUTES, key: :kms_key, **lockbox_options)
 
-    REQUIRED_ATTRIBUTES = %i[
-      veteran_name address1 city state
-    ].freeze
+    has_encrypted(
+      :veteran_name,
+      :address1, :address2, :address3, :address4, :address5,
+      :city, :state, :zip_code,
+      key: :kms_key, **lockbox_options
+    )
+
+    validates(
+      :veteran_name, :address1, :city, :state,
+      presence: true, if: -> { origin == 'frontend' }
+    )
 
-    validates(*REQUIRED_ATTRIBUTES, presence: true)
+    validates(
+      :veteran_name, :address1,
+      presence: true, if: -> { origin == 'backend' }
+    )
 
     enum origin: { frontend: 'f', backend: 'b' }
 

modules/vye/app/models/vye/award.rb

@@ -6,11 +6,12 @@ class Vye::Award < ApplicationRecord
 
     enum cur_award_ind: { current: 'C', future: 'F', past: 'P' }
 
-    REQUIRED_ATTRIBUTES = %i[
-      award_begin_date award_end_date begin_rsn cur_award_ind end_rsn
-      monthly_rate number_hours payment_date training_time type_hours type_training
-    ].freeze
-
-    validates(*REQUIRED_ATTRIBUTES, presence: true)
+    validates(
+      *%i[
+        award_end_date cur_award_ind end_rsn
+        monthly_rate number_hours payment_date training_time
+      ].freeze,
+      presence: true
+    )
   end
 end

modules/vye/app/models/vye/user_info.rb

@@ -4,11 +4,16 @@ module Vye
   class Vye::UserInfo < ApplicationRecord
     INCLUDES = %i[address_changes awards pending_documents verifications].freeze
 
-    self.ignored_columns += %i[
-      address_line2_ciphertext address_line3_ciphertext address_line4_ciphertext
-      address_line5_ciphertext address_line6_ciphertext
-      full_name_ciphertext icn ssn_digest suffix zip_ciphertext
-    ]
+    self.ignored_columns +=
+      [
+        :ssn_digest, :icn, # moved to UserProfile
+
+        :suffix, # not needed
+
+        :address_line2_ciphertext, :address_line3_ciphertext,             # moved to AddressChange
+        :address_line4_ciphertext, :address_line5_ciphertext,             # moved to AddressChange
+        :address_line6_ciphertext, :full_name_ciphertext, :zip_ciphertext # moved to AddressChange
+      ]
 
     belongs_to :user_profile
 
@@ -28,12 +33,12 @@ class Vye::UserInfo < ApplicationRecord
     delegate :icn, to: :user_profile, allow_nil: true
     delegate :pending_documents, to: :user_profile, allow_nil: true
 
-    %i[dob file_number ssn stub_nm].freeze.tap do |attributes|
-      has_kms_key
-      has_encrypted(*attributes, key: :kms_key, **lockbox_options)
+    has_kms_key
+    has_encrypted(:file_number, :ssn, :dob, :stub_nm, key: :kms_key, **lockbox_options)
 
-      validates(*attributes, presence: true)
-    end
+    validates :dob, :stub_nm, presence: true
+
+    validate :ssn_or_file_number_present
 
     validates(
       :cert_issue_date, :date_last_certified, :del_date, :fac_code, :indicator,
@@ -45,6 +50,12 @@ def verification_required
       verifications.empty?
     end
 
+    def ssn_or_file_number_present
+      return true if ssn.present? || file_number.present?
+
+      errors.add(:base, 'Either SSN or file number must be present.')
+    end
+
     scope :with_assos, -> { includes(:address_changes, :awards, user_profile: :pending_documents) }
   end
 end

modules/vye/app/models/vye/user_profile.rb

@@ -10,7 +10,7 @@ class Vye::UserProfile < ApplicationRecord
   digest_attribute :ssn
   digest_attribute :file_number
 
-  validates :ssn_digest, :file_number_digest, presence: true
+  validate :ssn_or_file_number_present
 
   scope :with_assos, -> { includes(:pending_documents, :user_infos) }
 
@@ -25,4 +25,12 @@ def self.find_and_update_icn(user:)
       result&.update!(icn: user.icn)
     end
   end
+
+  private
+
+  def ssn_or_file_number_present
+    return true if ssn_digest.present? || file_number_digest.present?
+
+    errors.add(:base, 'Either SSN or file number must be present.')
+  end
 end

modules/vye/config/bdn_line_extraction_config.yaml

@@ -0,0 +1,61 @@
+---
+:main_line:
+  :ssn: 9
+  :file_number: 9
+  :suffix: 2
+  :dob: 8
+  :mr_status: 1
+  :rem_ent: 7
+  :cert_issue_date: 8
+  :del_date: 8
+  :date_last_certified: 8
+  :veteran_name: 20
+  :address1: 20
+  :address2: 20
+  :address3: 20
+  :address4: 20
+  :address5: 20
+  :zip_code: 9
+  :stub_nm: 7
+  :rpo_code: 3
+  :fac_code: 8
+  :payment_amt: 7
+:award_line:
+  :award_begin_date: 8
+  :award_end_date: 8
+  :training_time: 1
+  :payment_date: 8
+  :monthly_rate: 7
+  :begin_rsn: 2
+  :end_rsn: 2
+  :type_training: 1
+  :number_hours: 2
+  :type_hours: 1
+  :cur_award_ind: 1
+:mappings:
+  :profile:
+  - :ssn
+  - :file_number
+  :info:
+  - :ssn
+  - :file_number
+  - :dob
+  - :mr_status
+  - :rem_ent
+  - :cert_issue_date
+  - :del_date
+  - :date_last_certified
+  - :stub_nm
+  - :rpo_code
+  - :fac_code
+  - :payment_amt
+  - :indicator
+  :address:
+  - :veteran_name
+  - :address1
+  - :address2
+  - :address3
+  - :address4
+  - :address5
+  - :zip_code
+:indicator: true

modules/vye/lib/vye/batch_transfer/ingress_files.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module VYE
+module Vye
   module BatchTransfer
     module IngressFiles
       module_function
@@ -10,6 +10,20 @@ module IngressFiles
 
       def bdn_feed_filename = BDN_FEED_FILENAME
       def tims_feed_filename = TIMS_FEED_FILENAME
+
+      def bdn_import(data)
+        data.each_line do |line|
+          parsed = BdnLineExtraction.new(line: line.chomp, result: {}, award_lines: [], awards: [])
+
+          profile = Vye::UserProfile.build(parsed.attributes[:profile])
+          info = profile.user_infos.build(parsed.attributes[:info])
+          info.address_changes.build({ origin: 'backend' }.merge(parsed.attributes[:address]))
+          parsed.attributes[:awards].each do |award|
+            info.awards.build(award)
+          end
+          profile.save!
+        end
+      end
     end
   end
 end

modules/vye/lib/vye/batch_transfer/ingress_files/bdn_line_extraction.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Vye
+  module BatchTransfer
+    module IngressFiles
+      BdnLineExtraction = Struct.new(:line, :result, :award_lines, :awards) do
+        def initialize(line:, result:, award_lines:, awards:)
+          super
+
+          extract_main
+          extract_award_lines
+          extract_indicator
+          extract_awards
+        end
+
+        def config
+          @config ||= YAML.load_file Vye::Engine.root / 'config/bdn_line_extraction_config.yaml'
+        end
+
+        def extract_main
+          config[:main_line]
+            .each do |field, length|
+              extracted = line.slice!(0...length).strip
+              result.update(field => extracted)
+            end
+        end
+
+        def extract_award_lines
+          (0...4)
+            .each do |_i|
+              dead = line[0...8].strip == ''
+              extracted = line.slice!(0...41)
+              award_lines << extracted unless dead
+            end
+        end
+
+        def extract_awards
+          # iterate over the fields for each award line and extract the data
+          # think list comprehensions in python, haskell, or erlang
+          award_lines
+            .each_with_index.to_a
+            .product(config[:award_line].each_pair.to_a)
+            .each do |(award_line, i), (field, length)|
+              extracted = award_line.slice!(0...length).strip
+              awards[i] ||= {}
+              awards[i].update(field => extracted)
+            end
+        end
+
+        def extract_indicator
+          return unless config[:indicator]
+
+          result.update(indicator: line.slice!(0...1).strip)
+        end
+
+        def attributes
+          raise 'incomplete extraction' unless line.blank? && award_lines.all?(&:blank?)
+
+          profile = result.slice(*config[:mappings][:profile])
+          info = result.slice(*config[:mappings][:info])
+          address = result.slice(*config[:mappings][:address])
+
+          { profile:, info:, address:, awards: }
+        end
+      end
+    end
+  end
+end

modules/vye/lib/vye/engine.rb

@@ -6,6 +6,7 @@ module Vye
   class Engine < Rails::Engine
     isolate_namespace Vye
     config.generators.api_only = true
+    config.autoload_paths << (root / 'lib')
 
     initializer 'model_core.factories', after: 'factory_bot.set_factory_paths' do
       FactoryBot.definition_file_paths << File.expand_path('../../spec/factories', __dir__) if defined?(FactoryBot)

modules/vye/spec/factories/vye/address_changes.rb

@@ -7,5 +7,6 @@
     city { Faker::Address.city }
     state { Faker::Address.state_abbr }
     zip_code { Faker::Address.zip_code }
+    origin { Vye::AddressChange.origins['frontend'] }
   end
 end

modules/vye/spec/fixtures/bdn_sample/WAVE.txt

@@ -0,0 +1 @@
+123456789           19800101E3600000198603281996020519860328JOHN APPLESEED      1 Mockingbird Ln    APT 1               Houston TX                                                  77401    JAPPLES316119071110011550                                         00000000198603281198603280003500  66 00 C                                                                                  A

modules/vye/spec/lib/vye/batch_transfer/ingress_files_spec.rb

@@ -3,7 +3,7 @@
 require 'rails_helper'
 require 'vye/batch_transfer/ingress_files'
 
-RSpec.describe VYE::BatchTransfer::IngressFiles do
+RSpec.describe Vye::BatchTransfer::IngressFiles do
   describe '#bdn_feed_filename' do
     it 'returns a string' do
       expect(described_class.bdn_feed_filename).to be_a(String)
@@ -15,4 +15,11 @@
       expect(described_class.tims_feed_filename).to be_a(String)
     end
   end
+
+  it 'imports lines from BDN extract' do
+    data = Vye::Engine.root / 'spec/fixtures/bdn_sample/WAVE.txt'
+    expect do
+      described_class.bdn_import(data)
+    end.not_to raise_error
+  end
 end