Merge master

department-of-veterans-affairs · Apr 30, 2024 · be7a8b4 · be7a8b4
2 parents 7612365 + c0a404a
commit be7a8b4
Show file tree

Hide file tree

Showing 37 changed files with 346 additions and 168 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -6,11 +6,16 @@
 Dangerfile @department-of-veterans-affairs/backend-review-group
 Dockerfile @department-of-veterans-affairs/backend-review-group
 Dockerfile-k8s @department-of-veterans-affairs/backend-review-group
-docker-compose* @department-of-veterans-affairs/backend-review-group
+docker-compose.yml @department-of-veterans-affairs/backend-review-group
+docker-compose-clamav.yml @department-of-veterans-affairs/backend-review-group
+docker-compose-deps.yml @department-of-veterans-affairs/backend-review-group
+docker-compose.review.yml @department-of-veterans-affairs/backend-review-group
+docker-compose.test.yml @department-of-veterans-affairs/backend-review-group
 Gemfile @department-of-veterans-affairs/backend-review-group
 Gemfile.lock @department-of-veterans-affairs/backend-review-group
 Jenkinsfile @department-of-veterans-affairs/backend-review-group
 Makefile @department-of-veterans-affairs/backend-review-group
+Procfile @department-of-veterans-affairs/backend-review-group
 .devcontainer @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/cto-engineers
 app/controllers/appeals_base_controller.rb @department-of-veterans-affairs/backend-review-group
 app/controllers/appeals_base_controller_v1.rb @department-of-veterans-affairs/backend-review-group
@@ -640,13 +645,13 @@ app/sidekiq/vbms @department-of-veterans-affairs/benefits-dependents-management
 app/sidekiq/vre/create_ch31_submissions_report_job.rb @department-of-veterans-affairs/benefits-non-disability @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/vre/submit1900_job.rb @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/webhooks @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-bin/fake_clamdscan @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/git_blame @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/rails @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/rake @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/rspec @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/setup @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/sidekiq_quiet @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+clamav_tmp @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/application.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/betamocks @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/betamocks/services_config.yml @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -707,7 +712,7 @@ config/initializers/backtrace_silencers.rb @department-of-veterans-affairs/va-ap
 config/initializers/betamocks.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/bgs.rb @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/breakers.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-config/initializers/clamscan.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+config/initializers/clamav.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/config.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/cookie_rotation.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/covid_vaccine_facilities.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/long-covid
@@ -804,6 +809,7 @@ lib/caseflow @department-of-veterans-affairs/lighthouse-banana-peels @department
 lib/central_mail @department-of-veterans-affairs/lighthouse-banana-peels @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/chip @department-of-veterans-affairs/vsa-healthcare-health-quest-1-backend @department-of-veterans-affairs/patient-check-in @department-of-veterans-affairs/backend-review-group
 lib/claim_letters @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+lib/clamav @department-of-veterans-affairs/backend-review-group
 lib/common/client/base.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/common/client/concerns/mhv_fhir_session_client.rb @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/common/client/concerns/mhv_jwt_session_client.rb @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -815,6 +821,7 @@ lib/common/client/middleware/request/remove_cookies.rb @department-of-veterans-a
 lib/common/client/middleware/response/soap_parser.rb @department-of-veterans-affairs/backend-review-group
 lib/common/exceptions/open_id_service_error.rb @department-of-veterans-affairs/lighthouse-pivot
 lib/common/file_helpers.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+lib/common/virus_scan.rb @department-of-veterans-affairs/backend-review-group
 lib/debt_management_center @department-of-veterans-affairs/vsa-debt-resolution @department-of-veterans-affairs/backend-review-group
 lib/decision_review @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/decision_review_v1 @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -921,6 +928,7 @@ lib/search @department-of-veterans-affairs/va-api-engineers @department-of-veter
 lib/sentry @department-of-veterans-affairs/backend-review-group
 lib/sentry_logging.rb @department-of-veterans-affairs/backend-review-group
 lib/sftp_writer @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
+lib/shrine @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/sidekiq/attr_package.rb @department-of-veterans-affairs/octo-identity @department-of-veterans-affairs/backend-review-group
 lib/sidekiq/error_tag.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/sidekiq/form526_backup_submission_process @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
@@ -1388,6 +1396,7 @@ spec/lib/sentry @department-of-veterans-affairs/va-api-engineers @department-of-
 spec/lib/sftp_writer @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sftp_writer/factory_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sftp_writer/remote_spec.rb @department-of-veterans-affairs/backend-review-group
+spec/lib/shrine @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sidekiq/attr_package_spec.rb @department-of-veterans-affairs/octo-identity @department-of-veterans-affairs/backend-review-group
 spec/lib/sidekiq/error_tag_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sidekiq/form526_backup_submission_process @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/dbex-trex @department-of-veterans-affairs/benefits-disability-2 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group

diff --git a/.github/workflows/audit_service_tags.yml b/.github/workflows/audit_service_tags.yml
@@ -36,10 +36,9 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           build-args: |
-            sidekiq_license=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }}
-            userid=${{ env.VETS_API_USER_ID }}
+            BUNDLE_ENTERPRISE__CONTRIBSYS__COM=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }}
+            USER_ID=${{ env.VETS_API_USER_ID }}
           context: .
-          target: builder
           push: false
           load: true
           tags: vets-api
@@ -48,8 +47,8 @@ jobs:
 
       - name: Setup Database
         run: |
-             docker-compose -f docker-compose.test.yml run vets-api bash \
-             -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -n 13 -e 'bin/rails db:reset'"
+             docker-compose -f docker-compose.test.yml run web bash \
+             -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true bundle exec parallel_test -n 13 -e 'bin/rails db:reset'"
 
       - name: Get changed files
         run: |
@@ -60,6 +59,6 @@ jobs:
 
       - name: Run service tags audit controllers task
         run: |
-             docker-compose -f docker-compose.test.yml run -e CHANGED_FILES=${{ env.CHANGED_FILES }} vets-api bash \
+             docker-compose -f docker-compose.test.yml run -e CHANGED_FILES=${{ env.CHANGED_FILES }} web bash \
              -c "CI=true DISABLE_BOOTSNAP=true bundle exec rake service_tags:audit_controllers_ci"
 
diff --git a/.gitignore b/.gitignore
@@ -103,3 +103,7 @@ node_modules
 # Ignore public folder (used for local document uploads)
 public
 
+# Ignore any files within clamav_tmp
+
+clamav_tmp/*
+!/clamav_tmp/.keep
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ruby:3.2.4-slim-bullseye AS rubyimg
+FROM ruby:3.2.4-slim-bookworm AS rubyimg
 FROM rubyimg AS modules
 
 WORKDIR /tmp
@@ -23,10 +23,9 @@ RUN groupadd --gid $USER_ID nonroot \
 
 WORKDIR /app
 
-RUN echo "deb http://ftp.debian.org/debian testing main contrib non-free" >> /etc/apt/sources.list
-RUN apt-get update
-RUN apt-get install -y -t testing poppler-utils build-essential libpq-dev git curl wget ca-certificates-java file
-RUN dpkg --configure -a && apt-get install -y -t bullseye imagemagick pdftk \
+RUN apt-get update --fix-missing
+RUN apt-get install -y poppler-utils build-essential libpq-dev git curl wget ca-certificates-java file \
+  imagemagick pdftk \
   && apt-get clean \
   && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -69,4 +68,4 @@ EXPOSE 3000
 
 USER nonroot
 
-CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0"]
+CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0"]
diff --git a/Gemfile b/Gemfile
@@ -55,7 +55,7 @@ gem 'bootsnap', require: false
 gem 'breakers'
 gem 'carrierwave'
 gem 'carrierwave-aws'
-gem 'clam_scan'
+gem 'clamav-client', require: 'clamav/client'
 gem 'combine_pdf'
 gem 'config'
 gem 'connect_vbms', git: 'https://github.com/adhocteam/connect_vbms', tag: 'v2.0.0.rc', require: 'vbms'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -301,7 +301,7 @@ GEM
       cork
       nap
       open4 (~> 1.3)
-    clam_scan (0.0.2)
+    clamav-client (3.2.0)
     cliver (0.3.2)
     coderay (1.1.3)
     coercible (1.0.0)
@@ -596,9 +596,12 @@ GEM
       kramdown (~> 2.0)
     language_server-protocol (3.17.0.3)
     libdatadog (5.0.0.1.0)
+    libdatadog (5.0.0.1.0-aarch64-linux)
     libdatadog (5.0.0.1.0-x86_64-linux)
     libddwaf (1.14.0.0.0)
       ffi (~> 1.0)
+    libddwaf (1.14.0.0.0-aarch64-linux)
+      ffi (~> 1.0)
     libddwaf (1.14.0.0.0-java)
       ffi (~> 1.0)
     libddwaf (1.14.0.0.0-x86_64-linux)
@@ -1076,6 +1079,7 @@ GEM
     zeitwerk (2.6.13)
 
 PLATFORMS
+  aarch64-linux
   java
   ruby
   x64-mingw32
@@ -1111,7 +1115,7 @@ DEPENDENCIES
   carrierwave-aws
   check_in!
   claims_api!
-  clam_scan
+  clamav-client
   combine_pdf
   config
   connect_vbms!

diff --git a/Makefile b/Makefile
@@ -7,16 +7,9 @@ else
     ENV_ARG	 := dev
 endif
 
-ifdef clam
-	FOREMAN_ARG := all=1
-else
-	FOREMAN_ARG := all=1,clamd=0,freshclam=0
-endif
-
-
 COMPOSE_DEV  := docker-compose
 COMPOSE_TEST := docker-compose -f docker-compose.test.yml
-BASH         := run --rm --service-ports vets-api bash
+BASH         := run --rm --service-ports web bash
 BASH_DEV     := $(COMPOSE_DEV) $(BASH) -c
 BASH_TEST    := $(COMPOSE_TEST) $(BASH) --login -c
 SPEC_PATH    := spec/ modules/
@@ -117,9 +110,9 @@ spec:  ## Runs spec tests
 .PHONY: spec_parallel_setup
 spec_parallel_setup:  ## Setup the parallel test dbs. This resets the current test db, as well as the parallel test dbs
 ifeq ($(ENV_ARG), dev)
-	@$(BASH_DEV) "RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -e 'bundle exec rake db:reset'"
+	@$(BASH_DEV) "RAILS_ENV=test DISABLE_BOOTSNAP=true bundle exec parallel_test -e 'bundle exec rake db:reset db:migrate'"
 else
-	@$(COMPOSE_TEST) $(BASH) -c "RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -e 'bundle exec rake db:reset'"
+	@$(COMPOSE_TEST) $(BASH) -c "RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -e 'bundle exec rake db:reset db:migrate'"
 endif
 
 .PHONY: spec_parallel
@@ -131,14 +124,14 @@ else
 endif
 
 .PHONY: up
-up: db  ## Starts the server and associated services with docker-compose, use `clam=1 make up` to run ClamAV
-	@$(BASH_DEV) "rm -f tmp/pids/server.pid && foreman start -m ${FOREMAN_ARG}"
+up: db  ## Starts the server and associated services with docker-compose
+	@$(BASH_DEV) "rm -f tmp/pids/server.pid && foreman start -m all=1"
 
 # NATIVE COMMANDS
 .PHONY: native-up
 native-up:
 	bundle install
-	foreman start -m ${FOREMAN_ARG}
+	foreman start -m all=1
 
 .PHONY: native-lint
 native-lint:

diff --git a/Procfile b/Procfile
@@ -1,4 +1,2 @@
 web: bundle exec puma -p 3000 -C ./config/puma.rb
 job: bundle exec sidekiq -q critical,4 -q tasker,3 -q default,2 -q low,1
-freshclam: /usr/bin/freshclam -d --config-file=config/freshclam.conf
-clamd: /usr/sbin/clamd -c config/clamd.conf
diff --git a/app/uploaders/uploader_virus_scan.rb b/app/uploaders/uploader_virus_scan.rb
@@ -16,13 +16,14 @@ class VirusFoundError < StandardError
   def validate_virus_free(file)
     return unless Rails.env.production?
 
-    temp_file_path = Common::FileHelpers.generate_temp_file(file.read)
+    temp_file_path = Common::FileHelpers.generate_clamav_temp_file(file.read)
     result = Common::VirusScan.scan(temp_file_path)
     File.delete(temp_file_path)
 
-    unless result.safe?
+    # Common::VirusScan result will return true or false
+    unless result # unless safe
       file.delete
-      raise VirusFoundError, result.body
+      raise VirusFoundError, "Virus Found + #{temp_file_path}"
     end
   end
 end
diff --git a/bin/fake_clamdscan b/bin/fake_clamdscan
diff --git a/clamav_tmp/.keep b/clamav_tmp/.keep
diff --git a/config/clamd.conf b/config/clamd.conf
@@ -1,5 +1,7 @@
 Foreground yes
-DatabaseDirectory /srv/vets-api/clamav/database
-LocalSocket /srv/vets-api/clamav/clamd.ctl
 TCPSocket 3310
-TCPAddr 127.0.0.1
+TCPAddr 127.0.0.1
+
+LogSyslog yes
+LogVerbose yes
+ExtendedDetectionInfo yes
diff --git a/config/freshclam.conf b/config/freshclam.conf
@@ -1,7 +1,7 @@
 Foreground yes
-PidFile /srv/vets-api/clamav/freshclam.pid
+PidFile /app/clamav/freshclam.pid
 Checks 8
-DatabaseDirectory /srv/vets-api/clamav/database
+DatabaseDirectory /app/clamav/database
 PrivateMirror dsva-vetsgov-utility-clamav.s3-us-gov-west-1.amazonaws.com
-NotifyClamd  /srv/vets-api/src/config/clamd.conf
+NotifyClamd  /app/config/clamd.conf
 ReceiveTimeout 600
diff --git a/config/initializers/clamav.rb b/config/initializers/clamav.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+if Rails.env.development?
+  # If running ClamAV through container
+  # Update host and port on settings.local.yml to override the socket
+  ENV['CLAMD_TCP_HOST'] = Settings.clamav.host
+  ENV['CLAMD_TCP_PORT'] = Settings.clamav.port
+
+  # If running ClamAV natively (via daemon)
+  # Update host and port on settings.local.yml to override the tcp connection
+  ENV['CLAMD_UNIX_SOCKET'] = '/usr/local/etc/clamav/clamd.sock'
+end
diff --git a/config/initializers/clamscan.rb b/config/initializers/clamscan.rb
diff --git a/config/settings.local.yml.example b/config/settings.local.yml.example
@@ -6,11 +6,11 @@
   # The relative path to department-of-veterans-affairs/vets-api-mockdata
   # cache_dir: ../vets-api-mockdata
 
-# binaries:
-  # For NATIVE and DOCKER installation
+# clamav:
   # A "virus scanner" that always returns success for development purposes
-  # NOTE: You may need to specify a full path instead of a relative path
-  # clamdscan: ./bin/fake_clamdscan
+  # mock: true
+  # host: '0.0.0.0'
+  # port: '33100'
 
 # NOTE: This file is excluded by railsconfig in the test env.
 # Use config/settings/test.local.yml instead.

diff --git a/config/settings.yml b/config/settings.yml
@@ -97,6 +97,13 @@ binaries:
   pdftk: pdftk
   clamdscan: /usr/bin/clamdscan
 
+clamav:
+  mock: false
+  # host & port here are only used in development here:
+  # config/initializers/clamav.rb
+  host: 'clamav'
+  port: '3310'
+
 db_encryption_key: f01ff8ebd1a2b053ad697ae1f0d86adb
 
 database_url: postgis:///vets-api
@@ -914,7 +921,7 @@ vetext:
 hqva_mobile:
   url: "https://veteran.apps.va.gov"
   mock: false
-  key_path: /srv/vets-api/secret/health-quest.key
+  key_path: /app/secret/health-quest.key
   development_key_path: modules/health_quest/config/rsa/sandbox_rsa
   timeout: 15
   facilities:
@@ -937,7 +944,7 @@ hqva_mobile:
     health_api: "health_api"
     pgd_api: "pgd_api"
     mock: false
-    key_path: /srv/vets-api/secret/health-quest.lighthouse.key
+    key_path: /app/secret/health-quest.lighthouse.key
     pgd_api_scopes:
       - "launch launch/patient"
       - "patient/Observation.read"
@@ -1034,7 +1041,7 @@ lighthouse:
       aud_claim_url: "https://deptva-eval.okta.com/oauth2/aus8nm1q0f7VQ0a482p7/v1/token"
       client_id: "0oaaxkp0aeXEJkMFw2p7"
       grant_type: "client_credentials"
-      api_key: /srv/vets-api/secret/lighthouse_fast_track_api.key
+      api_key: /app/secret/lighthouse_fast_track_api.key
   facilities:
     url: https://sandbox-api.va.gov
     api_key: fake_key
@@ -1071,7 +1078,7 @@ vbms:
   cert: vetsapi.client.vbms.aide.oit.va.gov.crt
   client_keyfile: vetsapi.client.vbms.aide.oit.va.gov.p12
   server_cert: vbms.aide.oit.va.gov.crt
-  environment_directory: /srv/vets-api/secret
+  environment_directory: /app/secret
   env: test
 
 vet_verification:

diff --git a/docker-compose-clamav.yml b/docker-compose-clamav.yml
@@ -0,0 +1,10 @@
+version: '3.4'
+services:
+  clamav:
+    volumes:
+     - shared-vol:/vets-api
+    image: clamav/clamav
+    ports:
+      - 33100:3310
+volumes:
+ shared-vol:
diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml
@@ -13,4 +13,11 @@ services:
       - ./data:/var/lib/postgresql/data:cached
     ports:
       - "54320:5432"
-
+  clamav:
+    volumes:
+     - shared-vol:/vets-api
+    image: clamav/clamav
+    ports:
+      - 33100:3310
+volumes:
+ shared-vol: