Revert "ClamAV in Containers (#15965)" (#16557)

This reverts commit 0857270.

.github/CODEOWNERS

@@ -6,16 +6,11 @@
 Dangerfile @department-of-veterans-affairs/backend-review-group
 Dockerfile @department-of-veterans-affairs/backend-review-group
 Dockerfile-k8s @department-of-veterans-affairs/backend-review-group
-docker-compose.yml @department-of-veterans-affairs/backend-review-group
-docker-compose-clamav.yml @department-of-veterans-affairs/backend-review-group
-docker-compose-deps.yml @department-of-veterans-affairs/backend-review-group
-docker-compose.review.yml @department-of-veterans-affairs/backend-review-group
-docker-compose.test.yml @department-of-veterans-affairs/backend-review-group
+docker-compose* @department-of-veterans-affairs/backend-review-group
 Gemfile @department-of-veterans-affairs/backend-review-group
 Gemfile.lock @department-of-veterans-affairs/backend-review-group
 Jenkinsfile @department-of-veterans-affairs/backend-review-group
 Makefile @department-of-veterans-affairs/backend-review-group
-Procfile @department-of-veterans-affairs/backend-review-group
 .devcontainer @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/cto-engineers
 app/controllers/appeals_base_controller.rb @department-of-veterans-affairs/backend-review-group
 app/controllers/appeals_base_controller_v1.rb @department-of-veterans-affairs/backend-review-group
@@ -644,13 +639,13 @@ app/sidekiq/vbms @department-of-veterans-affairs/benefits-dependents-management
 app/sidekiq/vre/create_ch31_submissions_report_job.rb @department-of-veterans-affairs/benefits-non-disability @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/vre/submit1900_job.rb @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/webhooks @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+bin/fake_clamdscan @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/git_blame @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/rails @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/rake @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/rspec @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/setup @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 bin/sidekiq_quiet @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-clamav_tmp @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/application.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/betamocks @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/betamocks/services_config.yml @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -711,7 +706,7 @@ config/initializers/backtrace_silencers.rb @department-of-veterans-affairs/va-ap
 config/initializers/betamocks.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/bgs.rb @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/breakers.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-config/initializers/clamav.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+config/initializers/clamscan.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/config.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/cookie_rotation.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 config/initializers/covid_vaccine_facilities.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/long-covid
@@ -808,7 +803,6 @@ lib/caseflow @department-of-veterans-affairs/lighthouse-banana-peels @department
 lib/central_mail @department-of-veterans-affairs/lighthouse-banana-peels @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/chip @department-of-veterans-affairs/vsa-healthcare-health-quest-1-backend @department-of-veterans-affairs/patient-check-in @department-of-veterans-affairs/backend-review-group
 lib/claim_letters @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-lib/clamav @department-of-veterans-affairs/backend-review-group
 lib/common/client/base.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/common/client/concerns/mhv_fhir_session_client.rb @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/common/client/concerns/mhv_jwt_session_client.rb @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -820,7 +814,6 @@ lib/common/client/middleware/request/remove_cookies.rb @department-of-veterans-a
 lib/common/client/middleware/response/soap_parser.rb @department-of-veterans-affairs/backend-review-group
 lib/common/exceptions/open_id_service_error.rb @department-of-veterans-affairs/lighthouse-pivot
 lib/common/file_helpers.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
-lib/common/virus_scan.rb @department-of-veterans-affairs/backend-review-group
 lib/debt_management_center @department-of-veterans-affairs/vsa-debt-resolution @department-of-veterans-affairs/backend-review-group
 lib/decision_review @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/decision_review_v1 @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -927,7 +920,6 @@ lib/search @department-of-veterans-affairs/va-api-engineers @department-of-veter
 lib/sentry @department-of-veterans-affairs/backend-review-group
 lib/sentry_logging.rb @department-of-veterans-affairs/backend-review-group
 lib/sftp_writer @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
-lib/shrine @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/sidekiq/attr_package.rb @department-of-veterans-affairs/octo-identity @department-of-veterans-affairs/backend-review-group
 lib/sidekiq/error_tag.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/sidekiq/form526_backup_submission_process @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
@@ -1394,7 +1386,6 @@ spec/lib/sentry @department-of-veterans-affairs/va-api-engineers @department-of-
 spec/lib/sftp_writer @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sftp_writer/factory_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sftp_writer/remote_spec.rb @department-of-veterans-affairs/backend-review-group
-spec/lib/shrine @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sidekiq/attr_package_spec.rb @department-of-veterans-affairs/octo-identity @department-of-veterans-affairs/backend-review-group
 spec/lib/sidekiq/error_tag_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/lib/sidekiq/form526_backup_submission_process @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/dbex-trex @department-of-veterans-affairs/benefits-disability-2 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group

.github/workflows/audit_service_tags.yml

@@ -36,9 +36,10 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           build-args: |
-            BUNDLE_ENTERPRISE__CONTRIBSYS__COM=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }}
-            USER_ID=${{ env.VETS_API_USER_ID }}
+            sidekiq_license=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }}
+            userid=${{ env.VETS_API_USER_ID }}
           context: .
+          target: builder
           push: false
           load: true
           tags: vets-api
@@ -47,8 +48,8 @@ jobs:
 
       - name: Setup Database
         run: |
-             docker-compose -f docker-compose.test.yml run web bash \
-             -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true bundle exec parallel_test -n 13 -e 'bin/rails db:reset'"
+             docker-compose -f docker-compose.test.yml run vets-api bash \
+             -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -n 13 -e 'bin/rails db:reset'"
 
       - name: Get changed files
         run: |
@@ -59,6 +60,6 @@ jobs:
 
       - name: Run service tags audit controllers task
         run: |
-             docker-compose -f docker-compose.test.yml run -e CHANGED_FILES=${{ env.CHANGED_FILES }} web bash \
+             docker-compose -f docker-compose.test.yml run -e CHANGED_FILES=${{ env.CHANGED_FILES }} vets-api bash \
              -c "CI=true DISABLE_BOOTSNAP=true bundle exec rake service_tags:audit_controllers_ci"
 

.github/workflows/code_checks.yml

@@ -54,9 +54,10 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           build-args: |
-            BUNDLE_ENTERPRISE__CONTRIBSYS__COM=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }}
-            USER_ID=${{ env.VETS_API_USER_ID }}
+            sidekiq_license=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }}
+            userid=${{ env.VETS_API_USER_ID }}
           context: .
+          target: builder
           push: false
           load: true
           tags: vets-api
@@ -65,13 +66,13 @@ jobs:
 
       - name: Setup Database
         run: |
-             docker-compose -f docker-compose.test.yml run web bash \
-             -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true bundle exec parallel_test -n 13 -e 'bin/rails db:reset'"
+             docker-compose -f docker-compose.test.yml run vets-api bash \
+             -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -n 13 -e 'bin/rails db:reset'"
 
       - name: Run Specs
         timeout-minutes: 20
         run: |
-             docker-compose -f docker-compose.test.yml run web bash \
+             docker-compose -f docker-compose.test.yml run vets-api bash \
              -c "CI=true DISABLE_BOOTSNAP=true bundle exec parallel_rspec spec/ modules/ -n 13 -o '--color --tty'"
 
       - name: Upload Coverage Report

.gitignore

@@ -102,7 +102,3 @@ node_modules
 # Ignore public folder (used for local document uploads)
 public
 
-# Ignore any files within clamav_tmp
-
-clamav_tmp/*
-!/clamav_tmp/.keep

Dockerfile

@@ -1,4 +1,9 @@
-FROM ruby:3.2.4-slim-bookworm AS rubyimg
+FROM ruby:3.2.4-slim-bookworm as rubyimg
+
+# XXX: using stretch here for pdftk dep, which is not availible after
+#      stretch (or in alpine) and is switched automatically to pdftk-java in buster
+#      https://github.com/department-of-veterans-affairs/va.gov-team/issues/3032
+
 FROM rubyimg AS modules
 
 WORKDIR /tmp
@@ -8,64 +13,96 @@ COPY modules/ modules/
 RUN find modules -type f ! \( -name Gemfile -o -name "*.gemspec" -o -path "*/lib/*/version.rb" \) -delete && \
     find modules -type d -empty -delete
 
-FROM rubyimg
-
-# Allow for setting ENV vars via --build-arg
-ARG BUNDLE_ENTERPRISE__CONTRIBSYS__COM \
-  RAILS_ENV=development \
-  USER_ID=1000
-ENV RAILS_ENV=$RAILS_ENV \
-  BUNDLE_ENTERPRISE__CONTRIBSYS__COM=$BUNDLE_ENTERPRISE__CONTRIBSYS__COM \
-  BUNDLER_VERSION=2.4.9
-
-RUN groupadd --gid $USER_ID nonroot \
-  && useradd --uid $USER_ID --gid nonroot --shell /bin/bash --create-home nonroot --home-dir /app
-
-WORKDIR /app
+###
+# shared build/settings for all child images, reuse these layers yo
+###
+FROM rubyimg AS base
 
+ARG userid=993
+SHELL ["/bin/bash", "-c"]
+RUN groupadd -g $userid -r vets-api && \
+    useradd -u $userid -r -m -d /srv/vets-api -g vets-api vets-api
 RUN apt-get update --fix-missing
-RUN apt-get install -y poppler-utils build-essential libpq-dev git curl wget ca-certificates-java file \
-  imagemagick pdftk \
-  && apt-get clean \
-  && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates-java && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y dumb-init imagemagick pdftk poppler-utils curl \
+    libpq5 vim libboost-all-dev clamav clamdscan clamav-daemon
+
+# The pki work below is for parity with the non-docker BRD deploys to mount certs into
+# the container, we need to get rid of it and refactor the configuration bits into
+# something more continer friendly in a later bunch of work
+RUN mkdir -p /srv/vets-api/{clamav/database,pki/tls,secure,src} && \
+    chown -R vets-api:vets-api /srv/vets-api && \
+    ln -s /srv/vets-api/pki /etc/pki
+# XXX: get rid of the CA trust manipulation when we have a better model for it
+COPY config/ca-trust/* /usr/local/share/ca-certificates/
+# rename .pem files to .crt because update-ca-certificates ignores files that are not .crt
+RUN cd /usr/local/share/ca-certificates ; for i in *.pem ; do mv $i ${i/pem/crt} ; done ; update-ca-certificates
 # Relax ImageMagick PDF security. See https://stackoverflow.com/a/59193253.
 RUN sed -i '/rights="none" pattern="PDF"/d' /etc/ImageMagick-6/policy.xml
-
-
-# Install fwdproxy.crt into trust store
-# Relies on update-ca-certificates being run in following step
-COPY config/ca-trust/*.crt /usr/local/share/ca-certificates/
-
-# Download VA Certs
-COPY ./import-va-certs.sh .
-RUN ./import-va-certs.sh
-
-COPY config/clamd.conf /etc/clamav/clamd.conf
-
-RUN mkdir -p /clamav_tmp && \
-    chown -R nonroot:nonroot /clamav_tmp && \
-    chmod 777 /clamav_tmp
-
-
-ENV LANG=C.UTF-8 \
-   BUNDLE_JOBS=4 \
-   BUNDLE_PATH=/usr/local/bundle/cache \
-   BUNDLE_RETRY=3
-
+WORKDIR /srv/vets-api/src
+
+###
+# dev stage; use --target=development to stop here
+# Be sure to pass required ARGs as `--build-arg`
+# This stage useful for mounting your local checkout with compose
+# into the container to dev against.
+###
+FROM base AS development
+
+ARG sidekiq_license
+ARG rails_env=development
+
+ENV BUNDLE_ENTERPRISE__CONTRIBSYS__COM=$sidekiq_license
+ENV RAILS_ENV=$rails_env
+ENV BUNDLER_VERSION=2.4.9
+
+# only extra dev/build opts go here, common packages go in base 👆
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    git build-essential libxml2-dev libxslt-dev libpq-dev
+COPY --chown=vets-api:vets-api config/freshclam.conf docker-entrypoint.sh ./
+USER vets-api
+# XXX: this is tacky
+RUN freshclam --config-file freshclam.conf
+RUN gem install vtk
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "./docker-entrypoint.sh"]
 RUN gem install bundler:${BUNDLER_VERSION} --no-document
 
-COPY --from=modules /tmp/modules modules/
-COPY Gemfile Gemfile.lock ./
-RUN bundle install \
+###
+# build stage; use --target=builder to stop here
+# Also be sure to add build-args from development stage above
+#
+# This is development with the app copied in and built.  The build results are used in
+# prod below, but also useful if you want to have a container with the app and not
+# mount your local checkout.
+###
+FROM development AS builder
+# XXX: move modules/ to seperate repos so we can only copy Gemfile* and install a slim layer
+ARG bundler_opts
+
+COPY --chown=vets-api:vets-api Gemfile Gemfile.lock ./
+COPY --chown=vets-api:vets-api --from=modules /tmp/modules modules/
+
+RUN bundle install --binstubs="${BUNDLE_APP_CONFIG}/bin" $bundler_opts \
   && rm -rf /usr/local/bundle/cache/*.gem \
   && find /usr/local/bundle/gems/ -name "*.c" -delete \
   && find /usr/local/bundle/gems/ -name "*.o" -delete \
   && find /usr/local/bundle/gems/ -name ".git" -type d -prune -execdir rm -rf {} +
-COPY --chown=nonroot:nonroot . .
-
-EXPOSE 3000
-
-USER nonroot
 
-CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0"]
+COPY --chown=vets-api:vets-api . .
+USER vets-api
+
+###
+# prod stage; default if no target given
+# to build prod you probably want options like below to get a good build
+# --build-arg sidekiq_license="$BUNDLE_ENTERPRISE__CONTRIBSYS__COM" --build-arg rails_env=production --build-arg bundler_opts="--no-cache --without development test"
+# This inherits from base again to avoid bringing in extra built time binary packages
+###
+FROM base AS production
+
+ENV RAILS_ENV=production
+COPY --from=builder $BUNDLE_APP_CONFIG $BUNDLE_APP_CONFIG
+COPY --from=builder --chown=vets-api:vets-api /srv/vets-api/src ./
+COPY --from=builder --chown=vets-api:vets-api /srv/vets-api/clamav/database ../clamav/database
+RUN if [ -d certs-tmp ] ; then cd certs-tmp ; for i in * ; do cp $i /usr/local/share/ca-certificates/${i/pem/crt} ; done ; fi && update-ca-certificates
+USER vets-api
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "./docker-entrypoint.sh"]

Gemfile

@@ -55,7 +55,7 @@ gem 'bootsnap', require: false
 gem 'breakers'
 gem 'carrierwave'
 gem 'carrierwave-aws'
-gem 'clamav-client', require: 'clamav/client'
+gem 'clam_scan'
 gem 'combine_pdf'
 gem 'config'
 gem 'connect_vbms', git: 'https://github.com/adhocteam/connect_vbms', tag: 'v2.0.0.rc', require: 'vbms'

Gemfile.lock

@@ -301,7 +301,7 @@ GEM
       cork
       nap
       open4 (~> 1.3)
-    clamav-client (3.2.0)
+    clam_scan (0.0.2)
     cliver (0.3.2)
     coderay (1.1.3)
     coercible (1.0.0)
@@ -596,12 +596,9 @@ GEM
       kramdown (~> 2.0)
     language_server-protocol (3.17.0.3)
     libdatadog (5.0.0.1.0)
-    libdatadog (5.0.0.1.0-aarch64-linux)
     libdatadog (5.0.0.1.0-x86_64-linux)
     libddwaf (1.14.0.0.0)
       ffi (~> 1.0)
-    libddwaf (1.14.0.0.0-aarch64-linux)
-      ffi (~> 1.0)
     libddwaf (1.14.0.0.0-java)
       ffi (~> 1.0)
     libddwaf (1.14.0.0.0-x86_64-linux)
@@ -1079,7 +1076,6 @@ GEM
     zeitwerk (2.6.13)
 
 PLATFORMS
-  aarch64-linux
   java
   ruby
   x64-mingw32
@@ -1115,7 +1111,7 @@ DEPENDENCIES
   carrierwave-aws
   check_in!
   claims_api!
-  clamav-client
+  clam_scan
   combine_pdf
   config
   connect_vbms!