diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0bcaba3a057..8979c9724a5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,11 +6,16 @@ Dangerfile @department-of-veterans-affairs/backend-review-group Dockerfile @department-of-veterans-affairs/backend-review-group Dockerfile-k8s @department-of-veterans-affairs/backend-review-group -docker-compose* @department-of-veterans-affairs/backend-review-group +docker-compose.yml @department-of-veterans-affairs/backend-review-group +docker-compose-clamav.yml @department-of-veterans-affairs/backend-review-group +docker-compose-deps.yml @department-of-veterans-affairs/backend-review-group +docker-compose.review.yml @department-of-veterans-affairs/backend-review-group +docker-compose.test.yml @department-of-veterans-affairs/backend-review-group Gemfile @department-of-veterans-affairs/backend-review-group Gemfile.lock @department-of-veterans-affairs/backend-review-group Jenkinsfile @department-of-veterans-affairs/backend-review-group Makefile @department-of-veterans-affairs/backend-review-group +Procfile @department-of-veterans-affairs/backend-review-group .devcontainer @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/cto-engineers app/controllers/appeals_base_controller.rb @department-of-veterans-affairs/backend-review-group app/controllers/appeals_base_controller_v1.rb @department-of-veterans-affairs/backend-review-group @@ -639,13 +644,13 @@ app/sidekiq/vbms @department-of-veterans-affairs/benefits-dependents-management app/sidekiq/vre/create_ch31_submissions_report_job.rb @department-of-veterans-affairs/benefits-non-disability @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/vre/submit1900_job.rb @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/webhooks @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group -bin/fake_clamdscan @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group bin/git_blame @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group bin/rails @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group bin/rake @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group bin/rspec @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group bin/setup @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group bin/sidekiq_quiet @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group +clamav_tmp @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/application.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/betamocks @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/betamocks/services_config.yml @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @@ -706,7 +711,7 @@ config/initializers/backtrace_silencers.rb @department-of-veterans-affairs/va-ap config/initializers/betamocks.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/initializers/bgs.rb @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/initializers/breakers.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group -config/initializers/clamscan.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group +config/initializers/clamav.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/initializers/config.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/initializers/cookie_rotation.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group config/initializers/covid_vaccine_facilities.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/long-covid @@ -803,6 +808,7 @@ lib/caseflow @department-of-veterans-affairs/lighthouse-banana-peels @department lib/central_mail @department-of-veterans-affairs/lighthouse-banana-peels @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/chip @department-of-veterans-affairs/vsa-healthcare-health-quest-1-backend @department-of-veterans-affairs/patient-check-in @department-of-veterans-affairs/backend-review-group lib/claim_letters @department-of-veterans-affairs/benefits-management-tools-be @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group +lib/clamav @department-of-veterans-affairs/backend-review-group lib/common/client/base.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/common/client/concerns/mhv_fhir_session_client.rb @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/common/client/concerns/mhv_jwt_session_client.rb @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @@ -814,6 +820,7 @@ lib/common/client/middleware/request/remove_cookies.rb @department-of-veterans-a lib/common/client/middleware/response/soap_parser.rb @department-of-veterans-affairs/backend-review-group lib/common/exceptions/open_id_service_error.rb @department-of-veterans-affairs/lighthouse-pivot lib/common/file_helpers.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group +lib/common/virus_scan.rb @department-of-veterans-affairs/backend-review-group lib/debt_management_center @department-of-veterans-affairs/vsa-debt-resolution @department-of-veterans-affairs/backend-review-group lib/decision_review @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/decision_review_v1 @department-of-veterans-affairs/Benefits-Team-1 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @@ -920,6 +927,7 @@ lib/search @department-of-veterans-affairs/va-api-engineers @department-of-veter lib/sentry @department-of-veterans-affairs/backend-review-group lib/sentry_logging.rb @department-of-veterans-affairs/backend-review-group lib/sftp_writer @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers +lib/shrine @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers lib/sidekiq/attr_package.rb @department-of-veterans-affairs/octo-identity @department-of-veterans-affairs/backend-review-group lib/sidekiq/error_tag.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers lib/sidekiq/form526_backup_submission_process @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers @@ -1386,6 +1394,7 @@ spec/lib/sentry @department-of-veterans-affairs/va-api-engineers @department-of- spec/lib/sftp_writer @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/lib/sftp_writer/factory_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/lib/sftp_writer/remote_spec.rb @department-of-veterans-affairs/backend-review-group +spec/lib/shrine @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/lib/sidekiq/attr_package_spec.rb @department-of-veterans-affairs/octo-identity @department-of-veterans-affairs/backend-review-group spec/lib/sidekiq/error_tag_spec.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/lib/sidekiq/form526_backup_submission_process @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/dbex-trex @department-of-veterans-affairs/benefits-disability-2 @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group diff --git a/.github/workflows/audit_service_tags.yml b/.github/workflows/audit_service_tags.yml index b81aad522b3..9391b69f875 100644 --- a/.github/workflows/audit_service_tags.yml +++ b/.github/workflows/audit_service_tags.yml @@ -36,10 +36,9 @@ jobs: uses: docker/build-push-action@v5 with: build-args: | - sidekiq_license=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }} - userid=${{ env.VETS_API_USER_ID }} + BUNDLE_ENTERPRISE__CONTRIBSYS__COM=${{ env.BUNDLE_ENTERPRISE__CONTRIBSYS__COM }} + USER_ID=${{ env.VETS_API_USER_ID }} context: . - target: builder push: false load: true tags: vets-api @@ -48,8 +47,8 @@ jobs: - name: Setup Database run: | - docker-compose -f docker-compose.test.yml run vets-api bash \ - -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -n 13 -e 'bin/rails db:reset'" + docker-compose -f docker-compose.test.yml run web bash \ + -c "CI=true RAILS_ENV=test DISABLE_BOOTSNAP=true bundle exec parallel_test -n 13 -e 'bin/rails db:reset'" - name: Get changed files run: | @@ -60,6 +59,6 @@ jobs: - name: Run service tags audit controllers task run: | - docker-compose -f docker-compose.test.yml run -e CHANGED_FILES=${{ env.CHANGED_FILES }} vets-api bash \ + docker-compose -f docker-compose.test.yml run -e CHANGED_FILES=${{ env.CHANGED_FILES }} web bash \ -c "CI=true DISABLE_BOOTSNAP=true bundle exec rake service_tags:audit_controllers_ci" diff --git a/.gitignore b/.gitignore index b8d480faeb8..38c53974184 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ node_modules # Ignore public folder (used for local document uploads) public +# Ignore any files within clamav_tmp + +clamav_tmp/* +!/clamav_tmp/.keep diff --git a/Gemfile.lock b/Gemfile.lock index cca173f651c..e3cf8f240a5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -596,9 +596,12 @@ GEM kramdown (~> 2.0) language_server-protocol (3.17.0.3) libdatadog (5.0.0.1.0) + libdatadog (5.0.0.1.0-aarch64-linux) libdatadog (5.0.0.1.0-x86_64-linux) libddwaf (1.14.0.0.0) ffi (~> 1.0) + libddwaf (1.14.0.0.0-aarch64-linux) + ffi (~> 1.0) libddwaf (1.14.0.0.0-java) ffi (~> 1.0) libddwaf (1.14.0.0.0-x86_64-linux) @@ -1076,6 +1079,7 @@ GEM zeitwerk (2.6.13) PLATFORMS + aarch64-linux java ruby x64-mingw32 diff --git a/Makefile b/Makefile index fc45c753431..cac52538ac9 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,6 @@ else ENV_ARG := dev endif -ifdef clam - FOREMAN_ARG := all=1 -else - FOREMAN_ARG := all=1,clamd=0,freshclam=0 -endif - - COMPOSE_DEV := docker-compose COMPOSE_TEST := docker-compose -f docker-compose.test.yml BASH := run --rm --service-ports web bash @@ -117,9 +110,9 @@ spec: ## Runs spec tests .PHONY: spec_parallel_setup spec_parallel_setup: ## Setup the parallel test dbs. This resets the current test db, as well as the parallel test dbs ifeq ($(ENV_ARG), dev) - @$(BASH_DEV) "RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -e 'bundle exec rake db:reset'" + @$(BASH_DEV) "RAILS_ENV=test DISABLE_BOOTSNAP=true bundle exec parallel_test -e 'bundle exec rake db:reset db:migrate'" else - @$(COMPOSE_TEST) $(BASH) -c "RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -e 'bundle exec rake db:reset'" + @$(COMPOSE_TEST) $(BASH) -c "RAILS_ENV=test DISABLE_BOOTSNAP=true parallel_test -e 'bundle exec rake db:reset db:migrate'" endif .PHONY: spec_parallel @@ -131,14 +124,14 @@ else endif .PHONY: up -up: db ## Starts the server and associated services with docker-compose, use `clam=1 make up` to run ClamAV - @$(BASH_DEV) "rm -f tmp/pids/server.pid && foreman start -m ${FOREMAN_ARG}" +up: db ## Starts the server and associated services with docker-compose + @$(BASH_DEV) "rm -f tmp/pids/server.pid && foreman start -m all=1" # NATIVE COMMANDS .PHONY: native-up native-up: bundle install - foreman start -m ${FOREMAN_ARG} + foreman start -m all=1 .PHONY: native-lint native-lint: diff --git a/Procfile b/Procfile index 074c2758193..9af216e5db4 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,2 @@ web: bundle exec puma -p 3000 -C ./config/puma.rb job: bundle exec sidekiq -q critical,4 -q tasker,3 -q default,2 -q low,1 -freshclam: /usr/bin/freshclam -d --config-file=config/freshclam.conf -clamd: /usr/sbin/clamd -c config/clamd.conf diff --git a/clamav_tmp/.keep b/clamav_tmp/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/config/initializers/clamav.rb b/config/initializers/clamav.rb index afe3384c171..bc13ae935c3 100644 --- a/config/initializers/clamav.rb +++ b/config/initializers/clamav.rb @@ -1,18 +1,6 @@ # frozen_string_literal: true -## If running clamav natively -# ENV['CLAMD_UNIX_SOCKET'] = '/usr/local/etc/clamav/clamd.sock' - -## Comment the following out (everything below) if you are running clamav natively - -## If running via docker if Rails.env.development? - ENV['CLAMD_TCP_HOST'] = 'clamav' - ENV['CLAMD_TCP_PORT'] = '3310' + ENV['CLAMD_TCP_HOST'] = Settings.clamav.host + ENV['CLAMD_TCP_PORT'] = Settings.clamav.port end - -# ## If running hybrid -# if Rails.env.development? -# ENV["CLAMD_TCP_HOST"] = "0.0.0.0" -# ENV["CLAMD_TCP_PORT"] = "33100" -# end diff --git a/config/settings.local.yml.example b/config/settings.local.yml.example index ad55d272e4a..c9db66102e6 100644 --- a/config/settings.local.yml.example +++ b/config/settings.local.yml.example @@ -6,11 +6,11 @@ # The relative path to department-of-veterans-affairs/vets-api-mockdata # cache_dir: ../vets-api-mockdata -# binaries: - # For NATIVE and DOCKER installation +# clamav: # A "virus scanner" that always returns success for development purposes - # NOTE: You may need to specify a full path instead of a relative path - # clamdscan: ./bin/fake_clamdscan + # mock: true + # host: '0.0.0.0' + # port: '33100' # NOTE: This file is excluded by railsconfig in the test env. # Use config/settings/test.local.yml instead. diff --git a/config/settings.yml b/config/settings.yml index 1625e49b708..e3f5af2d20c 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -99,6 +99,10 @@ binaries: clamav: mock: false + # host & port here are only used in development here: + # config/initializers/clamav.rb + host: 'clamav' + port: '3310' db_encryption_key: f01ff8ebd1a2b053ad697ae1f0d86adb diff --git a/docker-compose-clamav.yml b/docker-compose-clamav.yml index aa767e9888a..239aa2936a3 100644 --- a/docker-compose-clamav.yml +++ b/docker-compose-clamav.yml @@ -1,4 +1,4 @@ -version: '3.4' # test commit +version: '3.4' services: clamav: volumes: diff --git a/docker-compose.review.yml b/docker-compose.review.yml index 43769083ab7..4ce4b1999f9 100644 --- a/docker-compose.review.yml +++ b/docker-compose.review.yml @@ -1,55 +1,70 @@ version: '3.4' + +x-app: &common + build: + args: + BUNDLE_ENTERPRISE__CONTRIBSYS__COM: "${BUNDLE_ENTERPRISE__CONTRIBSYS__COM}" + USER_ID: ${VETS_API_USER_ID} + context: . + environment: + RAILS_ENV: development + BUNDLE_ENTERPRISE__CONTRIBSYS__COM: "${BUNDLE_ENTERPRISE__CONTRIBSYS__COM}" + "Settings.database_url": "postgis://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DATABASE:-vets_api_development}?pool=4" + "Settings.test_database_url": "postgis://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DATABASE:-vets_api_test}" + "Settings.redis.app_data.url": "redis://redis:6379" + "Settings.redis.sidekiq.url": "redis://redis:6379" + "Settings.redis.rails_cache.url": "redis://redis:6379" + "Settings.saml_ssoe.idp_metadata_file": "config/ssoe_idp_sqa_metadata_isam.xml" + "Settings.betamocks.cache_dir": "config/vets-api-mockdata" + image: vets-api:${DOCKER_IMAGE:-latest} + restart: unless-stopped + volumes: + - "../vets-api-mockdata:/cache" + - ../.secret:/srv/vets-api/secret:cached + - ../.pki:/srv/vets-api/pki:cached + - shared-vol:/tmp + working_dir: /app + depends_on: + - clamav + - postgres + - redis + links: + - clamav + - postgres + - redis + services: + clamav: + image: clamav/clamav + restart: unless-stopped + ports: + - 3310:3310 + volumes: + - shared-vol:/vets-api redis: image: redis:6.2-alpine restart: unless-stopped + ports: + - 6379:6379 postgres: - image: mdillon/postgis:11-alpine + command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c max_connections=200 environment: POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-password}" POSTGRES_USER: "${POSTGRES_USER:-postgres}" - volumes: - - db-data:/var/lib/postgresql/data:cached + PGDATA: /tmp + image: postgis/postgis:14-3.3-alpine ports: - - "54320:5432" - restart: unless-stopped - vets-api: - build: - context: . - target: development - args: - sidekiq_license: "${BUNDLE_ENTERPRISE__CONTRIBSYS__COM}" - userid: "${VETS_API_USER_ID}" - command: > - bash -c "bundle exec rake db:migrate || bundle exec rake db:setup db:migrate - && touch tmp/caching-dev.txt && foreman start -m all=1,clamd=0,freshclam=0" - image: "vets-api:${DOCKER_IMAGE:-latest}" + - 5432:5432 volumes: - - .:/srv/vets-api/src:cached - - dev_bundle:/usr/local/bundle - - ../.secret:/srv/vets-api/secret:cached - - ../.pki:/srv/vets-api/pki:cached + - ./data:/var/lib/postgresql/data:cached + web: + <<: *common + command: bash -c "bundle exec rake db:migrate || bundle exec rake db:reset db:migrate && bundle exec rails s -b 0.0.0.0" ports: - - "3000:3000" - environment: - "Settings.database_url": "postgis://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DATABASE:-vets_api_development}?pool=4" - "Settings.test_database_url": "postgis://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DATABASE:-vets_api_test}?pool=4" - "Settings.redis.app_data.url": "redis://redis:6379" - "Settings.redis.sidekiq.url": "redis://redis:6379" - "Settings.redis.rails_cache.url": "redis://redis:6379" - "Settings.binaries.clamdscan": "clamscan" # Not running a separate process within the container for clamdscan, so we use clamscan which requires no daemon - POSTGRES_HOST: "${POSTGRES_HOST:-postgres}" - POSTGRES_PORT: "${POSTGRES_PORT:-5432}" - POSTGRES_USER: "${POSTGRES_USER:-postgres}" - POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-password}" - PUMA_THREADS: "${PUMA_THREADS:-4}" - depends_on: - - postgres - - redis - links: - - postgres - - redis - restart: unless-stopped + - 3000:3000 + worker: + <<: *common + command: bundle exec sidekiq -q critical,4 -q tasker,3 -q default,2 -q low,1 + volumes: - db-data: - dev_bundle: + shared-vol: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index a69d19449dc..29597f70c13 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -8,7 +8,7 @@ services: environment: POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-password}" POSTGRES_USER: "${POSTGRES_USER:-postgres}" - vets-api: + web: build: context: . target: development @@ -22,7 +22,6 @@ services: environment: "Settings.database_url": "postgis://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DATABASE:-vets_api_development}?pool=4" "Settings.test_database_url": "postgis://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DATABASE:-vets_api_test}" - "Settings.binaries.clamdscan": "clamscan" # Not running a separate process within the container for clamdscan, so we use clamscan which requires no daemon "Settings.redis.app_data.url": "redis://redis:6379" "Settings.redis.sidekiq.url": "redis://redis:6379" POSTGRES_HOST: "${POSTGRES_HOST:-postgres}"