ci/cd work

.circleci/config.yml

@@ -2653,7 +2653,7 @@ workflows:
             - integration_tests_mtls
           filters:
             branches:
-              only: main
+              only: nolonger_main
 
       - deploy_stg_migrations:
           requires:

.envrc

@@ -415,7 +415,7 @@ if [ ! -r .nix-disable  ] && has nix-env; then
 
   # add the NIX_PROFILE bin path so that everything we just installed
   # is available on the path
-  PATH_add ${NIX_PROFILE}/bin
+  PATH_add "${NIX_PROFILE}"/bin
   # Add the node binaries to our path
   PATH_add ./node_modules/.bin
   # nix is immutable, so we need to specify a path for local changes, e.g.

.gitlab-ci.yml

@@ -40,10 +40,10 @@ variables:
   SERVER_IGNORE_BRANCH: &server_ignore_branch placeholder_branch_name
 
   RUNNER_TAG: &runner_tag milmove
+
   DOCKER_RUNNER_TAG: &docker_runner_tag eks_cluster_runner
 
-  postgres: &postgres postgres:16.4
-  #postgres: &postgres postgres:16.4
+  postgres: &postgres postgres:16
   redis: &redis redis:5.0.6
 
 stages:
@@ -128,18 +128,18 @@ stages:
 .setup_release_stg: &setup_release_stg
   #if main
   - export ECR_REPOSITORY_URI=${STG_ACCOUNT_ID}.dkr.ecr.${STG_REGION}.amazonaws.com
-  - export APP_DOCKER_FILE=Dockerfile.dp3
-  - export TASK_DOCKER_FILE=Dockerfile.tasks_dp3
+  - export APP_DOCKER_FILE=Dockerfile
+  - export TASK_DOCKER_FILE=Dockerfile.tasks
   #TODO: update demo to stg
-  - export APP_ENVIRONMENT=demo
+  - export APP_ENVIRONMENT=stg
 
 .setup_release_prd: &setup_release_prd
   #build off prd variables
   - export ECR_REPOSITORY_URI=${PRD_ACCOUNT_ID}.dkr.ecr.${PRD_REGION}.amazonaws.com
-  - export APP_DOCKER_FILE=Dockerfile.dp3
-  - export TASK_DOCKER_FILE=Dockerfile.tasks_dp3
+  - export APP_DOCKER_FILE=Dockerfile
+  - export TASK_DOCKER_FILE=Dockerfile
   #TODO: update exp to prod
-  - export APP_ENVIRONMENT=exp
+  - export APP_ENVIRONMENT=prd
 
 .kaniko_before_setup: &kaniko_before_setup
   # prep login for kaniko
@@ -148,7 +148,7 @@ stages:
   echo "{\"credHelpers\":{\"${ECR_REPOSITORY_URI}\":\"ecr-login\"}}" > /kaniko/.docker/config.json
 
 .check_dp3: &check_dp3
-  - if: $DP3_ENV == "exp" || $DP3_ENV == "loadtest" || $DP3_ENV == "demo"
+ - if: (($DP3_ENV == "exp" || $DP3_ENV == "loadtest" || $DP3_ENV == "demo") && $DP3_BRANCH == $CI_COMMIT_BRANCH)
 
 .check_main: &check_main
   - if: '$CI_COMMIT_BRANCH == "main"'
@@ -173,6 +173,7 @@ stages:
   - |
     mkdir -p /builds/milmove/mymove/.cache
     mkdir -p /builds/milmove/mymove/.cache/yarn
+    yarn config set network-timeout 600000 -g
     yarn install --frozen-lockfile --cache-folder /builds/milmove/mymove/.cache/yarn
     scripts/check-generated-code yarn.lock
     echo "yarn check dependencies"
@@ -217,29 +218,38 @@ stages:
 
 .setup_server_env_variables: &setup_server_env_variables
     - |
-      echo "make server_test_build for app"
-      export LOGIN_GOV_SECRET_KEY=$(echo $E2E_LOGIN_GOV_SECRET_KEY | base64 --decode)
-      export OKTA_CUST_CLIENT_ID=notrealkey
-      export OKTA_CUSTOMER_SECRET_KEY=notrealkey
-      export OKTA_OFFICE_SECRET_KEY=notrealkey1
-      export OKTA_ADMIN_SECRET_KEY=notrealkey2
-      export OKTA_TENANT_ORG_URL=test-milmove.okta.mil
-      export GOTEST_PARALLEL=8
-      export DB_PORT_TEST=5433
+      export MOVE_MIL_DOD_CA_CERT=$(cat config/tls/devlocal-ca.pem)
+      export MOVE_MIL_DOD_TLS_CERT=$(cat config/tls/devlocal-https.pem)
+      export MOVE_MIL_DOD_TLS_KEY=$(cat config/tls/devlocal-https.key)
       export DB_NAME=test_db
-      export DB_NAME_TEST=test_db
-      export DTOD_USE_MOCK='true'
-      export ENV=test
+      export DB_NAME_DEV=test_db
       export ENVIRONMENT=test
-      export SERVER_REPORT=1
-      export COVERAGE=1
-      export SERVE_API_INTERNAL='true'
-      export OKTA_CUSTOMER_CLIENT_ID=1q2w3e4r5t6y7u8i9o
-      export OKTA_ADMIN_CLIENT_ID=AQ1SW2DE3FR4G5
-      export OKTA_OFFICE_CLIENT_ID=9f9f9s8s90gig9
-      export OKTA_API_KEY=notrealapikey8675309
-      export OKTA_OFFICE_GROUP_ID=notrealgroupId
-      export OKTA_CUSTOMER_GROUP_ID=notrealcustomergroupId
+      export DOD_CA_PACKAGE=/builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
+      export GITLAB=1
+    # echo "make server_test_build for app"
+
+    # export LOGIN_GOV_SECRET_KEY=$(echo $E2E_LOGIN_GOV_SECRET_KEY | base64 --decode)
+    # export OKTA_CUST_CLIENT_ID=notrealkey
+    # export OKTA_CUSTOMER_SECRET_KEY=notrealkey
+    # export OKTA_OFFICE_SECRET_KEY=notrealkey1
+    # export OKTA_ADMIN_SECRET_KEY=notrealkey2
+    # export OKTA_TENANT_ORG_URL=test-milmove.okta.mil
+    # export GOTEST_PARALLEL=8
+    # export DB_PORT_TEST=5433
+    # export DB_NAME=test_db
+    # export DB_NAME_TEST=test_db
+    # export DTOD_USE_MOCK='true'
+    # export ENV=test
+    # export ENVIRONMENT=test
+    # export SERVER_REPORT=1
+    # export COVERAGE=1
+    # export SERVE_API_INTERNAL='true'
+    # export OKTA_CUSTOMER_CLIENT_ID=1q2w3e4r5t6y7u8i9o
+    # export OKTA_ADMIN_CLIENT_ID=AQ1SW2DE3FR4G5
+    # export OKTA_OFFICE_CLIENT_ID=9f9f9s8s90gig9
+    # export OKTA_API_KEY=notrealapikey8675309
+    # export OKTA_OFFICE_GROUP_ID=notrealgroupId
+    # export OKTA_CUSTOMER_GROUP_ID=notrealcustomergroupId
 
 # .setup_host_intergration_tests: &setup_host_intergration_tests
 #    - echo "Setting up /etc/hosts for local domain simulation"
@@ -368,10 +378,6 @@ pre_deps_golang:
     - *announce_failure
   cache:
     - <<: *go_cache
-  artifacts:
-    paths:
-      - /builds/milmove/mymove/bin/
-      - /builds/milmove/mymove/swagger/
   #TODO: Optimization potential
   # cache:
   #   key: "$CI_COMMIT_REF_SLUG-go"
@@ -618,6 +624,7 @@ pre_test:
       [ -d ~/transcom/mymove/spectral ] && cp -r ~/transcom/mymove/spectral /tmp/spectral_baseline || echo "Skipping saving baseline"
     - rm -rf ~/transcom/mymove/spectral
     - *install_yarn
+    - ./scripts/pre-commit-go-mod || exit 0
     - echo "Run pre-commit tests without golangci-lint, eslint, or prettier"
     - SKIP=golangci-lint,eslint,prettier,ato-go-linter,gomod,appcontext-linter pre-commit run --all-files
     - |
@@ -642,7 +649,7 @@ pre_test:
     - yarn danger ci --failOnErrors
     -  echo "Run spectral linter on all files"
     - ./scripts/ensure-spectral-lint /tmp/spectral_baseline spectral
-    - ./scripts/pre-commit-go-mod || exit 0
+
   allow_failure: true
   after_script:
     - *announce_failure
@@ -658,7 +665,6 @@ server_test:
     - pre_deps_golang
   before_script:
   - *setup_milmove_env
-  - *setup_generic_app_env_variables
   - *setup_server_env_variables
   services:
     - name: docker:dind
@@ -669,6 +675,7 @@ server_test:
     DOCKER_HOST: "tcp://docker-backend.gitlab-runner.svc.cluster.local:2375"
     DOCKER_TLS_CERTDIR: ""
     APPLICATION: app
+    GITLAB: 1
     # 8 since this runs on xlarge with 8 CPUs
     GOTEST_PARALLEL: 8
     DB_PASSWORD: mysecretpassword
@@ -696,24 +703,26 @@ server_test:
     OKTA_OFFICE_GROUP_ID: notrealgroupId
     OKTA_CUSTOMER_GROUP_ID: notrealcustomergroupId
   script:
-    - psql --version
-    - for i in $(seq 1 5); do go mod download && break || s=$? && sleep 5; done; (exit $s)
-    - scripts/check-generated-code go.sum
-    - make bin/swagger
     - echo "server test -- TODO Add steps need to potentially pass job id to file and persist"
+    - echo $GITLAB
     - make -j 2 bin/milmove bin/gotestsum
-    - make server_test for app
-    # - go install gotest.tools/gotestsum@latest
-    # - go mod tidy
-    #- bin/gotestsum --junitfile server_test_report.xml --format server_test
+    - make server_test
+    - go mod tidy
+    - bin/gotestsum --junitfile server_test_report.xml --format server_test
   allow_failure: true
+  cache:
+    - <<: *go_cache
+      policy: pull
   artifacts:
     paths:
       - /builds/milmove/mymove/bin/gotestsum
       - /builds/milmove/mymove/tmp/test-results
+      - /builds/milmove/mymove/server_test_report.xml
     when: always
     reports:
-      junit: /builds/milmove/mymove/tmp/test-results/gotest/app/go-test-report.xml
+      junit:
+        - /builds/milmove/mymove/tmp/test-results/gotest/app/go-test-report.xml
+        - /builds/milmove/mymove/server_test_report.xml
   after_script:
     - *announce_failure
   rules:
@@ -771,13 +780,14 @@ client_test:
     - pre_deps_yarn
   script:
     - echo "client test coverage"
-    - JEST_JUNIT_OUTPUT_DIR=jest-junit-reports yarn test:coverage -results=false >> $CI_PROJECT_DIR/coverage.output
+    - JEST_JUNIT_OUTPUT_DIR=jest-junit-reports yarn test:coverage -results=false  >> $CI_PROJECT_DIR/coverage.output
   artifacts:
     when: always
     reports:
       junit:
         - jest-junit-reports/junit.xml
     paths:
+      - /builds/milmove/mymove/coverage.output
       - /builds/milmove/mymove/coverage
       - /builds/milmove/mymove/jest-junit-reports
   after_script:
@@ -861,20 +871,15 @@ integration_tests:
   stage: test
   tags:
     - $RUNNER_TAG
-  image: $DOCKER_APP_IMAGE
   needs:
-    - pre_deps_yarn
-    - pre_deps_golang
-    - compile_app_client
-    - compile_app_server
     - integration_test_my
     - integration_test_office
     - integration_test_admin
     - integration_test_devseed
-  before_script: *setup_milmove_env
+    - integration_test_mtls
   script:
-    - echo "TODO Add steps"
-    - echo "integration_tests"
+    #no-op to lump integration tests into one status
+    - echo "integration_tests done"
   allow_failure: true
   after_script:
     - *announce_failure
@@ -1192,17 +1197,20 @@ push_otel_collector_image_dp3:
     - compile_app_server
     - compile_app_client
   script:
-    - echo "Logging in to Amazon ECR with Crane..."
-    - aws ecr get-login-password --region us-gov-west-1 | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin
+     - echo "Logging in to DockerHub with Crane..."
+     - echo "${DOCKERHUB_PASSWORD}" | crane auth login docker.io -u "${DOCKERHUB_USERNAME}" --password-stdin
 
-    - echo "Pulling the AWS OTel Collector image from the public registry with Crane..."
-    - crane pull --insecure public.ecr.aws/aws-observability/aws-otel-collector:v0.31.0 image.tar
+     - echo "Pulling the AWS OTel Collector image from Dockerhub with Crane..."
+     - crane pull $ECR_REPOSITORY_OTEL:$OTEL_VERSION image.tar
 
-    - echo "Pushing the image to our private ECR using Crane..."
-    - crane push --insecure image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA}
+     - echo "Logging in to AWS ECR with Crane..."
+     - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin
 
-    - echo "Cleaning up the temporary image file..."
-    - rm image.tar
+     - echo "Pushing the image to our private ECR using Crane..."
+     - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA}
+
+     - echo "Cleaning up the temporary image file..."
+     - rm image.tar
   allow_failure: false
   after_script:
     - *announce_failure
@@ -1447,17 +1455,20 @@ push_otel_collector_image_stg:
     - *setup_aws_vars_stg
     - *setup_release_stg
   script:
-    - echo "Logging in to Amazon ECR with Crane..."
-    - aws ecr get-login-password --region us-gov-west-1 | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin
+     - echo "Logging in to DockerHub with Crane..."
+     - echo "${DOCKERHUB_PASSWORD}" | crane auth login docker.io -u "${DOCKERHUB_USERNAME}" --password-stdin
+
+     - echo "Pulling the AWS OTel Collector image from Dockerhub with Crane..."
+     - crane pull $ECR_REPOSITORY_OTEL:$OTEL_VERSION image.tar
 
-    - echo "Pulling the AWS OTel Collector image from the public registry with Crane..."
-    - crane pull --insecure public.ecr.aws/aws-observability/aws-otel-collector:v0.31.0 image.tar
+     - echo "Logging in to AWS ECR with Crane..."
+     - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin
 
-    - echo "Pushing the image to our private ECR using Crane..."
-    - crane push --insecure image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA}
+     - echo "Pushing the image to our private ECR using Crane..."
+     - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA}
 
-    - echo "Cleaning up the temporary image file..."
-    - rm image.tar
+     - echo "Cleaning up the temporary image file..."
+     - rm image.tar
   allow_failure: false
   after_script:
     - *announce_failure
@@ -1720,17 +1731,20 @@ push_otel_collector_image_prd:
     - *setup_aws_vars_prd
     - *setup_release_prd
   script:
-    - echo "Logging in to Amazon ECR with Crane..."
-    - aws ecr get-login-password --region us-gov-west-1 | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin
+     - echo "Logging in to DockerHub with Crane..."
+     - echo "${DOCKERHUB_PASSWORD}" | crane auth login docker.io -u "${DOCKERHUB_USERNAME}" --password-stdin
+
+     - echo "Pulling the AWS OTel Collector image from Dockerhub with Crane..."
+     - crane pull $ECR_REPOSITORY_OTEL:$OTEL_VERSION image.tar
 
-    - echo "Pulling the AWS OTel Collector image from the public registry with Crane..."
-    - crane pull --insecure public.ecr.aws/aws-observability/aws-otel-collector:v0.31.0 image.tar
+     - echo "Logging in to AWS ECR with Crane..."
+     - aws ecr get-login-password --region $AWS_DEFAULT_REGION | crane auth login ${ECR_REPOSITORY_URI} -u AWS --password-stdin
 
-    - echo "Pushing the image to our private ECR using Crane..."
-    - crane push --insecure image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA}
+     - echo "Pushing the image to our private ECR using Crane..."
+     - crane push image.tar ${ECR_REPOSITORY_URI}/otel-collector:${CI_COMMIT_SHORT_SHA}
 
-    - echo "Cleaning up the temporary image file..."
-    - rm image.tar
+     - echo "Cleaning up the temporary image file..."
+     - rm image.tar
   allow_failure: false
   after_script:
     - *announce_failure

.pre-commit-config.yaml

@@ -160,7 +160,6 @@ repos:
     hooks:
       - id: gen-docs
         args: ['docs/adr']
-      - id: circleci-validate
       - id: markdown-toc
       - id: hadolint