Merge pull request #6 from mlibrary/faraday-and-solrcloud

Faraday and solrcloud
mlibrary · May 30, 2023 · 7e705fd · 7e705fd
2 parents a2235f4 + efa5cfa
commit 7e705fd
Show file tree

Hide file tree

Showing 19 changed files with 220 additions and 287 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1 +1,3 @@
 .gems
+sftp/
+tmp/
diff --git a/.env-dev-values b/.env-dev-values
@@ -12,3 +12,7 @@ ALMA_FILES_HOST="sftp"
 SSH_KEY_PATH="/etc/secret-volume/id_rsa"
 REINDEX_ON="true"
 DAILY_INDEX_ON="true"
+SIDEKIQ_SUPERVISOR_HOST="http://monitor:1080"
+SOLRCLOUD_ON="false"
+SUPERVISOR_ON="true"
+LIVE_SOLR_URL=""
diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml
@@ -7,36 +7,25 @@ on:
     types: [ completed ]
 
 jobs:
-  build:
+  build-unstable:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/build-unstable.yml@v1
+    with:
+      image_name: ${{ vars.IMAGE_NAME }}
+      tag: ${{ github.sha }}
+      dockerfile: Dockerfile
+    secrets: inherit
+
+  deploy-testing:
+    needs: build-unstable
     runs-on: ubuntu-latest
     environment: testing
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Log into Github Container Registry
-      uses: docker/login-action@v1
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Build and Push
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: Dockerfile.prod
-        push: true
-        secrets: |
-          "gh_package_read_token=${{ secrets.GH_PACKAGE_READ_TOKEN }}"
-        tags: | 
-           ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:latest
-           ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
-    - name: Deploy to testing
-      uses: mlibrary/deploy-to-kubernetes@v2
+    - name: Deploy to Testing
+      uses: mlibrary/deploy-to-kubernetes@v3
       with:
         registry_token: ${{ secrets.GITHUB_TOKEN }}
-        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
+        image: ${{ needs.build-unstable.outputs.image }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

diff --git a/.github/workflows/manual-deploy-production.yaml b/.github/workflows/manual-deploy-production.yaml
@@ -7,63 +7,26 @@ on:
         description: tag
         required: true
 
-env:
-  environment: production
-
 jobs:
-  deploy:
+  build-production:
+    name: Build production ${{ github.event.inputs.tag }}
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/build-production.yml@v1
+    with:
+      image_name: ${{ vars.IMAGE_NAME }}
+      tag: ${{ github.event.inputs.tag }}
+      dockerfile: Dockerfile
+    secrets: inherit
+
+  deploy-production:
+    needs: build-production
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - name: Clone latest repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Check that the tag exists in repo
-      id: tag_check
-      run: |
-        if git rev-parse 'refs/tags/${{ github.event.inputs.tag }}' &> /dev/null; then
-          echo '::set-output name=tag::${{ github.event.inputs.tag }}'
-        elif git rev-parse '${{ github.event.inputs.tag }}' &> /dev/null; then
-          echo "::set-output name=tag::`git rev-parse '${{ github.event.inputs.tag }}'`"
-        else
-          echo "Couldn't figure out tag from input: ${{ github.event.inputs.tag }}"
-          echo "Aborting deployment."
-          false
-        fi
-    - name: Log into Github Container Registry
-      uses: docker/login-action@v1
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Check that the tag exists in container registry
-      id: image_check
-      run: |
-        if docker manifest inspect ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ steps.tag_check.outputs.tag }} > /dev/null; then
-          echo '::set-output name=image_exists::true'
-          echo "image exists!"
-        else
-          echo "image doesn't exist; Starting to Build and push image"
-        fi
-    - name: Checkout Correct repository
-      if: ${{ steps.image_check.outputs.image_exists != 'true' }}
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.inputs.tag }}
-    - name: Build and Push
-      if: ${{ steps.image_check.outputs.image_exists != 'true' }}
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: Dockerfile.prod
-        push: true
-        tags: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{steps.tag_check.outputs.tag}}
-    - name: Deploy to ${{ env.environment }}
-      uses: mlibrary/deploy-to-kubernetes@v2
+    - name: Deploy to Production
+      uses: mlibrary/deploy-to-kubernetes@v3
       with:
         registry_token: ${{ secrets.GITHUB_TOKEN }}
-        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ steps.tag_check.outputs.tag }}
+        image: ghcr.io/mlibrary/${{ vars.IMAGE_NAME }}:${{ github.event.inputs.tag }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

diff --git a/.github/workflows/manual-deploy-testing.yaml b/.github/workflows/manual-deploy-testing.yaml
@@ -8,67 +8,27 @@ on:
         required: true
 
 
-env:
-  environment: testing
-
 jobs:
-  deploy:
-    name: "Deploy ${{ github.event.inputs.tag }} to testing"
+  build-unstable:
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/build-unstable.yml@v1
+    with:
+      image_name: ${{ vars.IMAGE_NAME }}
+      tag: ${{ github.event.inputs.tag }}
+      dockerfile: Dockerfile
+    secrets: inherit
+
+  deploy-testing:
+    needs: build-unstable
     runs-on: ubuntu-latest
     environment: testing
     steps:
-    - name: Clone latest repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Check that the tag exists in repo
-      id: tag_check
-      run: |
-        if git rev-parse 'refs/tags/${{ github.event.inputs.tag }}' &> /dev/null; then
-          echo '::set-output name=tag::${{ github.event.inputs.tag }}'
-        elif git rev-parse '${{ github.event.inputs.tag }}' &> /dev/null; then
-          echo "::set-output name=tag::`git rev-parse '${{ github.event.inputs.tag }}'`"
-        else
-          echo "Couldn't figure out tag from input: ${{ github.event.inputs.tag }}"
-          echo "Aborting deployment."
-          false
-        fi
-    - name: Log into Github Container Registry
-      uses: docker/login-action@v1
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Check that the tag exists in container registry
-      id: image_check
-      run: |
-        if docker manifest inspect ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ steps.tag_check.outputs.tag }} > /dev/null; then
-          echo '::set-output name=image_exists::true'
-          echo "image exists!"
-        else
-          echo "image doesn't exist; Starting to Build and push image"
-        fi
-    - name: Checkout Correct repository
-      if: ${{ steps.image_check.outputs.image_exists != 'true' }}
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.inputs.tag }}
-    - name: Build and Push
-      if: ${{ steps.image_check.outputs.image_exists != 'true' }}
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: Dockerfile.prod
-        push: true
-        secrets: |
-          "gh_package_read_token=${{ secrets.GH_PACKAGE_READ_TOKEN }}"
-        tags: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{steps.tag_check.outputs.tag}}
-    - name: Deploy to ${{ env.environment }}
-      uses: mlibrary/deploy-to-kubernetes@v2
+    - name: Deploy to Testing
+      uses: mlibrary/deploy-to-kubernetes@v3
       with:
         registry_token: ${{ secrets.GITHUB_TOKEN }}
-        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ steps.tag_check.outputs.tag }}
+        image: ${{ needs.build-unstable.outputs.image }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}
         namespace: ${{ secrets.NAMESPACE }}
+
diff --git a/.github/workflows/tag-release.yaml b/.github/workflows/tag-release.yaml
@@ -1,42 +1,29 @@
-name: Tag Latest Release Images and Deploy to Production
+name: Build and Deploy on Release
 
 on:
   release:
     types: [ released ]
 
-env:
-  TAG: ${{ github.event.release.tag_name }}
-
 jobs:
-  build:
+  build-production:
+    name: Build production ${{ github.event.release.tag_name }}
+    uses: mlibrary/platform-engineering-workflows/.github/workflows/build-production.yml@v1
+    with:
+      image_name: ${{ vars.IMAGE_NAME }}
+      tag: ${{ github.event.release.tag_name }}
+      dockerfile: Dockerfile
+    secrets: inherit
+
+  deploy-production:
+    needs: build-production
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
-    - name: Log into Github Container Registry
-      uses: docker/login-action@v1
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Clone latest repository
-      uses: actions/checkout@v2
-    - name: Tag latest release in Github Container Registry
-      env: 
-        UNSTABLE_IMAGE_ID: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}-unstable
-        PROD_IMAGE_ID: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}
-      run: |
-        docker pull $UNSTABLE_IMAGE_ID:${{ github.sha }}
-        docker tag $UNSTABLE_IMAGE_ID:${{ github.sha }} $PROD_IMAGE_ID:${{ env.TAG }}
-        docker tag $UNSTABLE_IMAGE_ID:${{ github.sha }} $PROD_IMAGE_ID:latest
-        docker push $PROD_IMAGE_ID:${{ env.TAG }}
-        docker push $PROD_IMAGE_ID:latest
-    - name: Deploy to production
-      uses: mlibrary/deploy-to-kubernetes@v2
+    - name: Deploy to Production
+      uses: mlibrary/deploy-to-kubernetes@v3
       with:
         registry_token: ${{ secrets.GITHUB_TOKEN }}
-        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ env.TAG }}
+        image: ghcr.io/mlibrary/${{ vars.IMAGE_NAME }}:${{ github.event.release.tag_name }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -13,11 +13,11 @@ jobs:
       - name: Create .env file
         run: cat .env-dev-values > .env
       - name: Load .env file
-        uses: xom9ikk/dotenv@v1.0.2
-      - name: Set up Ruby 3.1.0
+        uses: xom9ikk/dotenv@v2
+      - name: Set up Ruby 3.2
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1.0
+          ruby-version: 3.2
           bundler-cache: true
         env: 
           BUNDLE_RUBYGEMS__PKG__GITHUB__COM: ${{secrets.GH_PACKAGE_READ_TOKEN}}

diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,13 @@
-ARG RUBY_VERSION=3.1
-FROM ruby:${RUBY_VERSION}
+FROM ruby:3.2 AS development
 
-ARG BUNDLER_VERSION=2.3
 ARG UNAME=app
 ARG UID=1000
 ARG GID=1000
 
-LABEL maintainer="[email protected]"
+#RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
+  #vim-tiny 
 
-RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \
-  apt-transport-https \
-  vim-tiny \
-  ssh
-
-RUN gem install bundler:${BUNDLER_VERSION}
+RUN gem install bundler
 
 RUN groupadd -g ${GID} -o ${UNAME}
 RUN useradd -m -d /app -u ${UID} -g ${GID} -o -s /bin/bash ${UNAME}
@@ -27,3 +21,13 @@ ENV BUNDLE_PATH /gems
 WORKDIR /app
 
 CMD ["bundle", "exec", "ruby", "alma_webhook.rb", "-o", "0.0.0.0"]
+
+FROM development AS production
+
+ENV BUNDLE_WITHOUT development:test
+
+COPY --chown=${UID}:${GID} . /app
+
+RUN --mount=type=secret,id=gh_package_read_token,uid=1000 \
+  read_token="$(cat /run/secrets/gh_package_read_token)" \
+  && BUNDLE_RUBYGEMS__PKG__GITHUB__COM=${read_token} bundle install
diff --git a/Dockerfile.prod b/Dockerfile.prod
diff --git a/Gemfile b/Gemfile
@@ -4,6 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 gem "sinatra"
 gem "puma"
 gem "sidekiq"
+gem "faraday"
 
 source "https://rubygems.pkg.github.com/mlibrary" do
   gem "sftp"
@@ -18,4 +19,5 @@ group :development, :test do
   gem "webmock"
   gem "simplecov"
   gem "standard"
+  gem "climate_control"
 end
-Original file line number
+Diff line change
@@ -1 +1,3 @@
     .gems
+    sftp/
+    tmp/