Merge pull request #3 from mlibrary/move-to-sftp-gem

Move to sftp gem

.env-example

@@ -0,0 +1 @@
+BUNDLE_RUBYGEMS__PKG__GITHUB__COM="Your Github Personal Access Token with read:packages access"

.github/workflows/build-main.yaml

@@ -27,15 +27,16 @@ jobs:
         context: .
         file: Dockerfile.prod
         push: true
+        secrets: |
+          "gh_package_read_token=${{ secrets.GH_PACKAGE_READ_TOKEN }}"
         tags: | 
            ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:latest
            ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
     - name: Deploy to testing
-      uses: mlibrary/deploy-to-kubernetes@v1
+      uses: mlibrary/deploy-to-kubernetes@v2
       with:
-        github_username: ${{ github.actor }}
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        image: mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
+        registry_token: ${{ secrets.GITHUB_TOKEN }}
+        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

.github/workflows/manual-deploy-production.yaml

@@ -60,11 +60,10 @@ jobs:
         push: true
         tags: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{steps.tag_check.outputs.tag}}
     - name: Deploy to ${{ env.environment }}
-      uses: mlibrary/deploy-to-kubernetes@v1
+      uses: mlibrary/deploy-to-kubernetes@v2
       with:
-        github_username: ${{ github.actor }}
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        image: mlibrary/${{ secrets.IMAGE_NAME }}:${{ steps.tag_check.outputs.tag }}
+        registry_token: ${{ secrets.GITHUB_TOKEN }}
+        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

.github/workflows/manual-deploy-testing.yaml

@@ -59,13 +59,14 @@ jobs:
         context: .
         file: Dockerfile.prod
         push: true
+        secrets: |
+          "gh_package_read_token=${{ secrets.GH_PACKAGE_READ_TOKEN }}"
         tags: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{steps.tag_check.outputs.tag}}
     - name: Deploy to ${{ env.environment }}
-      uses: mlibrary/deploy-to-kubernetes@v1
+      uses: mlibrary/deploy-to-kubernetes@v2
       with:
-        github_username: ${{ github.actor }}
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        image: mlibrary/${{ secrets.IMAGE_NAME }}:${{ steps.tag_check.outputs.tag }}
+        registry_token: ${{ secrets.GITHUB_TOKEN }}
+        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

.github/workflows/tag-release.yaml

@@ -33,11 +33,10 @@ jobs:
         docker push $PROD_IMAGE_ID:${{ env.TAG }}
         docker push $PROD_IMAGE_ID:latest
     - name: Deploy to production
-      uses: mlibrary/deploy-to-kubernetes@v1
+      uses: mlibrary/deploy-to-kubernetes@v2
       with:
-        github_username: ${{ github.actor }}
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        image: mlibrary/${{ secrets.IMAGE_NAME }}:${{ env.TAG }}
+        registry_token: ${{ secrets.GITHUB_TOKEN }}
+        image: ghcr.io/mlibrary/${{ secrets.IMAGE_NAME }}:${{ github.sha }}
         cluster_ca: ${{ secrets.HATCHER_CLUSTER_CA }}
         cluster_server: https://hatcher.kubernetes.lib.umich.edu
         namespace_token: ${{ secrets.HATCHER_TOKEN }}

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Create .env file
         run: cat .env-dev-values > .env
       - name: Load .env file
@@ -19,6 +19,8 @@ jobs:
         with:
           ruby-version: 3.1.0
           bundler-cache: true
+        env: 
+          BUNDLE_RUBYGEMS__PKG__GITHUB__COM: ${{secrets.GH_PACKAGE_READ_TOKEN}}
       - name: Run linter for Ruby
         run: bundle exec standardrb
       - name: Run tests

Dockerfile.prod

@@ -23,13 +23,16 @@ RUN mkdir -p /gems && chown ${UID}:${GID} /gems
 
 
 COPY --chown=${UID}:${GID} . /app
-USER $UNAME
 
 ENV BUNDLE_PATH /gems
 ENV BUNDLE_WITHOUT development:test
 
 WORKDIR /app
 
-RUN bundle _${BUNDLER_VERSION}_ install
+RUN --mount=type=secret,id=gh_package_read_token \
+  read_token="$(cat /run/secrets/gh_package_read_token)" \
+  && BUNDLE_RUBYGEMS__PKG__GITHUB__COM=${read_token} bundle _${BUNDLER_VERSION}_ install
+
+USER $UNAME
 
 CMD ["bundle", "exec", "ruby", "alma_webhook.rb", "-o", "0.0.0.0"]

Gemfile

@@ -5,6 +5,10 @@ gem "sinatra"
 gem "puma"
 gem "sidekiq"
 
+source "https://rubygems.pkg.github.com/mlibrary" do
+  gem "sftp", "0.1.0"
+end
+
 group :development, :test do
   gem "pry"
   gem "pry-byebug"

Gemfile.lock

@@ -98,6 +98,11 @@ GEM
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
 
+GEM
+  remote: https://rubygems.pkg.github.com/mlibrary/
+  specs:
+    sftp (0.1.0)
+
 PLATFORMS
   x86_64-linux
 
@@ -107,6 +112,7 @@ DEPENDENCIES
   puma
   rack-test
   rspec
+  sftp (= 0.1.0)!
   sidekiq
   simplecov
   sinatra

README.md

@@ -16,6 +16,13 @@ build web container
 docker-compose build web
 ```
 
+set up your .env file
+```
+cp .env-example .env
+```
+
+Replace the value for `BUNDLE_RUBYGEMS__PKG__GITHUB__COM` in the `.env` file with your [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) that has `read:packages` access. 
+
 bundle install gems to a docker volume
 ```
 docker-compose run --rm web bundle install

alma_webhook.rb

@@ -1,14 +1,20 @@
 require "sinatra"
 require "json"
 require "sidekiq"
+require "sftp"
 require "byebug" if settings.environment == :development
 
 require "./lib/message_validator"
-require "./lib/sftp"
 require "./lib/indexing_action"
 require "./lib/indexing_jobs_generator"
 require "./lib/message_router"
 
+SFTP.configure do |config|
+  config.user = ENV.fetch("ALMA_FILES_USER")
+  config.host = ENV.fetch("ALMA_FILES_HOST")
+  config.key_path = ENV.fetch("SSH_KEY_PATH")
+end
+
 get "/" do
   content_type :json
   {"challenge" => params["challenge"]}.to_json

docker-compose.yml

@@ -8,6 +8,7 @@ services:
     ports:
       - "4567:4567"
     env_file:
+      - .env
       - .env-dev-values
     volumes:
       - .:/app

lib/indexing_jobs_generator.rb

@@ -6,7 +6,7 @@ def self.match?(data)
       data["job_instance"]["status"]["value"] == "COMPLETED_SUCCESS"
   end
 
-  def initialize(data: nil, job_id: nil, sftp: SFTP.new, logger: Logger.new($stdout),
+  def initialize(data: nil, job_id: nil, sftp: SFTP.client, logger: Logger.new($stdout),
     push_indexing_jobs: lambda do |job_name:, files:, solr_url:|
                           Sidekiq::Client.push_bulk("class" => job_name, "args" => files.map { |x| [x, solr_url] })
                         end)

spec/lib/indexing_jobs_generator_spec.rb

@@ -10,7 +10,7 @@
       "file_#{@job_id_from_data}_delete_1.tar.gz",
       "file_new.tar.gz"
     ]
-    @sftp_double = instance_double(SFTP, ls: @files)
+    @sftp_double = instance_double(SFTP::Client, ls: @files)
     @logger_double = instance_double(Logger, info: nil)
     @push_bulk_double = double("SidekiqClient", push_bulk: nil)
     @push_indexing_jobs = lambda do |job_name:, files:, solr_url:|
@@ -81,7 +81,7 @@
       "file_#{@job_id_from_data}_delete.tar.gz",
       "file_new.tar.gz"
     ]
-    @sftp_double = instance_double(SFTP, ls: @files)
+    @sftp_double = instance_double(SFTP::Client, ls: @files)
     @logger_double = instance_double(Logger, info: nil)
     @push_bulk_double = double("SidekiqClient", push_bulk: nil)
     @push_indexing_jobs = lambda do |job_name:, files:, solr_url:|