Add GH refresh workflow

.github/workflows/sqs.yml

@@ -0,0 +1,29 @@
+---
+name: refresh
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  push:
+    branches:
+      - main
+permissions:
+  contents: write
+jobs:
+  refresh:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.3.0
+          bundler-cache: true
+      - name: Commit user.name and email
+        run: |
+          git config --global user.email [email protected]
+          git config --global user.name intermittent.energy
+      - name: receive new files from SQS
+        env:
+          AWS_REGION: sa-east-1
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: ./receive.rb

Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem 'aws-sdk-sqs'
+gem 'ox'

Gemfile.lock

@@ -0,0 +1,28 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.3.0)
+    aws-partitions (1.926.0)
+    aws-sdk-core (3.194.2)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.8)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-sqs (1.73.0)
+      aws-sdk-core (~> 3, >= 3.193.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.8.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+    jmespath (1.6.2)
+    ox (2.14.18)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  aws-sdk-sqs
+  ox
+
+BUNDLED WITH
+   2.5.6

receive.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'aws-sdk-sqs'
+require 'json'
+
+queue_url = "https://sqs.sa-east-1.amazonaws.com/115154022797/ons-sns-sqs-github"
+sqs = Aws::SQS::Client.new(region: 'sa-east-1')
+paths = []
+receipt_handles = []
+
+loop do
+  receive_message_result = sqs.receive_message({
+    queue_url: queue_url,
+    #message_attribute_names: ["All"], # Receive all custom attributes.
+    max_number_of_messages: 10,
+    wait_time_seconds: 0 # Do not wait to check for the message.
+  })
+  receive_message_result.messages.each do |message|
+    #require 'pry' ; binding.pry
+    body = message.body
+    json = JSON.parse(body)
+    if json['Type'] == 'Notification'
+      # handle SNS wrapping
+      body = json['Message']
+      json = JSON.parse(body)
+    end
+    date = json['Data']
+    dir, file = date.split(/T/)
+    path = "#{dir}/#{file}.json"
+    Dir.mkdir dir rescue nil
+    File.write(path, message.body)
+    paths << path
+    receipt_handles << message.receipt_handle
+    puts path
+  end
+  break if receive_message_result.messages.length <10
+end
+
+raise 'no files' if paths.empty?
+
+system "git", "add", "-v", *paths
+system "git", "commit", "-m", "Updated data from ONS"
+system "git", "push"
+
+i=0
+receipt_handles.each_slice(10) do |receipt_handles_batch|
+  sqs.delete_message_batch({
+    queue_url: queue_url,
+    entries: receipt_handles_batch.map do |receipt_handle|
+      {
+        id: (i += 1).to_s,
+        receipt_handle:
+      }
+    end
+  })
+  puts "delete"
+end