diff --git a/.github/workflows/sqs.yml b/.github/workflows/sqs.yml new file mode 100644 index 0000000000..74b4693e05 --- /dev/null +++ b/.github/workflows/sqs.yml @@ -0,0 +1,29 @@ +--- +name: refresh +on: + schedule: + - cron: "0 0 * * *" + push: + branches: + - main +permissions: + contents: write +jobs: + refresh: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.3.0 + bundler-cache: true + - name: Commit user.name and email + run: | + git config --global user.email git@intermittent.energy + git config --global user.name intermittent.energy + - name: receive new files from SQS + env: + AWS_REGION: sa-east-1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: ./receive.rb diff --git a/.github/workspaces/sqs.yml b/.github/workspaces/sqs.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000000..0c663bb58d --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +gem 'aws-sdk-sqs' +gem 'ox' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000000..987d51751e --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,28 @@ +GEM + remote: https://rubygems.org/ + specs: + aws-eventstream (1.3.0) + aws-partitions (1.926.0) + aws-sdk-core (3.194.2) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.651.0) + aws-sigv4 (~> 1.8) + jmespath (~> 1, >= 1.6.1) + aws-sdk-sqs (1.73.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) + aws-sigv4 (1.8.0) + aws-eventstream (~> 1, >= 1.0.2) + jmespath (1.6.2) + ox (2.14.18) + +PLATFORMS + ruby + x86_64-linux + +DEPENDENCIES + aws-sdk-sqs + ox + +BUNDLED WITH + 2.5.6 diff --git a/receive.rb b/receive.rb new file mode 100755 index 0000000000..0cf1e4a47e --- /dev/null +++ b/receive.rb @@ -0,0 +1,57 @@ +#!/usr/bin/env ruby +require 'bundler/setup' +require 'aws-sdk-sqs' +require 'json' + +queue_url = "https://sqs.sa-east-1.amazonaws.com/115154022797/ons-sns-sqs-github" +sqs = Aws::SQS::Client.new(region: 'sa-east-1') +paths = [] +receipt_handles = [] + +loop do + receive_message_result = sqs.receive_message({ + queue_url: queue_url, + #message_attribute_names: ["All"], # Receive all custom attributes. + max_number_of_messages: 10, + wait_time_seconds: 0 # Do not wait to check for the message. + }) + receive_message_result.messages.each do |message| + #require 'pry' ; binding.pry + body = message.body + json = JSON.parse(body) + if json['Type'] == 'Notification' + # handle SNS wrapping + body = json['Message'] + json = JSON.parse(body) + end + date = json['Data'] + dir, file = date.split(/T/) + path = "#{dir}/#{file}.json" + Dir.mkdir dir rescue nil + File.write(path, message.body) + paths << path + receipt_handles << message.receipt_handle + puts path + end + break if receive_message_result.messages.length <10 +end + +raise 'no files' if paths.empty? + +system "git", "add", "-v", *paths +system "git", "commit", "-m", "Updated data from ONS" +system "git", "push" + +i=0 +receipt_handles.each_slice(10) do |receipt_handles_batch| + sqs.delete_message_batch({ + queue_url: queue_url, + entries: receipt_handles_batch.map do |receipt_handle| + { + id: (i += 1).to_s, + receipt_handle: + } + end + }) + puts "delete" +end