Skip to content

Commit

Permalink
Merge pull request #15735 from CDCgov/dkrylov/azure_resource_costs
Browse files Browse the repository at this point in the history 
Dkrylov/azure resource costs
  • Loading branch information
@dkrylovsb
dkrylovsb authored Sep 3, 2024
2 parents d6391f8 + ef28841 commit 64f0143
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 14 deletions.
14 changes: 12 additions & 2 deletions .github/actions/az-cost/action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ runs:
- name: Install azure-cost-cli
shell: bash
run: |
dotnet new tool-manifest
dotnet new tool-manifest --force
dotnet tool install azure-cost-cli
- name: Run azure-cost-cli for daily totals
Expand All @@ -38,6 +38,16 @@ runs:
dotnet azure-cost accumulatedCost -g ${{ inputs.rg }} -t custom --from $(date --date='${{ inputs.days_ago }} days ago' '+%m/%d/%Y') --to $(date --date='1 days ago' '+%m/%d/%Y') -o csv >> $GITHUB_OUTPUT
echo "$EOF" >> $GITHUB_OUTPUT
- name: Run azure-cost-cli for daily totals by resource
if: inputs.total_format == 'dailyByResource'
id: run_azure_cost_daily_by_resource
shell: bash
run: |
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "RESULT<<$EOF" >> $GITHUB_OUTPUT
dotnet azure-cost dailyCosts -g ${{ inputs.rg }} -t custom --from $(date --date='${{ inputs.days_ago }} days ago' '+%m/%d/%Y') --to $(date --date='1 days ago' '+%m/%d/%Y') -o csv --exclude-meter-details --dimension ConsumedService >> $GITHUB_OUTPUT
echo "$EOF" >> $GITHUB_OUTPUT
- name: Run azure-cost-cli for total
if: inputs.total_format == 'summary'
id: run_azure_cost_summary
Expand All @@ -52,5 +62,5 @@ runs:
run: |
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "RESULT<<$EOF" >> $GITHUB_OUTPUT
echo -e "${{ steps.run_azure_cost_daily.outputs.RESULT }}${{ steps.run_azure_cost_summary.outputs.RESULT }}" >> $GITHUB_OUTPUT
echo -e "${{ steps.run_azure_cost_daily.outputs.RESULT }}${{ steps.run_azure_cost_daily_by_resource.outputs.RESULT }}${{ steps.run_azure_cost_summary.outputs.RESULT }}" >> $GITHUB_OUTPUT
echo "$EOF" >> $GITHUB_OUTPUT
60 changes: 48 additions & 12 deletions .github/workflows/export_cost_data.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,57 @@ on:
schedule:
# The workflow runs every day at 3PM
- cron: "0 19 * * *" #UTC-5

jobs:
export_cost_data:
name: Export Azure Cost Data to Storage
strategy:
fail-fast: false
matrix:
include:
- rg: "prime-data-hub-staging"
env: "staging"
pwd_key: "POSTGRESQL_STAGING_PWD"
db: "pdhstaging-pgsql"
- rg: "prime-data-hub-prod"
env: "prod"
pwd_key: "POSTGRESQL_PROD_PWD"
db: "pdhprod-pgsql"
runs-on: ubuntu-latest
steps:
- name: "Check out changes"
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
- name: Connect to VPN and login to Azure
uses: ./.github/actions/vpn-azure
with:
env-name: prod
env-name: ${{ matrix.env }}
sp-creds: ${{ secrets.SERVICE_PRINCIPAL_CREDS }}
tf-auth: false
- name: Run Az Cost CLI
id: az-cost
uses: ./.github/actions/az-cost
with:
rg: prime-data-hub-prod
rg: ${{ matrix.rg }}
days_ago: 180
total_format: daily
- name: Get Az Cost Result
run: |
echo "${{ steps.az-cost.outputs.result }}" > costs_raw.csv
touch costs.csv
sed 's/\r//' costs_raw.csv > costs.csv
cat ./costs.csv
sed 's/\r//' costs_raw.csv > costs_total.csv
cat ./costs_total.csv
- name: Run Az Cost CLI By Resource
id: az-cost-by-resource
uses: ./.github/actions/az-cost
with:
rg: ${{ matrix.rg }}
days_ago: 180
total_format: dailyByResource
- name: Get Az Cost Result By Resource
run: |
echo "${{ steps.az-cost-by-resource.outputs.result }}" > costs_by_res_raw.csv
touch costs_by_res.csv
sed 's/\r//' costs_by_res_raw.csv > costs_by_res.csv
cat ./costs_by_res.csv
- name: Get runner ip
id: runner_ip
uses: ./.github/actions/runner-ip
Expand All @@ -39,15 +63,15 @@ jobs:
id: add-db-access
shell: bash
run: |
az postgres server update -g prime-data-hub-prod -n pdhprod-pgsql --public-network-access "Enabled"
az postgres server update -g ${{ matrix.rg }} -n ${{ matrix.db }} --public-network-access "Enabled"
sleep 10;
az postgres server firewall-rule create -g prime-data-hub-prod -s pdhprod-pgsql -n github_actions_runner \
az postgres server firewall-rule create -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner \
--start-ip-address ${{ steps.runner_ip.outputs.ip-address }} --end-ip-address ${{ steps.runner_ip.outputs.ip-address }}
- name: Store cost data in DB
uses: ./.github/actions/db-query
with:
pass: ${{ secrets['POSTGRESQL_PROD_PWD'] }}
host: pdhprod-pgsql
pass: ${{ secrets[matrix.pwd_key] }}
host: ${{ matrix.db }}
port: 5432
user: prime
database: prime_data_hub
Expand All @@ -60,12 +84,24 @@ jobs:
Currency varchar(3)
);
TRUNCATE Azure_Costs;
\copy Azure_Costs (Date, Cost, CostUsd, Currency) FROM './costs.csv' WITH DELIMITER ',' CSV HEADER;
\copy Azure_Costs (Date, Cost, CostUsd, Currency) FROM './costs_total.csv' WITH DELIMITER ',' CSV HEADER;
CREATE TABLE IF NOT EXISTS Azure_Resource_Costs (
Date date,
Name varchar(128),
Cost NUMERIC(13, 8),
CostUsd NUMERIC(13, 8),
Currency varchar(3),
Tags varchar(256)
);
TRUNCATE Azure_Resource_Costs;
\copy Azure_Resource_Costs (Date, Name, Cost, CostUsd, Currency, Tags) FROM './costs_by_res.csv' WITH DELIMITER ',' CSV HEADER;
# Remove runner db access
- name: Remove DB access
if: always() && steps.add-db-access.outcome == 'success'
shell: bash
run: |
az postgres server firewall-rule delete -g prime-data-hub-prod -s pdhprod-pgsql -n github_actions_runner --yes
az postgres server firewall-rule delete -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner --yes
sleep 10;
az postgres server update -g prime-data-hub-prod -n pdhprod-pgsql --public-network-access "Disabled"
az postgres server update -g ${{ matrix.rg }} -n ${{ matrix.db }} --public-network-access "Disabled"

0 comments on commit 64f0143

Please sign in to comment.