From bde68c5d846f40f4db09f4eb1a971e84e3a2bdd9 Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Fri, 30 Aug 2024 16:03:18 -0400 Subject: [PATCH 1/8] Modified export_cost_data.yml GHA workflow to add costs by resource to the export. --- .github/actions/az-cost/action.yml | 12 +++++++++- .github/workflows/export_cost_data.yml | 31 +++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/actions/az-cost/action.yml b/.github/actions/az-cost/action.yml index 5cafdf639fb..b1520c52d78 100644 --- a/.github/actions/az-cost/action.yml +++ b/.github/actions/az-cost/action.yml @@ -38,6 +38,16 @@ runs: dotnet azure-cost accumulatedCost -g ${{ inputs.rg }} -t custom --from $(date --date='${{ inputs.days_ago }} days ago' '+%m/%d/%Y') --to $(date --date='1 days ago' '+%m/%d/%Y') -o csv >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT + - name: Run azure-cost-cli for daily totals by resource + if: inputs.total_format == 'dailyByResource' + id: run_azure_cost_daily_by_resource + shell: bash + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "RESULT<<$EOF" >> $GITHUB_OUTPUT + dotnet azure-cost dailyCosts -g ${{ inputs.rg }} -t custom --from $(date --date='${{ inputs.days_ago }} days ago' '+%m/%d/%Y') --to $(date --date='1 days ago' '+%m/%d/%Y') -o csv --exclude-meter-details --dimension ConsumedService >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + - name: Run azure-cost-cli for total if: inputs.total_format == 'summary' id: run_azure_cost_summary @@ -52,5 +62,5 @@ runs: run: | EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) echo "RESULT<<$EOF" >> $GITHUB_OUTPUT - echo -e "${{ steps.run_azure_cost_daily.outputs.RESULT }}${{ steps.run_azure_cost_summary.outputs.RESULT }}" >> $GITHUB_OUTPUT + echo -e "${{ steps.run_azure_cost_daily.outputs.RESULT }}${{ steps.run_azure_cost_daily_by_resource.outputs.RESULT }}${{ steps.run_azure_cost_summary.outputs.RESULT }}" >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index 7609207fb42..fe3fcab9ac9 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -29,8 +29,21 @@ jobs: run: | echo "${{ steps.az-cost.outputs.result }}" > costs_raw.csv touch costs.csv - sed 's/\r//' costs_raw.csv > costs.csv - cat ./costs.csv + sed 's/\r//' costs_raw.csv > costs_total.csv + cat ./costs_total.csv + - name: Run Az Cost CLI By Resource + id: az-cost-by-resource + uses: ./.github/actions/az-cost + with: + rg: prime-data-hub-prod + days_ago: 180 + total_format: dailyByResource + - name: Get Az Cost Result By Resource + run: | + echo "${{ steps.az-cost-by-resource.outputs.result }}" > costs_by_res_raw.csv + touch costs_by_res.csv + sed 's/\r//' costs_by_res_raw.csv > costs_by_res.csv + cat ./costs_by_res.csv - name: Get runner ip id: runner_ip uses: ./.github/actions/runner-ip @@ -60,7 +73,19 @@ jobs: Currency varchar(3) ); TRUNCATE Azure_Costs; - \copy Azure_Costs (Date, Cost, CostUsd, Currency) FROM './costs.csv' WITH DELIMITER ',' CSV HEADER; + \copy Azure_Costs (Date, Cost, CostUsd, Currency) FROM './costs_total.csv' WITH DELIMITER ',' CSV HEADER; + + CREATE TABLE IF NOT EXISTS Azure_Resource_Costs ( + Date date, + Name varchar(128), + Cost NUMERIC(13, 8), + CostUsd NUMERIC(13, 8), + Currency varchar(3), + Tags varchar(256) + ); + TRUNCATE Azure_Resource_Costs; + \copy Azure_Resource_Costs (Date, Name, Cost, CostUsd, Currency, Tags) FROM './costs_by_res.csv' WITH DELIMITER ',' CSV HEADER; + # Remove runner db access - name: Remove DB access if: always() && steps.add-db-access.outcome == 'success' From ea415c73f8db05c99123eb7b9b60cdc01e9bf9f7 Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Fri, 30 Aug 2024 16:06:12 -0400 Subject: [PATCH 2/8] Added trigger on: push: branches: - dkrylov/azure_resource_costs --- .github/workflows/export_cost_data.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index fe3fcab9ac9..8c06b54548a 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -1,10 +1,9 @@ name: Export Cost Data on: - schedule: - # The workflow runs every day at 3PM - - cron: "0 19 * * *" #UTC-5 - + push: + branches: + - dkrylov/azure_resource_costs jobs: export_cost_data: name: Export Azure Cost Data to Storage From 46b9706b17179ed6d9fc3bc952508286dcaab762 Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Fri, 30 Aug 2024 16:08:38 -0400 Subject: [PATCH 3/8] Added `dotnet new tool-manifest --force` --- .github/actions/az-cost/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/az-cost/action.yml b/.github/actions/az-cost/action.yml index b1520c52d78..ce0d8330fbc 100644 --- a/.github/actions/az-cost/action.yml +++ b/.github/actions/az-cost/action.yml @@ -25,7 +25,7 @@ runs: - name: Install azure-cost-cli shell: bash run: | - dotnet new tool-manifest + dotnet new tool-manifest --force dotnet tool install azure-cost-cli - name: Run azure-cost-cli for daily totals From ec72d7358be2de3a716b5d7f166438cf49cd7327 Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Tue, 3 Sep 2024 14:54:57 -0400 Subject: [PATCH 4/8] Added matrix config to export_cost_data.yml. --- .github/workflows/export_cost_data.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index 8c06b54548a..f3be7bb84d0 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -7,6 +7,14 @@ on: jobs: export_cost_data: name: Export Azure Cost Data to Storage + strategy: + fail-fast: false + matrix: + include: + - rg: "prime-data-hub-staging" + env: "staging" + pwd_key: "POSTGRESQL_STAGING_PWD" + db: "pdhstaging-pgsql" runs-on: ubuntu-latest steps: - name: "Check out changes" @@ -14,14 +22,14 @@ jobs: - name: Connect to VPN and login to Azure uses: ./.github/actions/vpn-azure with: - env-name: prod + env-name: ${{ matrix.env }} sp-creds: ${{ secrets.SERVICE_PRINCIPAL_CREDS }} tf-auth: false - name: Run Az Cost CLI id: az-cost uses: ./.github/actions/az-cost with: - rg: prime-data-hub-prod + rg: ${{ matrix.rg }} days_ago: 180 total_format: daily - name: Get Az Cost Result @@ -34,7 +42,7 @@ jobs: id: az-cost-by-resource uses: ./.github/actions/az-cost with: - rg: prime-data-hub-prod + rg: ${{ matrix.rg }} days_ago: 180 total_format: dailyByResource - name: Get Az Cost Result By Resource @@ -51,15 +59,15 @@ jobs: id: add-db-access shell: bash run: | - az postgres server update -g prime-data-hub-prod -n pdhprod-pgsql --public-network-access "Enabled" + az postgres server update -g ${{ matrix.rg }} -n ${{ matrix.db }} --public-network-access "Enabled" sleep 10; - az postgres server firewall-rule create -g prime-data-hub-prod -s pdhprod-pgsql -n github_actions_runner \ + az postgres server firewall-rule create -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner \ --start-ip-address ${{ steps.runner_ip.outputs.ip-address }} --end-ip-address ${{ steps.runner_ip.outputs.ip-address }} - name: Store cost data in DB uses: ./.github/actions/db-query with: - pass: ${{ secrets['POSTGRESQL_PROD_PWD'] }} - host: pdhprod-pgsql + pass: ${{ secrets[matrix.pwd_key] }} + host: ${{ matrix.db }} port: 5432 user: prime database: prime_data_hub @@ -90,6 +98,6 @@ jobs: if: always() && steps.add-db-access.outcome == 'success' shell: bash run: | - az postgres server firewall-rule delete -g prime-data-hub-prod -s pdhprod-pgsql -n github_actions_runner --yes + az postgres server firewall-rule delete -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner --yes sleep 10; - az postgres server update -g prime-data-hub-prod -n pdhprod-pgsql --public-network-access "Disabled" + az postgres server update -g ${{ matrix.rg }} -n ${{ matrix.db }} --public-network-access "Disabled" From ae8fd5c69cdad168ac57cb2697b2fd051510c5d2 Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Tue, 3 Sep 2024 15:02:24 -0400 Subject: [PATCH 5/8] Added a retry to remove-db-access. --- .github/workflows/export_cost_data.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index f3be7bb84d0..a0c557e2cf6 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -96,6 +96,9 @@ jobs: # Remove runner db access - name: Remove DB access if: always() && steps.add-db-access.outcome == 'success' + uses: nick-fields/retry@v3 + max_attempts: 3 + retry_wait_seconds: 30 shell: bash run: | az postgres server firewall-rule delete -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner --yes From e7ad02bddac2ffc3c9515115dff31759e79ab8db Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Tue, 3 Sep 2024 15:03:32 -0400 Subject: [PATCH 6/8] Revert "Added a retry to remove-db-access." This reverts commit ae8fd5c69cdad168ac57cb2697b2fd051510c5d2. --- .github/workflows/export_cost_data.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index a0c557e2cf6..f3be7bb84d0 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -96,9 +96,6 @@ jobs: # Remove runner db access - name: Remove DB access if: always() && steps.add-db-access.outcome == 'success' - uses: nick-fields/retry@v3 - max_attempts: 3 - retry_wait_seconds: 30 shell: bash run: | az postgres server firewall-rule delete -g ${{ matrix.rg }} -s ${{ matrix.db }} -n github_actions_runner --yes From bccb517b5bba357a764f68d8f054ee5a9f5d2473 Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Tue, 3 Sep 2024 15:13:53 -0400 Subject: [PATCH 7/8] Added Prod to the matrix. --- .github/workflows/export_cost_data.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index f3be7bb84d0..252efcd9dcc 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -15,6 +15,10 @@ jobs: env: "staging" pwd_key: "POSTGRESQL_STAGING_PWD" db: "pdhstaging-pgsql" + - rg: "prime-data-hub-prod" + env: "prod" + pwd_key: "POSTGRESQL_PROD_PWD" + db: "pdhprod-pgsql" runs-on: ubuntu-latest steps: - name: "Check out changes" From 25df8644e0493135f2e112176a53bcea6fedea1c Mon Sep 17 00:00:00 2001 From: Denis Krylov Date: Tue, 3 Sep 2024 15:43:48 -0400 Subject: [PATCH 8/8] The workflow runs every day at 3PM --- .github/workflows/export_cost_data.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/export_cost_data.yml b/.github/workflows/export_cost_data.yml index 252efcd9dcc..b7f451d3b25 100644 --- a/.github/workflows/export_cost_data.yml +++ b/.github/workflows/export_cost_data.yml @@ -1,9 +1,9 @@ name: Export Cost Data on: - push: - branches: - - dkrylov/azure_resource_costs + schedule: + # The workflow runs every day at 3PM + - cron: "0 19 * * *" #UTC-5 jobs: export_cost_data: name: Export Azure Cost Data to Storage