Merge pull request #124 from dfe-analytical-services/azure-pipelines-dev

Set up dev CI with Azure Pipelines

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,24 @@
+<!-- 
+Hey, thanks for raising a PR! We're excited to see what you've done!
+To help us review the changes, please complete each section in this template by replacing '...' with details to help the reviewers of this pull request. 
+-->
+
+# Brief overview of changes
+
+...
+
+## Why are these changes being made?
+
+...
+
+## Detailed description of changes
+
+...
+
+## Additional information for reviewers
+
+...
+
+## Issue ticket number/s and link
+
+...

.github/workflows/automatedTests.yaml

@@ -1,12 +1,12 @@
 on:
   push:
     branches:
-      - master
+      - main
 
   pull_request:
 
 
-name: automatedTests
+name: Automated tests
 
 jobs:
   automatedTests:
@@ -24,11 +24,20 @@ jobs:
 
       - name: Install git2r dependencies
         run: sudo apt-get install -y libgit2-dev
+
+      - name: Cache renv packages
+        id: cache-renv
+        uses: actions/cache@v4
+        with:
+          path: screener-renv-cache
+          key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-renv-
       
       - uses: r-lib/actions/setup-renv@v2
 
       - name: Run tests 
         shell: Rscript {0}
         run: |
           shinytest2::test_app()
-          
+          

.github/workflows/deployShinyApps.yaml

@@ -1,9 +1,9 @@
 on:
   push:
     branches:
-      - master
+      - main
 
-name: deploy-shiny
+name: shinyapps.io deploy
 
 jobs:
   deployShiny:

.hooks/pre-commit.R

@@ -1,5 +1,5 @@
 #!/usr/bin/env Rscript
-message("\nRunning commit hooks...", fill = TRUE)
+message("\nRunning commit hooks...")
 
 message("\n")
 
@@ -15,6 +15,24 @@ if (any(style_output)) {
   message("...code styling checks passed")
 }
 
+message("\n")
+message("\n2. Rebuilding manifest.json...")
+if (system.file(package = "rsconnect") != "" & system.file(package = "git2r") != "") {
+  if (!any(grepl("manifest.json", git2r::status()))) {
+    rsconnect::writeManifest()
+    git2r::add(path = "manifest.json")
+  }
+  message("...manifest.json rebuilt\n")
+} else {
+  if (system.file(package = "rsconnect") == "") {
+    message("rsconnect is not installed")
+  }
+  if (system.file(package = "git2r") == "") {
+    message("git2r is not installed")
+  }
+  message("...this step has been skipped")
+}
+
 message("\n")
 
 # End of hooks

R/standard-data-prep/update-geography-lookups.R → R/manual_scripts/update-geography-lookups.R

@@ -1,9 +1,9 @@
 # Code for updating geography lookups =================================
 # source functions and dependencies
 
-source("R/standard-data-prep/utils.R")
+source("R/manual_scripts/utils.R")
 
-# PCON LA =====================================================================
+# Ward PCON LAD LA ============================================================
 # Download latest from: https://geoportal.statistics.gov.uk/search?q=LUP_WPC&sort=Date%20Created%7Ccreated%7Cdesc
 # ...and save into /data/downloaded_source_data
 # Then add the file path into the function below to update the lookup to
@@ -12,11 +12,11 @@ source("R/standard-data-prep/utils.R")
 # Last title of file used on Open Geography Portal (in case it's helpful):
 # Ward to Westminster Parliamentary Constituency to Local Authority District to UTLA
 
-# Files used so far: 2017, 2019, 2020, 2021, 2022, 2023
+# Files used so far: 2017, 2019, 2020, 2021, 2022, 2023, 2024
 
 write_updated_lookup(
   new_lookup = tidy_downloaded_lookup(
-    open_geography_file = "data/downloaded_source_data/2023.csv",
+    open_geography_file = "data/downloaded_source_data/Ward_to_Westminster_Parliamentary_Constituency_to_LAD_to_UTLA_(July_2024)_Lookup_in_UK.csv",
     shorthand_lookup = open_geog_shorthand_lookup
   ),
   lookup_filepath = "data/ward_lad_la_pcon_hierarchy.csv"

R/standard-data-prep/utils.R → R/manual_scripts/utils.R

@@ -46,7 +46,8 @@ tidy_downloaded_lookup <- function(
     shorthand_lookup) {
   # Read in the downloaded open geography file --------------------------------
   message("Reading in new data from: ", open_geography_file)
-  new_data <- read_csv(open_geography_file, show_col_types = FALSE)
+  new_data <- read_csv(open_geography_file, show_col_types = FALSE) %>%
+    select(-ends_with("NMW")) # remove Welsh name cols
 
   # Extract the year from columns ---------------------------------------------
   new_year <- names(new_data) %>%

README.md

@@ -1,4 +1,5 @@
-[![automatedTests](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/automatedTests.yaml/badge.svg?branch=master)](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/automatedTests.yaml) [![deployShinyApps](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/deployShinyApps.yaml/badge.svg?branch=master)](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/deployShinyApps.yaml)
+[![automatedTests](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/automatedTests.yaml/badge.svg?branch=main)](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/automatedTests.yaml) [![deployShinyApps](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/deployShinyApps.yaml/badge.svg?branch=main)](https://github.com/dfe-analytical-services/dfe-published-data-qa/actions/workflows/deployShinyApps.yaml)
+[![Build Status](https://dfe-gov-uk.visualstudio.com/stats-development/_apis/build/status%2Fdfe-analytical-services.dfe-published-data-qa?repoName=dfe-analytical-services%2Fdfe-published-data-qa&branchName=main)](https://dfe-gov-uk.visualstudio.com/stats-development/_build/latest?definitionId=1381&repoName=dfe-analytical-services%2Fdfe-published-data-qa&branchName=main)
 
 <h1 align="center">
   <br>
@@ -79,7 +80,7 @@ All functions created to screen files have an associated failing file, and unit
 
 End to end UI tests have been created using shinytest2 that test the app loads, that the reset button really does clear everything, and that files that we'd expect to hit each particular stage of the screening do so.
 
-GitHub Actions provide CI by running the automated tests and checks for code styling, with badges in this README referring to the latest state of the master branch. The yaml files for these workflows can be found in the `.github/workflows` folder.
+GitHub Actions provide CI by running the automated tests and checks for code styling, with badges in this README referring to the latest state of the main branch. The yaml files for these workflows can be found in the `.github/workflows` folder.
 
 ### Deployment
 
@@ -113,25 +114,31 @@ There is a GitHub action workflow that automatically runs this against PRs to en
 
 The standard geographies used in this tool are sourced from (or developed with) the Office for National Statistics (ONS). As such, they are maintained in line with the ONS's [open geography portal](https://geoportal.statistics.gov.uk/). 
 
-#### Updating pcon_la or lad_lsip lookups
+#### Updating ward_pcon_lad_la or lad_lsip lookups
 
-There's an R script in the R/standard-data-prep folder containing functions to update both the pcon_la and lad_lsip lookups. To use this, download the latest version of the file from the searches linked in the comments of that script, and then save a copy of that data in a new folder called /data/downloaded_source_data/. The functions will then append any new data from that CSV into the lookup files.
+There's an R script in the `R/manual_scripts/` folder containing functions to update both the geography lookups. To use this, download the latest version of the file from the searches linked in the comments of that script, and then save a copy of that data in a new folder called `/data/downloaded_source_data/`. The functions will then append any new data from that CSV into the lookup files.
 
-#### Local skills improvement plan area (LSIP)
+* [Ward / PCon / LAD / LA lookup on Open Geography Portal (2024)](https://geoportal.statistics.gov.uk/datasets/62eb9df29a2f4521b5076a419ff9a47e_0/explore)
 
-These were developed in direct communications with the ONS ([ONS Geography e-mail](mailto:[email protected])) and they provided the basis for the code and name file used in this tool. At the time of writing, this has not been added to the open geography portal, although boundary files have been ([LSIP boundary map files](https://geoportal.statistics.gov.uk/search?collection=Dataset&sort=name&tags=all(BDY_LSIP%2CAUG_2023))).
+Local skills improvement plan area (LSIP) to LAD lookup was developed in direct communications with the ONS ([ONS Geography e-mail](mailto:[email protected])) and they provided the basis for the code and name file used in this tool. 
 
-#### Wards
+* [LAD / LSIP lookup on Open Geography Portal (2023)](https://geoportal.statistics.gov.uk/datasets/effcab9660fd4375baaed44c2bd23719_0/explore?q=local%20skills%20lad)
 
-The Ward to LAD lookup has been downloaded from the [Open Geography Portal administrative lookups](https://geoportal.statistics.gov.uk/search?q=LUP_WD_LAD&sort=Title%7Ctitle%7Cdesc). 
+#### Boundary files
 
-For example, in March 2024 we downloaded and used the 'Ward to Local Authority District (May 2023) Lookup in the United Kingdom' data set.
+Boundary files for use when mapping aren't available in this repo but are published on the Open Geography Portal. 
 
-#### PCon 2024 lookup
+* [Regions boundary map files](https://geoportal.statistics.gov.uk/search?q=BDY_RGN&sort=Title%7Ctitle%7Casc)
 
-We have recently added a lookup including provisional 2024 Parliamentary Constituency boundaries. This is available in the `data/pcon_2024_v2.csv` file, and further information on that file can be found on the [Open Geography Portal](https://geoportal.statistics.gov.uk/datasets/ons::postcode-to-new-westminster-parliamentary-constituencies-may-2024-lookup-in-the-uk/about).
+* [Local authority boundary map files (Counties and Unitary Authorities)](https://geoportal.statistics.gov.uk/search?q=BDY_CTYUA&sort=Date%20Created%7Ccreated%7Cdesc)
 
-Long term we aim to remove this and instead update the Ward > PCon > LAD > LA lookup once that is released by ONS.
+* [Local authority district boundary map files](https://geoportal.statistics.gov.uk/search?q=BDY_LAD&sort=Date%20Created%7Ccreated%7Cdesc)
+
+* [Westminster Constituency boundary map files](https://geoportal.statistics.gov.uk/search?q=BDY_PCON&sort=Date%20Created%7Ccreated%7Cdesc)
+
+* [Ward boundary map files](https://geoportal.statistics.gov.uk/search?q=BDY_WD&sort=Title%7Ctitle%7Casc)
+
+* [LSIP boundary map files](https://geoportal.statistics.gov.uk/search?collection=Dataset&sort=name&tags=all(BDY_LSIP%2CAUG_2023))
 
 ---
 

azure-pipelines-dev.yml

@@ -0,0 +1,31 @@
+name: ees-data-screener-$(rev:r)
+
+pool:
+  name: RSConnect
+
+trigger:
+  branches:
+    include:
+    - development
+
+  paths:
+    exclude:
+    - azure-pipelines.yml
+    - azure-pipelines-dev.yml
+
+resources:
+  repositories:
+    - repository: posit-deploy
+      type: git
+      name: ESFA-Data-Science/posit-deploy
+
+extends:
+  template: deploy.yml@posit-deploy
+  parameters:
+    appName: dfe-published-data-qa
+    environments:
+    - name: Pre-Production
+      appId: 460
+    approvers: |
+     [email protected]
+     [email protected]

azure-pipelines.yml

@@ -0,0 +1,33 @@
+name: ees-data-screener-$(rev:r)
+
+pool:
+  name: RSConnect
+
+trigger:
+  branches:
+    include:
+    - main
+
+  paths:
+    exclude:
+    - azure-pipelines.yml
+
+resources:
+  repositories:
+    - repository: posit-deploy
+      type: git
+      name: ESFA-Data-Science/posit-deploy
+
+extends:
+  template: deploy.yml@posit-deploy
+  parameters:
+    appName: dfe-published-data-qa
+    environments:
+    - name: Pre-Production
+      appId: 459
+    - name: Production
+      appId: 123
+    approvers: |
+     [email protected]
+     [email protected]
+     [email protected]