From bad90a8fcc3d1b0992036c1742345d41b8c00110 Mon Sep 17 00:00:00 2001 From: Hassan Mir Date: Wed, 27 Nov 2024 18:02:07 +0000 Subject: [PATCH 1/4] Update sanitise script --- db/scripts/sanitise.sql | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/db/scripts/sanitise.sql b/db/scripts/sanitise.sql index 0fd6ae9b0..bb3ea2c05 100644 --- a/db/scripts/sanitise.sql +++ b/db/scripts/sanitise.sql @@ -25,7 +25,9 @@ SET -- no update required -- Assessment --- no update required +UPDATE "assessments" +SET + recommendation_assessor_note = CASE WHEN recommendation_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END; -- Countries -- no update required @@ -51,12 +53,17 @@ UPDATE "further_information_request_items" SET failure_reason_assessor_feedback = CASE WHEN failure_reason_assessor_feedback IS NULL THEN NULL ELSE '[sanitised]' END, response = CASE WHEN response IS NULL THEN NULL ELSE '[sanitised]' END; + contact_email = CASE WHEN contact_email IS NULL THEN NULL ELSE 'sanitised@example.com' END; + contact_name = CASE WHEN contact_name IS NULL THEN NULL ELSE '[sanitised]' END; -- FurtherInformationRequest UPDATE "further_information_requests" SET failure_assessor_note = CASE WHEN failure_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END; +-- MailDeliveryFailure +DELETE FROM "mail_delivery_failures"; + -- Note UPDATE "notes" SET @@ -107,6 +114,9 @@ SET UPDATE "teachers" SET email = CASE WHEN email IS NULL THEN NULL ELSE CONCAT('teacher', id, '@example.com') END, + canonical_email = CASE WHEN email IS NULL THEN NULL ELSE CONCAT('teacher', id, '@example.com') END, + gov_one_email = CASE WHEN email IS NULL THEN NULL ELSE CONCAT('teacher', id, '@example.com') END, + email_domain = CASE WHEN email IS NULL THEN NULL ELSE '@example.com' END, current_sign_in_ip = '0.0.0.0', last_sign_in_ip = '0.0.0.0'; @@ -116,7 +126,9 @@ SET annotation = CASE WHEN annotation IS NULL THEN NULL ELSE '[sanitised]' END; -- Uploads --- no update required +UPDATE "uploads" +SET + filename = CASE WHEN filename IS NULL THEN NULL ELSE '[sanitised]' END; -- WorkHistory UPDATE "work_histories" @@ -125,4 +137,5 @@ SET city = '[sanitised]', job = '[sanitised]', contact_email = 'sanitised@example.com', + canonical_contact_email = 'sanitised@example.com', contact_name = '[sanitised]'; From a14e54e88acd363e574ee8bd480a9d34c5252a33 Mon Sep 17 00:00:00 2001 From: Hassan Mir Date: Wed, 27 Nov 2024 22:33:07 +0000 Subject: [PATCH 2/4] More sanitisation --- db/scripts/sanitise.sql | 54 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/db/scripts/sanitise.sql b/db/scripts/sanitise.sql index bb3ea2c05..11292594d 100644 --- a/db/scripts/sanitise.sql +++ b/db/scripts/sanitise.sql @@ -27,7 +27,16 @@ SET -- Assessment UPDATE "assessments" SET - recommendation_assessor_note = CASE WHEN recommendation_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END; + recommendation_assessor_note = CASE WHEN recommendation_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END, + age_range_note = CASE WHEN age_range_note IS NULL THEN NULL ELSE '[sanitised]' END, + qualifications_assessor_note = CASE WHEN qualifications_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END, + subjects_note = CASE WHEN subjects_note IS NULL THEN NULL ELSE '[sanitised]' END; + +-- ConsentRequest +UPDATE "consent_requests" +SET + verify_note = CASE WHEN verify_note IS NULL THEN NULL ELSE '[sanitised]' END, + review_note = CASE WHEN review_note IS NULL THEN NULL ELSE '[sanitised]' END; -- Countries -- no update required @@ -52,14 +61,15 @@ UPDATE "feature_flags_features" UPDATE "further_information_request_items" SET failure_reason_assessor_feedback = CASE WHEN failure_reason_assessor_feedback IS NULL THEN NULL ELSE '[sanitised]' END, - response = CASE WHEN response IS NULL THEN NULL ELSE '[sanitised]' END; - contact_email = CASE WHEN contact_email IS NULL THEN NULL ELSE 'sanitised@example.com' END; + response = CASE WHEN response IS NULL THEN NULL ELSE '[sanitised]' END, + contact_email = CASE WHEN contact_email IS NULL THEN NULL ELSE 'sanitised@example.com' END, contact_name = CASE WHEN contact_name IS NULL THEN NULL ELSE '[sanitised]' END; -- FurtherInformationRequest UPDATE "further_information_requests" SET - failure_assessor_note = CASE WHEN failure_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END; + failure_assessor_note = CASE WHEN failure_assessor_note IS NULL THEN NULL ELSE '[sanitised]' END, + review_note = CASE WHEN review_note IS NULL THEN NULL ELSE '[sanitised]' END; -- MailDeliveryFailure DELETE FROM "mail_delivery_failures"; @@ -69,6 +79,13 @@ UPDATE "notes" SET text = CASE WHEN text IS NULL THEN NULL ELSE '[sanitised]' END; +-- ProfessionalStandingRequests +UPDATE "professional_standing_requests" +SET + location_note = CASE WHEN location_note IS NULL THEN NULL ELSE '[sanitised]' END, + review_note = CASE WHEN review_note IS NULL THEN NULL ELSE '[sanitised]' END, + verify_note = CASE WHEN verify_note IS NULL THEN NULL ELSE '[sanitised]' END; + -- Qualification UPDATE "qualifications" SET @@ -78,11 +95,30 @@ SET complete_date = '01/01/2002', certificate_date = '01/01/2002'; +-- QualificationRequest +UPDATE "qualification_requests" +SET + review_note = CASE WHEN review_note IS NULL THEN NULL ELSE '[sanitised]' END, + verify_note = CASE WHEN verify_note IS NULL THEN NULL ELSE '[sanitised]' END; + -- Region -- no update required -- ReferenceRequest --- no update required +UPDATE "qualification_requests" +SET + additional_information_response = CASE WHEN additional_information_response IS NULL THEN NULL ELSE '[sanitised]' END, + children_comment = CASE WHEN children_comment IS NULL THEN NULL ELSE '[sanitised]' END, + contact_comment = CASE WHEN contact_comment IS NULL THEN NULL ELSE '[sanitised]' END, + dates_comment = CASE WHEN dates_comment IS NULL THEN NULL ELSE '[sanitised]' END, + hours_comment = CASE WHEN hours_comment IS NULL THEN NULL ELSE '[sanitised]' END, + lessons_comment = CASE WHEN lessons_comment IS NULL THEN NULL ELSE '[sanitised]' END, + misconduct_comment = CASE WHEN misconduct_comment IS NULL THEN NULL ELSE '[sanitised]' END, + reports_comment = CASE WHEN reports_comment IS NULL THEN NULL ELSE '[sanitised]' END, + review_note = CASE WHEN review_note IS NULL THEN NULL ELSE '[sanitised]' END, + satisfied_comment = CASE WHEN satisfied_comment IS NULL THEN NULL ELSE '[sanitised]' END, + verify_note = CASE WHEN verify_note IS NULL THEN NULL ELSE '[sanitised]' END, + slug = CASE WHEN slug IS NULL THEN NULL ELSE concat('slug-', id) END; -- ReminderEmail -- no update required @@ -117,13 +153,19 @@ SET canonical_email = CASE WHEN email IS NULL THEN NULL ELSE CONCAT('teacher', id, '@example.com') END, gov_one_email = CASE WHEN email IS NULL THEN NULL ELSE CONCAT('teacher', id, '@example.com') END, email_domain = CASE WHEN email IS NULL THEN NULL ELSE '@example.com' END, + access_your_teaching_qualifications_url = CASE WHEN access_your_teaching_qualifications_url IS NULL THEN NULL ELSE '[sanitised]' END, current_sign_in_ip = '0.0.0.0', last_sign_in_ip = '0.0.0.0'; -- TimelineEvent UPDATE "timeline_events" SET - annotation = CASE WHEN annotation IS NULL THEN NULL ELSE '[sanitised]' END; + annotation = CASE WHEN annotation IS NULL THEN NULL ELSE '[sanitised]' END, + age_range_note = CASE WHEN age_range_note IS NULL THEN NULL ELSE '[sanitised]' END, + new_value = CASE WHEN new_value IS NULL THEN NULL ELSE '[sanitised]' END, + old_value = CASE WHEN old_value IS NULL THEN NULL ELSE '[sanitised]' END, + note_text = CASE WHEN note_text IS NULL THEN NULL ELSE '[sanitised]' END, + subjects_note = CASE WHEN subjects_note IS NULL THEN NULL ELSE '[sanitised]' END; -- Uploads UPDATE "uploads" From 64045366de99d041dc11bd5d971e60d1c1999eaf Mon Sep 17 00:00:00 2001 From: Hassan Mir Date: Wed, 27 Nov 2024 22:42:08 +0000 Subject: [PATCH 3/4] Adding dqt_match as PII column --- config/analytics_hidden_pii.yml | 1 + db/scripts/sanitise.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/config/analytics_hidden_pii.yml b/config/analytics_hidden_pii.yml index 140682a55..5aa229ed6 100644 --- a/config/analytics_hidden_pii.yml +++ b/config/analytics_hidden_pii.yml @@ -6,6 +6,7 @@ - family_name - given_names - registration_number + - dqt_match :assessments: - recommendation_assessor_note :further_information_request_items: diff --git a/db/scripts/sanitise.sql b/db/scripts/sanitise.sql index 11292594d..8caed7a2b 100644 --- a/db/scripts/sanitise.sql +++ b/db/scripts/sanitise.sql @@ -13,6 +13,7 @@ SET date_of_birth = '2000-01-01', alternative_given_names = '', alternative_family_name = '', + dqt_match = '{}', registration_number = ''; -- AssessmentSectionFailureReason From dd3fdbe66a8434bc1f196e8c8813eb4ba51bc642 Mon Sep 17 00:00:00 2001 From: Hassan Mir Date: Thu, 28 Nov 2024 15:28:46 +0000 Subject: [PATCH 4/4] Fix --- db/scripts/sanitise.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/scripts/sanitise.sql b/db/scripts/sanitise.sql index 8caed7a2b..9cdda1f14 100644 --- a/db/scripts/sanitise.sql +++ b/db/scripts/sanitise.sql @@ -106,7 +106,7 @@ SET -- no update required -- ReferenceRequest -UPDATE "qualification_requests" +UPDATE "reference_requests" SET additional_information_response = CASE WHEN additional_information_response IS NULL THEN NULL ELSE '[sanitised]' END, children_comment = CASE WHEN children_comment IS NULL THEN NULL ELSE '[sanitised]' END,