From fb66248957a5256e2b5b2c27a0cd4a51b77d4f66 Mon Sep 17 00:00:00 2001 From: Gail Terman Date: Sun, 19 May 2024 19:21:33 -0400 Subject: [PATCH 01/21] fix missing null check --- app/workers/registration_sync_worker.rb | 54 +++++++++++++------------ db/structure.sql | 1 - 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index d4548b7d2..9290f5e3d 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -41,13 +41,13 @@ def phase1(page_size: 500) results = svc.people_by_page(page: 1, page_size: page_size) store_reg_data(data: results['data']) - last_page = results['meta']['last_page'].to_i + last_page = results.dig('meta', 'last_page')&.to_i || 0 for page in (2..last_page) do results = svc.people_by_page(page: page, page_size: page_size) if !results["message"] store_reg_data(data: results['data']) else - puts "We had an issue with the Clyde ... people by page #{page}, #{page_size}" + puts "We had an issue with the Clyde ... people by page #{page}, #{page_size}, #{last_page}, #{results["message"]}" end end end @@ -79,31 +79,33 @@ def phase2 def store_reg_data(data:) RegistrationSyncDatum.transaction do - data.each do |d| - # puts "#{d['id']} -> #{d['full_name']} -> #{d['email']}" - # preferred_name, alternative_email - # TODO: move to an adapter when we have to support multiple reg services - next unless d['attending_status'] != 'Not Attending' - # Products to exclude from matching - next if [ - 'Chengdu', - 'Volunteer', - 'Apocryphal', - 'Infant', - 'Installment', - 'Hall Pass', - 'Staff', - ].include? d['product_list_name'] + if data + data.each do |d| + # puts "#{d['id']} -> #{d['full_name']} -> #{d['email']}" + # preferred_name, alternative_email + # TODO: move to an adapter when we have to support multiple reg services + next unless d['attending_status'] != 'Not Attending' + # Products to exclude from matching + next if [ + 'Chengdu', + 'Volunteer', + 'Apocryphal', + 'Infant', + 'Installment', + 'Hall Pass', + 'Staff', + ].include? d['product_list_name'] - RegistrationSyncDatum.create( - reg_id: d['id'], - name: d['full_name'], - email: d['email'], - registration_number: d['ticket_number'], - preferred_name: d['preferred_name'], - alternative_email: d['alternative_email'], - raw_info: d - ) + RegistrationSyncDatum.create( + reg_id: d['id'], + name: d['full_name'], + email: d['email'], + registration_number: d['ticket_number'], + preferred_name: d['preferred_name'], + alternative_email: d['alternative_email'], + raw_info: d + ) + end end end end diff --git a/db/structure.sql b/db/structure.sql index 1cee53788..a860fa618 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -3856,4 +3856,3 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240515001411'); - From 63d6832713252ccc0e71d1dcae53741b682a63f4 Mon Sep 17 00:00:00 2001 From: Gail Terman Date: Sun, 19 May 2024 21:47:10 -0400 Subject: [PATCH 02/21] bad commit message because i'm tired --- .../registrations/person_sync_table.vue | 32 +++++++++++++++---- .../registrations/registration_sync_table.vue | 4 +-- app/workers/registration_sync_worker.rb | 6 ++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/app/javascript/registrations/person_sync_table.vue b/app/javascript/registrations/person_sync_table.vue index c548ea413..949e1f8b5 100644 --- a/app/javascript/registrations/person_sync_table.vue +++ b/app/javascript/registrations/person_sync_table.vue @@ -5,18 +5,38 @@ :columns="columns" ref="person-sync-table" stateName="person-sync-table-search-state" + :showControls="false" > + @@ -53,4 +73,4 @@ export default { this.$refs['person-sync-table'].fetchPaged() } } - \ No newline at end of file + diff --git a/app/javascript/registrations/registration_sync_table.vue b/app/javascript/registrations/registration_sync_table.vue index 9b871f02a..64d946ac6 100644 --- a/app/javascript/registrations/registration_sync_table.vue +++ b/app/javascript/registrations/registration_sync_table.vue @@ -11,7 +11,7 @@ 
{{ item.raw_info | pretty }}
@@ -49,4 +49,4 @@ export default { this.$refs['registration-sync-table'].fetchPaged() } } - \ No newline at end of file + diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index 9290f5e3d..531bfaf5e 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -16,6 +16,7 @@ def perform puts "--- Sync Phase 1 #{Time.now}" # Hack because of staxo bug with page size on their staging server page_size = Rails.env == 'production' ? 500 : 30 + puts "page size: #{page_size}" phase1(page_size: page_size) puts "--- Sync Phase 2 #{Time.now}" # Phase 2 @@ -37,6 +38,9 @@ def phase1(page_size: 500) # Get the clyde service and use the AUTH key that we have svc = ClydeService.get_svc(token: ENV['CLYDE_AUTH_KEY']) + if !svc.token { + raise "Missing auth token! abort abort abort!" + } results = svc.people_by_page(page: 1, page_size: page_size) store_reg_data(data: results['data']) @@ -106,6 +110,8 @@ def store_reg_data(data:) raw_info: d ) end + else + puts "There was an error! Data is empty" end end end From f37741c33558b5b28c05cb7298beadf4d84a6368 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 21 May 2024 16:42:43 -0400 Subject: [PATCH 03/21] Change matching criteria --- app/controllers/concerns/resource_methods.rb | 6 +- .../person_sync_data_controller.rb | 11 ++ app/lib/migration_helpers/plano_views.rb | 16 ++- app/models/job_status.rb | 12 ++ app/models/person_sync_datum.rb | 106 +++++++++++++++++- app/models/publication_status.rb | 3 +- app/models/registration_sync_datum.rb | 4 + app/models/registration_sync_status.rb | 12 ++ .../person_sync_datum_serializer.rb | 102 +++++++++++++++++ .../registration_sync_datum_serializer.rb | 4 + app/workers/registration_sync_worker.rb | 21 ++-- ...21184252_add_badgename_to_reg_sync_data.rb | 5 + ...0521193119_add_indexes_to_reg_sync_data.rb | 7 ++ db/structure.sql | 33 +++++- test/factories/registration_sync_data.rb | 4 + test/models/registration_sync_datum_test.rb | 4 + 16 files changed, 327 insertions(+), 23 deletions(-) create mode 100644 db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb create mode 100644 db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb diff --git a/app/controllers/concerns/resource_methods.rb b/app/controllers/concerns/resource_methods.rb index 50c827beb..d0495464d 100644 --- a/app/controllers/concerns/resource_methods.rb +++ b/app/controllers/concerns/resource_methods.rb @@ -249,7 +249,7 @@ def collection .where(collection_where) # anpther where? - q = q.distinct if join_tables && !join_tables.empty? + q = q.distinct if (join_tables && !join_tables.empty?) || make_distinct? q = q.order(order_string) @@ -762,6 +762,10 @@ def array_col?(col_name:) false end + def make_distinct? + false + end + def permitted_params() _permitted_params(model: nil) end diff --git a/app/controllers/person_sync_data_controller.rb b/app/controllers/person_sync_data_controller.rb index 3136cd15b..74f184c9c 100644 --- a/app/controllers/person_sync_data_controller.rb +++ b/app/controllers/person_sync_data_controller.rb @@ -15,12 +15,23 @@ def default_scope(query: nil) .where('registration_sync_data.reg_id in (select reg_id from registration_map_counts)') end + def select_fields + PersonSyncDatum.select( + ::PersonSyncDatum.arel_table[Arel.star], + 'name_sort_by' + ) + end + def serializer_includes [ :registration_sync_data ] end + def make_distinct? + true + end + def includes [ :email_addresses, diff --git a/app/lib/migration_helpers/plano_views.rb b/app/lib/migration_helpers/plano_views.rb index f91cd9f0c..c8f3bc09c 100644 --- a/app/lib/migration_helpers/plano_views.rb +++ b/app/lib/migration_helpers/plano_views.rb @@ -25,12 +25,22 @@ def self.create_registration_sync_matches select p.name, null as email, p.id as pid, rsd.reg_id, rsd.id as rid, 'name' as mtype from people p join registration_sync_data rsd - on rsd."name" ilike p.name + on + ( + rsd."name" ilike p.name OR + rsd."preferred_name" ilike p.name OR + rsd."badge_name" ilike p.name + ) union select null as name, e.email, e.person_id as pid, rsd.reg_id, rsd.id as rid, 'email' as mtype - from email_addresses e + from email_addresses e join registration_sync_data rsd - on rsd."email" ilike e.email + on + ( + rsd."email" ilike e.email or + rsd."alternative_email" ilike e.email + ) + where e.isdefault = true SQL ActiveRecord::Base.connection.execute(query) diff --git a/app/models/job_status.rb b/app/models/job_status.rb index 47c2b3f89..c6f48c74d 100644 --- a/app/models/job_status.rb +++ b/app/models/job_status.rb @@ -1,3 +1,15 @@ +# == Schema Information +# +# Table name: job_statuses +# +# id :uuid not null, primary key +# lock_version :integer default(0) +# status :string +# submit_time :datetime +# type :string +# created_at :datetime not null +# updated_at :datetime not null +# class JobStatus < ApplicationRecord validates_inclusion_of :status, in: %w[inprogress completed] end diff --git a/app/models/person_sync_datum.rb b/app/models/person_sync_datum.rb index ae6fbf6f8..ea3fe1a69 100644 --- a/app/models/person_sync_datum.rb +++ b/app/models/person_sync_datum.rb @@ -1,9 +1,109 @@ -# This links the person to possible matches from the -# registration system +# == Schema Information +# +# Table name: people +# +# id :uuid not null, primary key +# accommodations :string(10000) +# age_at_convention :string +# attendance_type :string(200) +# availability_notes :string +# bio :text +# black_diaspora :string(10000) +# bsky :string +# can_photo :enum default("no") +# can_photo_exceptions :string(10000) +# can_record :enum default("no") +# can_record_exceptions :string(10000) +# can_share :boolean default(FALSE), not null +# can_stream :enum default("no") +# can_stream_exceptions :string(10000) +# comments :text +# con_state :enum default("not_set") +# confirmation_sent_at :datetime +# confirmation_token :string +# confirmed_at :datetime +# current_sign_in_at :datetime +# current_sign_in_ip :inet +# date_reg_synced :datetime +# demographic_categories :string +# do_not_assign_with :string(10000) +# encrypted_password :string default(""), not null +# ethnicity :string(400) +# excluded_demographic_categories :string +# facebook :string +# failed_attempts :integer default(0), not null +# fediverse :string +# flickr :string +# gender :string(400) +# global_diaspora :string +# indigenous :string(10000) +# instagram :string +# integrations :jsonb not null +# is_local :boolean default(FALSE) +# job_title :string +# language :string(5) default("") +# languages_fluent_in :string(10000) +# last_sign_in_at :datetime +# last_sign_in_ip :inet +# linkedin :string +# lock_version :integer default(0) +# locked_at :datetime +# moderation_experience :string(10000) +# name :string default("") +# name_sort_by :string +# name_sort_by_confirmed :boolean default(FALSE) +# needs_accommodations :boolean default(FALSE) +# non_anglophone :string +# non_us_centric_perspectives :string(10000) +# opted_in :boolean default(FALSE), not null +# organization :string +# othered :string(10000) +# othersocialmedia :string +# pronouns :string(400) +# pseudonym :string +# pseudonym_sort_by :string +# pseudonym_sort_by_confirmed :boolean default(FALSE) +# published_name :string +# published_name_sort_by :string +# reddit :string +# reg_attending_status :string +# registered :boolean default(FALSE), not null +# registration_number :string +# registration_type :string +# remember_created_at :datetime +# reset_password_sent_at :datetime +# reset_password_token :string +# romantic_sexual_orientation :string +# sign_in_count :integer default(0), not null +# tiktok :string +# timezone :string(500) +# twelve_hour :boolean default(TRUE) +# twitch :string +# twitter :string +# unconfirmed_email :string +# unlock_token :string +# website :string +# willing_to_moderate :boolean default(FALSE) +# year_of_birth :integer +# youtube :string +# created_at :datetime not null +# updated_at :datetime not null +# reg_id :string +# +# Indexes +# +# index_people_on_bio (bio) USING gin +# index_people_on_confirmation_token (confirmation_token) UNIQUE +# index_people_on_name (name) USING gin +# index_people_on_pseudonym (pseudonym) USING gin +# index_people_on_published_name (published_name) USING gin +# index_people_on_reset_password_token (reset_password_token) UNIQUE +# index_people_on_unlock_token (unlock_token) UNIQUE +# class PersonSyncDatum < Person has_many :registration_sync_matches, class_name: 'Registration::RegistrationSyncMatch', foreign_key: 'pid' has_many :registration_sync_data, through: :registration_sync_matches -end \ No newline at end of file +end diff --git a/app/models/publication_status.rb b/app/models/publication_status.rb index 69a94af26..3206a819c 100644 --- a/app/models/publication_status.rb +++ b/app/models/publication_status.rb @@ -1,11 +1,12 @@ # == Schema Information # -# Table name: publication_statuses +# Table name: job_statuses # # id :uuid not null, primary key # lock_version :integer default(0) # status :string # submit_time :datetime +# type :string # created_at :datetime not null # updated_at :datetime not null # diff --git a/app/models/registration_sync_datum.rb b/app/models/registration_sync_datum.rb index 3de9d076c..d804e15e1 100644 --- a/app/models/registration_sync_datum.rb +++ b/app/models/registration_sync_datum.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) # diff --git a/app/models/registration_sync_status.rb b/app/models/registration_sync_status.rb index 9badee244..fbefdf9c3 100644 --- a/app/models/registration_sync_status.rb +++ b/app/models/registration_sync_status.rb @@ -1,3 +1,15 @@ +# == Schema Information +# +# Table name: job_statuses +# +# id :uuid not null, primary key +# lock_version :integer default(0) +# status :string +# submit_time :datetime +# type :string +# created_at :datetime not null +# updated_at :datetime not null +# # class RegistrationSyncStatus < JobStatus end diff --git a/app/serializers/person_sync_datum_serializer.rb b/app/serializers/person_sync_datum_serializer.rb index b066994a5..2957e9ff7 100644 --- a/app/serializers/person_sync_datum_serializer.rb +++ b/app/serializers/person_sync_datum_serializer.rb @@ -1,3 +1,105 @@ +# == Schema Information +# +# Table name: people +# +# id :uuid not null, primary key +# accommodations :string(10000) +# age_at_convention :string +# attendance_type :string(200) +# availability_notes :string +# bio :text +# black_diaspora :string(10000) +# bsky :string +# can_photo :enum default("no") +# can_photo_exceptions :string(10000) +# can_record :enum default("no") +# can_record_exceptions :string(10000) +# can_share :boolean default(FALSE), not null +# can_stream :enum default("no") +# can_stream_exceptions :string(10000) +# comments :text +# con_state :enum default("not_set") +# confirmation_sent_at :datetime +# confirmation_token :string +# confirmed_at :datetime +# current_sign_in_at :datetime +# current_sign_in_ip :inet +# date_reg_synced :datetime +# demographic_categories :string +# do_not_assign_with :string(10000) +# encrypted_password :string default(""), not null +# ethnicity :string(400) +# excluded_demographic_categories :string +# facebook :string +# failed_attempts :integer default(0), not null +# fediverse :string +# flickr :string +# gender :string(400) +# global_diaspora :string +# indigenous :string(10000) +# instagram :string +# integrations :jsonb not null +# is_local :boolean default(FALSE) +# job_title :string +# language :string(5) default("") +# languages_fluent_in :string(10000) +# last_sign_in_at :datetime +# last_sign_in_ip :inet +# linkedin :string +# lock_version :integer default(0) +# locked_at :datetime +# moderation_experience :string(10000) +# name :string default("") +# name_sort_by :string +# name_sort_by_confirmed :boolean default(FALSE) +# needs_accommodations :boolean default(FALSE) +# non_anglophone :string +# non_us_centric_perspectives :string(10000) +# opted_in :boolean default(FALSE), not null +# organization :string +# othered :string(10000) +# othersocialmedia :string +# pronouns :string(400) +# pseudonym :string +# pseudonym_sort_by :string +# pseudonym_sort_by_confirmed :boolean default(FALSE) +# published_name :string +# published_name_sort_by :string +# reddit :string +# reg_attending_status :string +# registered :boolean default(FALSE), not null +# registration_number :string +# registration_type :string +# remember_created_at :datetime +# reset_password_sent_at :datetime +# reset_password_token :string +# romantic_sexual_orientation :string +# sign_in_count :integer default(0), not null +# tiktok :string +# timezone :string(500) +# twelve_hour :boolean default(TRUE) +# twitch :string +# twitter :string +# unconfirmed_email :string +# unlock_token :string +# website :string +# willing_to_moderate :boolean default(FALSE) +# year_of_birth :integer +# youtube :string +# created_at :datetime not null +# updated_at :datetime not null +# reg_id :string +# +# Indexes +# +# index_people_on_bio (bio) USING gin +# index_people_on_confirmation_token (confirmation_token) UNIQUE +# index_people_on_name (name) USING gin +# index_people_on_pseudonym (pseudonym) USING gin +# index_people_on_published_name (published_name) USING gin +# index_people_on_reset_password_token (reset_password_token) UNIQUE +# index_people_on_unlock_token (unlock_token) UNIQUE +# # class PersonSyncDatumSerializer include JSONAPI::Serializer diff --git a/app/serializers/registration_sync_datum_serializer.rb b/app/serializers/registration_sync_datum_serializer.rb index 99ee06454..f60763385 100644 --- a/app/serializers/registration_sync_datum_serializer.rb +++ b/app/serializers/registration_sync_datum_serializer.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) # diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index 531bfaf5e..adec51847 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -15,9 +15,7 @@ def perform # Phase 1 - get the data from Clyde and store it puts "--- Sync Phase 1 #{Time.now}" # Hack because of staxo bug with page size on their staging server - page_size = Rails.env == 'production' ? 500 : 30 - puts "page size: #{page_size}" - phase1(page_size: page_size) + phase1(page_size: 500) puts "--- Sync Phase 2 #{Time.now}" # Phase 2 phase2 @@ -29,7 +27,7 @@ def perform status.save! end puts "--- Sync Complete #{Time.now}" - end + end # Phase 1 is to suck up the data from Reg and put it into a temp store # for matching @@ -38,9 +36,9 @@ def phase1(page_size: 500) # Get the clyde service and use the AUTH key that we have svc = ClydeService.get_svc(token: ENV['CLYDE_AUTH_KEY']) - if !svc.token { + if !svc.token raise "Missing auth token! abort abort abort!" - } + end results = svc.people_by_page(page: 1, page_size: page_size) store_reg_data(data: results['data']) @@ -102,11 +100,12 @@ def store_reg_data(data:) RegistrationSyncDatum.create( reg_id: d['id'], - name: d['full_name'], - email: d['email'], - registration_number: d['ticket_number'], - preferred_name: d['preferred_name'], - alternative_email: d['alternative_email'], + name: d['full_name']&.strip, + email: d['email']&.strip, + registration_number: d['ticket_number']&.strip, + preferred_name: d['preferred_name']&.strip, + alternative_email: d['alternative_email']&.strip, + badge_name: d['badge_name']&.strip, raw_info: d ) end diff --git a/db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb b/db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb new file mode 100644 index 000000000..f0892bfb8 --- /dev/null +++ b/db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb @@ -0,0 +1,5 @@ +class AddBadgenameToRegSyncData < ActiveRecord::Migration[6.1] + def change + add_column :registration_sync_data, :badge_name, :string, default: nil + end +end diff --git a/db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb b/db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb new file mode 100644 index 000000000..1c9fe1b49 --- /dev/null +++ b/db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb @@ -0,0 +1,7 @@ +class AddIndexesToRegSyncData < ActiveRecord::Migration[6.1] + def change + add_index :registration_sync_data, :badge_name, using: :gin, opclass: :gin_trgm_ops + add_index :registration_sync_data, :preferred_name, using: :gin, opclass: :gin_trgm_ops + add_index :registration_sync_data, :alternative_email, using: :gin, opclass: :gin_trgm_ops + end +end diff --git a/db/structure.sql b/db/structure.sql index a860fa618..5ec0e5401 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1612,7 +1612,8 @@ CREATE TABLE public.registration_sync_data ( raw_info jsonb DEFAULT '{}'::jsonb NOT NULL, lock_version integer, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL + updated_at timestamp(6) without time zone NOT NULL, + badge_name character varying ); @@ -1628,7 +1629,7 @@ CREATE VIEW public.registration_sync_matches AS rsd.id AS rid, 'name'::text AS mtype FROM (public.people p - JOIN public.registration_sync_data rsd ON (((rsd.name)::text ~~* (p.name)::text))) + JOIN public.registration_sync_data rsd ON ((((rsd.name)::text ~~* (p.name)::text) OR ((rsd.preferred_name)::text ~~* (p.name)::text) OR ((rsd.badge_name)::text ~~* (p.name)::text)))) UNION SELECT NULL::character varying AS name, e.email, @@ -1637,7 +1638,8 @@ UNION rsd.id AS rid, 'email'::text AS mtype FROM (public.email_addresses e - JOIN public.registration_sync_data rsd ON (((rsd.email)::text ~~* (e.email)::text))); + JOIN public.registration_sync_data rsd ON ((((rsd.email)::text ~~* (e.email)::text) OR ((rsd.alternative_email)::text ~~* (e.email)::text)))) + WHERE (e.isdefault = true); -- @@ -3230,6 +3232,20 @@ CREATE INDEX index_published_programme_item_assignments_on_person_id ON public.p CREATE INDEX index_published_sessions_on_format_id ON public.published_sessions USING btree (format_id); +-- +-- Name: index_registration_sync_data_on_alternative_email; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_registration_sync_data_on_alternative_email ON public.registration_sync_data USING gin (alternative_email public.gin_trgm_ops); + + +-- +-- Name: index_registration_sync_data_on_badge_name; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_registration_sync_data_on_badge_name ON public.registration_sync_data USING gin (badge_name public.gin_trgm_ops); + + -- -- Name: index_registration_sync_data_on_email; Type: INDEX; Schema: public; Owner: - -- @@ -3244,6 +3260,13 @@ CREATE INDEX index_registration_sync_data_on_email ON public.registration_sync_d CREATE INDEX index_registration_sync_data_on_name ON public.registration_sync_data USING gin (name public.gin_trgm_ops); +-- +-- Name: index_registration_sync_data_on_preferred_name; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_registration_sync_data_on_preferred_name ON public.registration_sync_data USING gin (preferred_name public.gin_trgm_ops); + + -- -- Name: index_registration_sync_data_on_reg_id; Type: INDEX; Schema: public; Owner: - -- @@ -3853,6 +3876,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240303213410'), ('20240423130325'), ('20240429160250'), -('20240515001411'); +('20240515001411'), +('20240521184252'), +('20240521193119'); diff --git a/test/factories/registration_sync_data.rb b/test/factories/registration_sync_data.rb index 57ac18455..ce5e2faa7 100644 --- a/test/factories/registration_sync_data.rb +++ b/test/factories/registration_sync_data.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) # diff --git a/test/models/registration_sync_datum_test.rb b/test/models/registration_sync_datum_test.rb index 12f00b5de..b317a6b45 100644 --- a/test/models/registration_sync_datum_test.rb +++ b/test/models/registration_sync_datum_test.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) # From f1d33fc8c26e0fcdbd98a7b2a9921967823f97bb Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 21 May 2024 16:45:51 -0400 Subject: [PATCH 04/21] add badge name to serializer --- app/serializers/registration_sync_datum_serializer.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/serializers/registration_sync_datum_serializer.rb b/app/serializers/registration_sync_datum_serializer.rb index f60763385..376107f3b 100644 --- a/app/serializers/registration_sync_datum_serializer.rb +++ b/app/serializers/registration_sync_datum_serializer.rb @@ -31,6 +31,7 @@ class RegistrationSyncDatumSerializer attributes :id, :lock_version, :created_at, :updated_at, :reg_id, :registration_number, :name, :email, :preferred_name, :alternative_email, + :badge_name, :raw_info # The people that this data could be matched to From 199c596964f8b96e559e621a9e701b02b5ed7a64 Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 22 May 2024 12:59:12 -0400 Subject: [PATCH 05/21] Improved matching (names) --- app/javascript/registrations/person_sync_table.vue | 2 +- app/lib/migration_helpers/plano_views.rb | 3 +++ app/workers/registration_sync_worker.rb | 11 +++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/javascript/registrations/person_sync_table.vue b/app/javascript/registrations/person_sync_table.vue index 949e1f8b5..49b18c9dd 100644 --- a/app/javascript/registrations/person_sync_table.vue +++ b/app/javascript/registrations/person_sync_table.vue @@ -27,7 +27,7 @@
{{ reg_data.email }}
{{ reg_data.registration_number }}
Preferred Name: {{ reg_data.preferred_name }}
-
Badge Name: {{ reg_data.raw_info.badge_title }}
+
Badge Name: {{ reg_data.badge_name }}
Alternative Email: {{ reg_data.alternative_email }}
Product: {{ reg_data.raw_info.product }}
Attending Status: {{ reg_data.raw_info.attending_status }}
diff --git a/app/lib/migration_helpers/plano_views.rb b/app/lib/migration_helpers/plano_views.rb index c8f3bc09c..8fd908366 100644 --- a/app/lib/migration_helpers/plano_views.rb +++ b/app/lib/migration_helpers/plano_views.rb @@ -30,6 +30,9 @@ def self.create_registration_sync_matches rsd."name" ilike p.name OR rsd."preferred_name" ilike p.name OR rsd."badge_name" ilike p.name + or rsd."name" ilike p.pseudonym + or rsd."preferred_name" ilike p.pseudonym + or rsd."badge_name" ilike p.pseudonym ) union select null as name, e.email, e.person_id as pid, rsd.reg_id, rsd.id as rid, 'email' as mtype diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index adec51847..af7706ca0 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -11,10 +11,8 @@ class RegistrationSyncWorker def perform PublishedSession.transaction do - # DO WORK # Phase 1 - get the data from Clyde and store it puts "--- Sync Phase 1 #{Time.now}" - # Hack because of staxo bug with page size on their staging server phase1(page_size: 500) puts "--- Sync Phase 2 #{Time.now}" # Phase 2 @@ -32,15 +30,15 @@ def perform # Phase 1 is to suck up the data from Reg and put it into a temp store # for matching def phase1(page_size: 500) - RegistrationSyncDatum.connection.truncate(RegistrationSyncDatum.table_name) - # Get the clyde service and use the AUTH key that we have svc = ClydeService.get_svc(token: ENV['CLYDE_AUTH_KEY']) if !svc.token raise "Missing auth token! abort abort abort!" end + RegistrationSyncDatum.connection.truncate(RegistrationSyncDatum.table_name) results = svc.people_by_page(page: 1, page_size: page_size) + store_reg_data(data: results['data']) last_page = results.dig('meta', 'last_page')&.to_i || 0 @@ -73,8 +71,9 @@ def phase2 datum = RegistrationSyncDatum.find_by reg_id: match.reg_id + # TODO: update the match status + # automatic match, assisted match, manual match, no match. IdentityService.update_reg_info(person: person, details: datum.raw_info) - person.save! end end end @@ -105,7 +104,7 @@ def store_reg_data(data:) registration_number: d['ticket_number']&.strip, preferred_name: d['preferred_name']&.strip, alternative_email: d['alternative_email']&.strip, - badge_name: d['badge_name']&.strip, + badge_name: d['badge']&.strip, raw_info: d ) end From 38975c835e7dd99f9c86e1ad12c4e2abcfb792c2 Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 22 May 2024 15:45:19 -0400 Subject: [PATCH 06/21] Add reg match to person Put in place table for dismissed possible matches --- app/models/dismissed_reg_sync_match.rb | 19 ++++++ app/models/person.rb | 3 + app/models/person_sync_datum.rb | 1 + app/serializers/person_serializer.rb | 2 + .../person_sync_datum_serializer.rb | 6 +- app/services/identity_service.rb | 5 +- ...74506_create_dismissed_reg_sync_matches.rb | 13 ++++ .../20240522190737_add_person_match_enum.rb | 20 ++++++ db/structure.sql | 65 ++++++++++++++++++- spec/models/person_spec.rb | 1 + test/factories/dismissed_reg_sync_matches.rb | 22 +++++++ test/models/dismissed_reg_sync_match_test.rb | 24 +++++++ 12 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 app/models/dismissed_reg_sync_match.rb create mode 100644 db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb create mode 100644 db/migrate/20240522190737_add_person_match_enum.rb create mode 100644 test/factories/dismissed_reg_sync_matches.rb create mode 100644 test/models/dismissed_reg_sync_match_test.rb diff --git a/app/models/dismissed_reg_sync_match.rb b/app/models/dismissed_reg_sync_match.rb new file mode 100644 index 000000000..226f1b1ee --- /dev/null +++ b/app/models/dismissed_reg_sync_match.rb @@ -0,0 +1,19 @@ +# == Schema Information +# +# Table name: dismissed_reg_sync_matches +# +# id :uuid not null, primary key +# lock_version :integer +# created_at :datetime not null +# updated_at :datetime not null +# person_id :uuid not null +# reg_id :string not null +# +# Indexes +# +# idx_person_reg_id (person_id,reg_id) UNIQUE +# index_dismissed_reg_sync_matches_on_person_id (person_id) +# index_dismissed_reg_sync_matches_on_reg_id (reg_id) +# +class DismissedRegSyncMatch < ApplicationRecord +end diff --git a/app/models/person.rb b/app/models/person.rb index d9f77419b..d5b16e7db 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -67,6 +67,7 @@ # published_name_sort_by :string # reddit :string # reg_attending_status :string +# reg_match :enum default("no_match") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string @@ -255,6 +256,8 @@ def completed rejected: 'rejected' } + enum reg_match: {none: 'none', automatic: 'automatic', assisted: 'assisted', manual: 'manual', self: 'self'}, _scopes: false + nilify_blanks only: [ :bio, :pseudonym, diff --git a/app/models/person_sync_datum.rb b/app/models/person_sync_datum.rb index ea3fe1a69..8d2c923d8 100644 --- a/app/models/person_sync_datum.rb +++ b/app/models/person_sync_datum.rb @@ -67,6 +67,7 @@ # published_name_sort_by :string # reddit :string # reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string diff --git a/app/serializers/person_serializer.rb b/app/serializers/person_serializer.rb index 1a2270534..d192f2336 100644 --- a/app/serializers/person_serializer.rb +++ b/app/serializers/person_serializer.rb @@ -67,6 +67,7 @@ # published_name_sort_by :string # reddit :string # reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string @@ -152,6 +153,7 @@ class PersonSerializer #< ActiveModel::Serializer :reg_id, :reg_attending_status, :date_reg_synced + :reg_match # status and comments hidden except for staff protected_attributes :con_state, :comments diff --git a/app/serializers/person_sync_datum_serializer.rb b/app/serializers/person_sync_datum_serializer.rb index 2957e9ff7..b2da13f04 100644 --- a/app/serializers/person_sync_datum_serializer.rb +++ b/app/serializers/person_sync_datum_serializer.rb @@ -67,6 +67,7 @@ # published_name_sort_by :string # reddit :string # reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string @@ -100,7 +101,6 @@ # index_people_on_reset_password_token (reset_password_token) UNIQUE # index_people_on_unlock_token (unlock_token) UNIQUE # -# class PersonSyncDatumSerializer include JSONAPI::Serializer @@ -108,8 +108,8 @@ class PersonSyncDatumSerializer :pseudonym, :pseudonym_sort_by, :pseudonym_sort_by_confirmed, :published_name, :published_name_sort_by, :job_title, :organization, :reg_id, - :primary_email, :contact_email - + :primary_email, :contact_email, + :reg_match, :date_reg_synced has_many :email_addresses, if: Proc.new { |record, params| AccessControlService.shared_attribute_access?(instance: record, person: params[:current_person]) }, diff --git a/app/services/identity_service.rb b/app/services/identity_service.rb index d14b55b8f..11b0313ee 100644 --- a/app/services/identity_service.rb +++ b/app/services/identity_service.rb @@ -109,7 +109,7 @@ def self.clear_person_reg_info(person:) person.date_reg_synced = Time.now end - def self.update_reg_info(person:, details:) + def self.update_reg_info(person:, details:, reg_match: Person.reg_matches[:self]) # If the Ticket Numbers do not match then we reset cause there may be an issue if person.registration_number && person.registration_number != details['ticket_number'] clear_person_reg_info(person: person) @@ -125,6 +125,9 @@ def self.update_reg_info(person:, details:) person.reg_attending_status = details['attending_status'] # Need to store time of last sync person.date_reg_synced = Time.now + # How the registration match was done + person.reg_match = reg_match + # Attendance type in Plano is one of # in_person, hybrid, virtual # Clyde does not map to these well. Recommend that we get this from survey and Person profile diff --git a/db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb b/db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb new file mode 100644 index 000000000..f72f738df --- /dev/null +++ b/db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb @@ -0,0 +1,13 @@ +class CreateDismissedRegSyncMatches < ActiveRecord::Migration[6.1] + def change + create_table :dismissed_reg_sync_matches, id: :uuid do |t| + t.uuid :person_id, index: true, null: false + t.string :reg_id, index: true, null: false + + t.integer :lock_version + t.timestamps + end + + add_index :dismissed_reg_sync_matches, [:person_id, :reg_id], unique: true, name: "idx_person_reg_id" + end +end diff --git a/db/migrate/20240522190737_add_person_match_enum.rb b/db/migrate/20240522190737_add_person_match_enum.rb new file mode 100644 index 000000000..62900a8b9 --- /dev/null +++ b/db/migrate/20240522190737_add_person_match_enum.rb @@ -0,0 +1,20 @@ +class AddPersonMatchEnum < ActiveRecord::Migration[6.1] + def change + reversible do |change| + change.down do + remove_column :people, :reg_match + + execute <<-SQL + DROP TYPE reg_match_enum; + SQL + end + change.up do + execute <<-SQL + CREATE TYPE reg_match_enum AS ENUM ('none', 'automatic', 'assisted', 'manual', 'self'); + SQL + + add_column :people, :reg_match, :reg_match_enum, default: 'none' + end + end + end +end diff --git a/db/structure.sql b/db/structure.sql index 5ec0e5401..e9814a875 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -223,6 +223,19 @@ CREATE TYPE public.phone_type_enum AS ENUM ( ); +-- +-- Name: reg_match_enum; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.reg_match_enum AS ENUM ( + 'none', + 'automatic', + 'assisted', + 'manual', + 'self' +); + + -- -- Name: schedule_approval_enum; Type: TYPE; Schema: public; Owner: - -- @@ -667,7 +680,8 @@ END) STORED, non_anglophone character varying, fediverse character varying, bsky character varying, - reg_attending_status character varying + reg_attending_status character varying, + reg_match public.reg_match_enum DEFAULT 'none'::public.reg_match_enum ); @@ -925,6 +939,20 @@ CREATE TABLE public.curated_tags ( ); +-- +-- Name: dismissed_reg_sync_matches; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.dismissed_reg_sync_matches ( + id uuid DEFAULT public.gen_random_uuid() NOT NULL, + person_id uuid NOT NULL, + reg_id character varying NOT NULL, + lock_version integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + -- -- Name: email_addresses; Type: TABLE; Schema: public; Owner: - -- @@ -1629,7 +1657,7 @@ CREATE VIEW public.registration_sync_matches AS rsd.id AS rid, 'name'::text AS mtype FROM (public.people p - JOIN public.registration_sync_data rsd ON ((((rsd.name)::text ~~* (p.name)::text) OR ((rsd.preferred_name)::text ~~* (p.name)::text) OR ((rsd.badge_name)::text ~~* (p.name)::text)))) + JOIN public.registration_sync_data rsd ON ((((rsd.name)::text ~~* (p.name)::text) OR ((rsd.preferred_name)::text ~~* (p.name)::text) OR ((rsd.badge_name)::text ~~* (p.name)::text) OR ((rsd.name)::text ~~* (p.pseudonym)::text) OR ((rsd.preferred_name)::text ~~* (p.pseudonym)::text) OR ((rsd.badge_name)::text ~~* (p.pseudonym)::text)))) UNION SELECT NULL::character varying AS name, e.email, @@ -2442,6 +2470,14 @@ ALTER TABLE ONLY public.curated_tags ADD CONSTRAINT curated_tags_pkey PRIMARY KEY (id); +-- +-- Name: dismissed_reg_sync_matches dismissed_reg_sync_matches_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.dismissed_reg_sync_matches + ADD CONSTRAINT dismissed_reg_sync_matches_pkey PRIMARY KEY (id); + + -- -- Name: email_addresses email_addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2945,6 +2981,13 @@ CREATE INDEX fk_configurations_parameters_idx ON public.configurations USING btr CREATE UNIQUE INDEX fl_configurations_unique_index ON public.configurations USING btree (parameter); +-- +-- Name: idx_person_reg_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_person_reg_id ON public.dismissed_reg_sync_matches USING btree (person_id, reg_id); + + -- -- Name: idx_tagname_on_context; Type: INDEX; Schema: public; Owner: - -- @@ -3036,6 +3079,20 @@ CREATE INDEX index_audit_survey_versions_on_item_type_and_item_id ON public.audi CREATE INDEX index_convention_roles_on_person_id ON public.convention_roles USING btree (person_id); +-- +-- Name: index_dismissed_reg_sync_matches_on_person_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_dismissed_reg_sync_matches_on_person_id ON public.dismissed_reg_sync_matches USING btree (person_id); + + +-- +-- Name: index_dismissed_reg_sync_matches_on_reg_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_dismissed_reg_sync_matches_on_reg_id ON public.dismissed_reg_sync_matches USING btree (reg_id); + + -- -- Name: index_email_addresses_on_email; Type: INDEX; Schema: public; Owner: - -- @@ -3878,6 +3935,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240429160250'), ('20240515001411'), ('20240521184252'), -('20240521193119'); +('20240521193119'), +('20240522174506'), +('20240522190737'); diff --git a/spec/models/person_spec.rb b/spec/models/person_spec.rb index 757babad5..b83635eb1 100644 --- a/spec/models/person_spec.rb +++ b/spec/models/person_spec.rb @@ -67,6 +67,7 @@ # published_name_sort_by :string # reddit :string # reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string diff --git a/test/factories/dismissed_reg_sync_matches.rb b/test/factories/dismissed_reg_sync_matches.rb new file mode 100644 index 000000000..7436a6883 --- /dev/null +++ b/test/factories/dismissed_reg_sync_matches.rb @@ -0,0 +1,22 @@ +# == Schema Information +# +# Table name: dismissed_reg_sync_matches +# +# id :uuid not null, primary key +# lock_version :integer +# created_at :datetime not null +# updated_at :datetime not null +# person_id :uuid not null +# reg_id :string not null +# +# Indexes +# +# idx_person_reg_id (person_id,reg_id) UNIQUE +# index_dismissed_reg_sync_matches_on_person_id (person_id) +# index_dismissed_reg_sync_matches_on_reg_id (reg_id) +# +FactoryBot.define do + factory :dismissed_reg_sync_match do + + end +end diff --git a/test/models/dismissed_reg_sync_match_test.rb b/test/models/dismissed_reg_sync_match_test.rb new file mode 100644 index 000000000..c923a01d3 --- /dev/null +++ b/test/models/dismissed_reg_sync_match_test.rb @@ -0,0 +1,24 @@ +# == Schema Information +# +# Table name: dismissed_reg_sync_matches +# +# id :uuid not null, primary key +# lock_version :integer +# created_at :datetime not null +# updated_at :datetime not null +# person_id :uuid not null +# reg_id :string not null +# +# Indexes +# +# idx_person_reg_id (person_id,reg_id) UNIQUE +# index_dismissed_reg_sync_matches_on_person_id (person_id) +# index_dismissed_reg_sync_matches_on_reg_id (reg_id) +# +require "test_helper" + +class DismissedRegSyncMatchTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 2f748cae7706d614ebc1293da83c427d03d1971c Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 27 May 2024 12:00:03 -0400 Subject: [PATCH 07/21] Add methods for match and dismiss --- .../person_sync_data_controller.rb | 59 +++++++++++++++++++ app/lib/migration_helpers/plano_views.rb | 4 +- app/models/dismissed_reg_sync_match.rb | 2 + app/models/person.rb | 2 +- app/policies/person_sync_datum_policy.rb | 9 +++ app/workers/registration_sync_worker.rb | 5 +- config/routes.rb | 3 +- lib/tasks/rbac.rake | 6 ++ 8 files changed, 84 insertions(+), 6 deletions(-) diff --git a/app/controllers/person_sync_data_controller.rb b/app/controllers/person_sync_data_controller.rb index 74f184c9c..421f348f6 100644 --- a/app/controllers/person_sync_data_controller.rb +++ b/app/controllers/person_sync_data_controller.rb @@ -5,6 +5,65 @@ class PersonSyncDataController < ResourceController DEFAULT_SORTBY = 'name_sort_by' DEFAULT_ORDER = 'asc'.freeze + # + # + # + def match + model_class.transaction do + authorize model_class, policy_class: policy_class + + reg_id = params[:reg_id] if params[:reg_id] + person_id = params[:person_id] if params[:person_id] + + # one of 'assisted' or 'manual' + reg_match = params[:reg_match] if params[:reg_match] + + raise "Type of match should be 'assisted' or 'manual' you gave '#{reg_match}'" unless ['assisted', 'manual'].include? reg_match + raise "No reg id, person id, or match type specified" unless reg_id && person_id && reg_match + + # Get the reg sync data + datum = RegistrationSyncDatum.find_by reg_id: reg_id + + # Get the person + person = Person.find person_id + + # Update the person with the reg data + IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: reg_match) + + render status: :ok, + json: { message: "Matched" }.to_json, + content_type: 'application/json' + end + end + + # + # Method to dismiss a match + # POST request, parameters are reg_id and person_id + # + def dismiss_match + model_class.transaction do + authorize model_class, policy_class: policy_class + + reg_id = params[:reg_id] if params[:reg_id] + person_id = params[:person_id] if params[:person_id] + + raise "No reg id or person id specified" unless reg_id && person_id + + existing = DismissedRegSyncMatch.find_by reg_id: reg_id, person_id: person_id + + if !existing + DismissedRegSyncMatch.create!({ + reg_id: reg_id, + person_id: person_id + }) + end + + render status: :ok, + json: { message: "Dismissed Match" }.to_json, + content_type: 'application/json' + end + end + # by default get the data that is not already mapped to a person def default_scope(query: nil) return nil unless query diff --git a/app/lib/migration_helpers/plano_views.rb b/app/lib/migration_helpers/plano_views.rb index 8fd908366..d01cc64b5 100644 --- a/app/lib/migration_helpers/plano_views.rb +++ b/app/lib/migration_helpers/plano_views.rb @@ -53,7 +53,9 @@ def self.create_registration_map_counts query = <<-SQL.squish CREATE OR REPLACE VIEW registration_map_counts AS select rsm.reg_id, rsm.pid, count(rsm.pid) as sub_count - from registration_sync_matches rsm + from registration_sync_matches rsm + where rsm.pid not in (select person_id from dismissed_reg_sync_matches) + and rsm.reg_id not in (select reg_id from dismissed_reg_sync_matches) group by rsm.reg_id, rsm.pid SQL diff --git a/app/models/dismissed_reg_sync_match.rb b/app/models/dismissed_reg_sync_match.rb index 226f1b1ee..f12b2ac1d 100644 --- a/app/models/dismissed_reg_sync_match.rb +++ b/app/models/dismissed_reg_sync_match.rb @@ -16,4 +16,6 @@ # index_dismissed_reg_sync_matches_on_reg_id (reg_id) # class DismissedRegSyncMatch < ApplicationRecord + # + belongs_to :person end diff --git a/app/models/person.rb b/app/models/person.rb index d5b16e7db..5203cd50e 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -67,7 +67,7 @@ # published_name_sort_by :string # reddit :string # reg_attending_status :string -# reg_match :enum default("no_match") +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string diff --git a/app/policies/person_sync_datum_policy.rb b/app/policies/person_sync_datum_policy.rb index 736b4f51a..5293512f7 100644 --- a/app/policies/person_sync_datum_policy.rb +++ b/app/policies/person_sync_datum_policy.rb @@ -1,4 +1,13 @@ class PersonSyncDatumPolicy < PlannerPolicy + + def dismiss_match + allowed?(action: :dismiss_match) + end + + def match + allowed?(action: :match) + end + class Scope < PlannerPolicy::Scope def resolve scope.all diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index af7706ca0..1a8ad9544 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -71,9 +71,8 @@ def phase2 datum = RegistrationSyncDatum.find_by reg_id: match.reg_id - # TODO: update the match status - # automatic match, assisted match, manual match, no match. - IdentityService.update_reg_info(person: person, details: datum.raw_info) + # If we match via the worker it is an "automatic match" + IdentityService.update_reg_info(person: person, details: datum.raw_info, Person.reg_matches[:automatic]) end end end diff --git a/config/routes.rb b/config/routes.rb index a1272bea1..277f1001f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -211,8 +211,9 @@ get 'people', to: 'registration_sync_data#people' end + post 'person_sync_datum/dismiss_match', to: 'person_sync_data#dismiss_match' + post 'person_sync_datum/match', to: 'person_sync_data#match' resources :person_sync_data, path: 'person_sync_datum' - # get registration_sync_data # Curated tags are the list of tags for a given context etc resources :curated_tags, path: 'curated_tag' diff --git a/lib/tasks/rbac.rake b/lib/tasks/rbac.rake index dbc32b218..32efd5b7c 100644 --- a/lib/tasks/rbac.rake +++ b/lib/tasks/rbac.rake @@ -1011,6 +1011,12 @@ namespace :rbac do "update": true, "people": true, "synchronize": true + }, + "PersonSyncDatum": { + "index": true, + "show": true, + "dismiss_match": true, + "match": true } }) end From 946a7a589ec1b8c8abaa52259872afc6a5c06e70 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 27 May 2024 17:23:13 -0400 Subject: [PATCH 08/21] fix missing param name --- app/workers/registration_sync_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index 1a8ad9544..2fefb77f9 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -72,7 +72,7 @@ def phase2 datum = RegistrationSyncDatum.find_by reg_id: match.reg_id # If we match via the worker it is an "automatic match" - IdentityService.update_reg_info(person: person, details: datum.raw_info, Person.reg_matches[:automatic]) + IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) end end end From 707e62867f09ffe7b74038f9d03c50f2008e2d2b Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 28 May 2024 07:01:17 -0400 Subject: [PATCH 09/21] fix datum - do not need people from reg --- app/controllers/registration_sync_data_controller.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app/controllers/registration_sync_data_controller.rb b/app/controllers/registration_sync_data_controller.rb index 3810aee56..1dd08a315 100644 --- a/app/controllers/registration_sync_data_controller.rb +++ b/app/controllers/registration_sync_data_controller.rb @@ -45,16 +45,4 @@ def default_scope(query: nil) query.where('reg_id not in (select reg_id from people where reg_id is not null)') .where('reg_id in (select reg_id from registration_map_counts)') end - - def serializer_includes - [ - :people - ] - end - - def includes - [ - :people - ] - end end From 4491397164482266c75e4176e7d76008518c13b7 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 28 May 2024 08:13:30 -0400 Subject: [PATCH 10/21] make person audit unlimited --- app/models/person.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/person.rb b/app/models/person.rb index 5203cd50e..501f9979b 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -116,7 +116,9 @@ class Person < ApplicationRecord # acts_as_taggable acts_as_taggable_on :tags - has_paper_trail versions: { class_name: 'Audit::PersonVersion' }, ignore: [:updated_at, :created_at, :lock_version, :integrations] + has_paper_trail versions: { class_name: 'Audit::PersonVersion' }, + ignore: [:updated_at, :created_at, :lock_version, :integrations], + limit: nil before_destroy :check_if_assigned before_save :check_primary_email From 46e0f58b9fb5d50c5f8a504568e989fc80f021fa Mon Sep 17 00:00:00 2001 From: Gail Terman Date: Sun, 2 Jun 2024 14:02:29 -0400 Subject: [PATCH 11/21] possibly working manual sync --- .../person_sync_data_controller.rb | 1 + app/javascript/people/people_admin_tab.vue | 11 +- .../people/person-edit-reg-number.vue | 119 ++++++++++++++++++ .../registrations/display_sync_data.vue | 31 +++++ .../registrations/person_sync_table.vue | 17 +-- .../store/person_sync_datum.mixin.js | 35 ++++++ .../store/person_sync_datum.store.js | 22 ++++ .../store/registration_sync_datum.store.js | 26 +++- app/policies/person_sync_datum_policy.rb | 4 +- 9 files changed, 249 insertions(+), 17 deletions(-) create mode 100644 app/javascript/people/person-edit-reg-number.vue create mode 100644 app/javascript/registrations/display_sync_data.vue create mode 100644 app/javascript/store/person_sync_datum.mixin.js diff --git a/app/controllers/person_sync_data_controller.rb b/app/controllers/person_sync_data_controller.rb index 421f348f6..de35f8ee1 100644 --- a/app/controllers/person_sync_data_controller.rb +++ b/app/controllers/person_sync_data_controller.rb @@ -28,6 +28,7 @@ def match person = Person.find person_id # Update the person with the reg data + IdentityService.clear_person_reg_info(person: person); IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: reg_match) render status: :ok, diff --git a/app/javascript/people/people_admin_tab.vue b/app/javascript/people/people_admin_tab.vue index cd19afac2..3abf54e2d 100644 --- a/app/javascript/people/people_admin_tab.vue +++ b/app/javascript/people/people_admin_tab.vue @@ -7,7 +7,7 @@
Registered
{{selected.registered ? 'Yes' : 'No'}}
Registration ID
-
{{selected.registration_number || 'Unknown'}}
+
{{selected.registration_number || 'Unknown'}}
Resync Registration
@@ -20,6 +20,7 @@ Save Comments +
@@ -29,6 +30,9 @@ import { modelMixinNoProp } from '@/store/model.mixin'; import { personModel as model ,RESYNC_PERSON } from '@/store/person.store'; import { PERSON_CON_STATE, PERSON_RESYNC_SUCCESS, PERSON_RESYNC_FAILURE } from '@/constants/strings'; import { mapActions } from 'vuex'; +import EditModal from '@/components/edit_modal.vue'; +import EditButton from '@/components/edit_button.vue'; +import PersonEditRegNumber from './person-edit-reg-number.vue'; const commentsMixin = makeSelectedFieldMixin('comments'); @@ -38,6 +42,11 @@ export default { modelMixinNoProp, commentsMixin ], + components: { + EditModal, + EditButton, + PersonEditRegNumber, + }, data: () => ({ model, PERSON_CON_STATE diff --git a/app/javascript/people/person-edit-reg-number.vue b/app/javascript/people/person-edit-reg-number.vue new file mode 100644 index 000000000..aec0d89fd --- /dev/null +++ b/app/javascript/people/person-edit-reg-number.vue @@ -0,0 +1,119 @@ + + + + + + diff --git a/app/javascript/registrations/display_sync_data.vue b/app/javascript/registrations/display_sync_data.vue new file mode 100644 index 000000000..72273a25a --- /dev/null +++ b/app/javascript/registrations/display_sync_data.vue @@ -0,0 +1,31 @@ + + + + + diff --git a/app/javascript/registrations/person_sync_table.vue b/app/javascript/registrations/person_sync_table.vue index 49b18c9dd..9a6d16e35 100644 --- a/app/javascript/registrations/person_sync_table.vue +++ b/app/javascript/registrations/person_sync_table.vue @@ -23,14 +23,7 @@
-
{{ reg_data.name }}
-
{{ reg_data.email }}
-
{{ reg_data.registration_number }}
-
Preferred Name: {{ reg_data.preferred_name }}
-
Badge Name: {{ reg_data.badge_name }}
-
Alternative Email: {{ reg_data.alternative_email }}
-
Product: {{ reg_data.raw_info.product }}
-
Attending Status: {{ reg_data.raw_info.attending_status }}
+
Match @@ -44,16 +37,18 @@ diff --git a/app/javascript/navbar/side-navbar.vue b/app/javascript/navbar/side-navbar.vue index 2378b8120..91f34a3a8 100644 --- a/app/javascript/navbar/side-navbar.vue +++ b/app/javascript/navbar/side-navbar.vue @@ -16,7 +16,7 @@ Profile Admin Configurations - Registrations +
diff --git a/app/javascript/people/person-edit-reg-number.vue b/app/javascript/people/person-edit-reg-number.vue index d85211753..4c488a81f 100644 --- a/app/javascript/people/person-edit-reg-number.vue +++ b/app/javascript/people/person-edit-reg-number.vue @@ -92,43 +92,6 @@ export default { this.getRegById({id: this.regId}).then(() => { this.searched = true; }) - - // for testing, remove me - /*if(this.regId == "#123") { - this.regData = { - "id": "548cd721-5b5c-4b7a-a9e1-966523493beb", - "lock_version": 0, - "created_at": "2024-05-20T01:44:14.130Z", - "updated_at": "2024-05-20T01:44:14.130Z", - "reg_id": "49", - "registration_number": "#123", - "name": "Chris Baker", - "email": "fangorn73@hotmail.com", - "preferred_name": "Chris Baker", - "alternative_email": null, - "badge_name": null, - "raw_info": { - "id": 49, - "badge": "Fangorn", - "email": "fangorn73@hotmail.com", - "product": "Full Attending Adult", - "location": "England", - "full_name": "Chris Baker", - "date_added": "2022-08-26T12:25:18.000000Z", - "badge_title": "Glasgow 2024 Guest of Honour", - "wsfs_status": null, - "date_updated": "2022-08-26T15:58:37.000000Z", - "ticket_number": "#0001", - "preferred_name": "Chris Baker", - "attending_status": "In Person", - "alternative_email": null, - "product_list_name": "Full Adult", - "participant_activities": [] - } -} - } else { - this.regData = null; - }*/ } } } From e7598834e01c10b49b16d10c7c648f7c1f28e59c Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 3 Jun 2024 10:02:43 -0400 Subject: [PATCH 15/21] Keep match job stats --- app/models/job_status.rb | 1 + app/models/publication_status.rb | 1 + app/models/registration_sync_status.rb | 2 +- app/services/clyde_service.rb | 3 +- app/workers/registration_sync_worker.rb | 49 +++++++++++++++---- .../20240602172220_add_status_info_to_job.rb | 5 ++ db/structure.sql | 9 +++- 7 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 db/migrate/20240602172220_add_status_info_to_job.rb diff --git a/app/models/job_status.rb b/app/models/job_status.rb index c6f48c74d..59ac4f58d 100644 --- a/app/models/job_status.rb +++ b/app/models/job_status.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # lock_version :integer default(0) +# result :jsonb # status :string # submit_time :datetime # type :string diff --git a/app/models/publication_status.rb b/app/models/publication_status.rb index 3206a819c..57dc5991f 100644 --- a/app/models/publication_status.rb +++ b/app/models/publication_status.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # lock_version :integer default(0) +# result :jsonb # status :string # submit_time :datetime # type :string diff --git a/app/models/registration_sync_status.rb b/app/models/registration_sync_status.rb index fbefdf9c3..f94aa1c97 100644 --- a/app/models/registration_sync_status.rb +++ b/app/models/registration_sync_status.rb @@ -4,12 +4,12 @@ # # id :uuid not null, primary key # lock_version :integer default(0) +# result :jsonb # status :string # submit_time :datetime # type :string # created_at :datetime not null # updated_at :datetime not null # -# class RegistrationSyncStatus < JobStatus end diff --git a/app/services/clyde_service.rb b/app/services/clyde_service.rb index 71b08829c..4450613be 100644 --- a/app/services/clyde_service.rb +++ b/app/services/clyde_service.rb @@ -3,7 +3,8 @@ module ClydeService def self.base_url # Ensure that there is no trailing / in the base url - ::Integration.find_by({name: "clyde"})&.config["base_url"].chomp("/") + # ::Integration.find_by({name: "clyde"})&.config["base_url"].chomp("/") + 'https://registration.glasgow2024.org' end class Client diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index 2fefb77f9..afc747f77 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -11,16 +11,23 @@ class RegistrationSyncWorker def perform PublishedSession.transaction do - # Phase 1 - get the data from Clyde and store it - puts "--- Sync Phase 1 #{Time.now}" - phase1(page_size: 500) - puts "--- Sync Phase 2 #{Time.now}" - # Phase 2 - phase2 + # get the data from Clyde and store it + puts "--- Sync Load Phase #{Time.now}" + load_phase(page_size: 500) + puts "--- Update Phase #{Time.now}" + number_updated = update_phase + puts "--- Match Phase #{Time.now}" + number_matched = matched_phase # update the status status = RegistrationSyncStatus.order('created_at desc').first status = RegistrationSyncStatus.new if status == nil + + status.result = { + updated: number_updated, + matched: number_matched, + not_found: Person.where("reg_id is null").count + } status.status = 'completed' status.save! end @@ -29,7 +36,7 @@ def perform # Phase 1 is to suck up the data from Reg and put it into a temp store # for matching - def phase1(page_size: 500) + def load_phase(page_size: 500) # Get the clyde service and use the AUTH key that we have svc = ClydeService.get_svc(token: ENV['CLYDE_AUTH_KEY']) if !svc.token @@ -52,8 +59,26 @@ def phase1(page_size: 500) end end + def update_phase + number_updated = 0 + Person.transaction do + existing = Person.where("reg_id is not null") + existing.each do |person| + datum = RegistrationSyncDatum.find_by reg_id: person.reg_id + + next unless datum + + # TODO: only update if data has changed ... + IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) + number_updated += 1 + end + end + + number_updated + end + # Find good matches and update their information with that from the reg service - def phase2 + def matched_phase # Find all the people that have an unique match for name AND email # (i.e. there is no other match for that registration info) matched = Registration::RegistrationMapCount.where( @@ -63,18 +88,22 @@ def phase2 ).where("sub_count = 2") # Update those people with matched information + number_matched = 0 Person.transaction do matched.each do |match| person = match.person - # If the person has been mapped to another reg then we ignore it - next if (person.reg_id && (person.reg_id != match.reg_id)) + # If the person has already been mapped then we ignore it + next if person.reg_id #(person.reg_id && (person.reg_id != match.reg_id)) datum = RegistrationSyncDatum.find_by reg_id: match.reg_id # If we match via the worker it is an "automatic match" IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) + number_matched += 1 end end + + number_matched end def store_reg_data(data:) diff --git a/db/migrate/20240602172220_add_status_info_to_job.rb b/db/migrate/20240602172220_add_status_info_to_job.rb new file mode 100644 index 000000000..3343acc0a --- /dev/null +++ b/db/migrate/20240602172220_add_status_info_to_job.rb @@ -0,0 +1,5 @@ +class AddStatusInfoToJob < ActiveRecord::Migration[6.1] + def change + add_column :job_statuses, :result, :jsonb + end +end diff --git a/db/structure.sql b/db/structure.sql index e9814a875..b0f5a3524 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1062,7 +1062,8 @@ CREATE TABLE public.job_statuses ( created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, lock_version integer DEFAULT 0, - type character varying + type character varying, + result jsonb ); @@ -1679,6 +1680,9 @@ CREATE VIEW public.registration_map_counts AS rsm.pid, count(rsm.pid) AS sub_count FROM public.registration_sync_matches rsm + WHERE ((NOT (rsm.pid IN ( SELECT dismissed_reg_sync_matches.person_id + FROM public.dismissed_reg_sync_matches))) AND (NOT ((rsm.reg_id)::text IN ( SELECT dismissed_reg_sync_matches.reg_id + FROM public.dismissed_reg_sync_matches)))) GROUP BY rsm.reg_id, rsm.pid; @@ -3937,6 +3941,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240521184252'), ('20240521193119'), ('20240522174506'), -('20240522190737'); +('20240522190737'), +('20240602172220'); From 59d2a8a95168a20e71b5906e75155df77ea56ca9 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 3 Jun 2024 13:08:36 -0400 Subject: [PATCH 16/21] Improve the matching and update to be faster --- app/services/clyde_service.rb | 3 +-- app/services/identity_service.rb | 9 ++++--- app/workers/registration_sync_worker.rb | 34 +++++++++++++------------ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/app/services/clyde_service.rb b/app/services/clyde_service.rb index 4450613be..71b08829c 100644 --- a/app/services/clyde_service.rb +++ b/app/services/clyde_service.rb @@ -3,8 +3,7 @@ module ClydeService def self.base_url # Ensure that there is no trailing / in the base url - # ::Integration.find_by({name: "clyde"})&.config["base_url"].chomp("/") - 'https://registration.glasgow2024.org' + ::Integration.find_by({name: "clyde"})&.config["base_url"].chomp("/") end class Client diff --git a/app/services/identity_service.rb b/app/services/identity_service.rb index 11b0313ee..659c26cd0 100644 --- a/app/services/identity_service.rb +++ b/app/services/identity_service.rb @@ -114,7 +114,7 @@ def self.update_reg_info(person:, details:, reg_match: Person.reg_matches[:self] if person.registration_number && person.registration_number != details['ticket_number'] clear_person_reg_info(person: person) # If the Reg numbers do not match then we reset cause there may be an issue - elsif person.reg_id && person.reg_id != details['id'] + elsif person.reg_id && person.reg_id != details['id'].to_s clear_person_reg_info(person: person) else person.reg_id = details['id'] unless person.reg_id @@ -124,17 +124,20 @@ def self.update_reg_info(person:, details:, reg_match: Person.reg_matches[:self] person.registered = details['attending_status'] != 'Not Attending' person.reg_attending_status = details['attending_status'] # Need to store time of last sync - person.date_reg_synced = Time.now # How the registration match was done person.reg_match = reg_match + # This will ensure update is done only any of the reg data has changed + if person.changed? + person.date_reg_synced = Time.now + end # Attendance type in Plano is one of # in_person, hybrid, virtual # Clyde does not map to these well. Recommend that we get this from survey and Person profile # in Plano instead. # person.attendance_type = end - person.save! + person.changed? ? person.save! : false end def self.create_identity_from_clyde(details:) diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index afc747f77..f3b98897a 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -65,12 +65,11 @@ def update_phase existing = Person.where("reg_id is not null") existing.each do |person| datum = RegistrationSyncDatum.find_by reg_id: person.reg_id - - next unless datum - - # TODO: only update if data has changed ... - IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) - number_updated += 1 + + if datum + res = IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) + number_updated += 1 if res + end end end @@ -82,24 +81,27 @@ def matched_phase # Find all the people that have an unique match for name AND email # (i.e. there is no other match for that registration info) matched = Registration::RegistrationMapCount.where( - "pid in (?)", Registration::RegistrationMapPeopleCount.where("count = 1").pluck(:pid) + "pid in (select pid from registration_map_people_counts where count = 1)" + ).where( + "reg_id in (select reg_id from registration_map_reg_counts where count = 1)" ).where( - "reg_id in (?)", Registration::RegistrationMapRegCount.where("count = 1").pluck(:reg_id) + "pid not in (select id from people where reg_id is not null)" ).where("sub_count = 2") # Update those people with matched information number_matched = 0 - Person.transaction do - matched.each do |match| + matched.each do |match| + Person.transaction do person = match.person - # If the person has already been mapped then we ignore it - next if person.reg_id #(person.reg_id && (person.reg_id != match.reg_id)) - datum = RegistrationSyncDatum.find_by reg_id: match.reg_id + # If the person has already been mapped then we ignore it + if person.reg_id.nil? + datum = RegistrationSyncDatum.find_by reg_id: match.reg_id - # If we match via the worker it is an "automatic match" - IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) - number_matched += 1 + # If we match via the worker it is an "automatic match" + IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) + number_matched += 1 + end end end From f3edd60113068bf44bb7b9755dbbfb30b1e9e126 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 3 Jun 2024 13:57:58 -0400 Subject: [PATCH 17/21] endpoint for the last sync's stats --- app/controllers/registration_sync_data_controller.rb | 9 +++++++++ app/policies/registration_sync_datum_policy.rb | 6 +++++- config/routes.rb | 1 + lib/tasks/rbac.rake | 9 ++++++--- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/controllers/registration_sync_data_controller.rb b/app/controllers/registration_sync_data_controller.rb index 1dd08a315..2d8bffbfb 100644 --- a/app/controllers/registration_sync_data_controller.rb +++ b/app/controllers/registration_sync_data_controller.rb @@ -5,6 +5,15 @@ class RegistrationSyncDataController < ResourceController DEFAULT_SORTBY = 'registration_sync_data.name' DEFAULT_ORDER = 'asc'.freeze + def sync_statistics + authorize current_person, policy_class: policy_class + status = RegistrationSyncStatus.order('created_at desc').first + + result = status ? status.result : {} + + render status: :ok, json: result.to_json, content_type: 'application/json' + end + def synchronize authorize current_person, policy_class: policy_class diff --git a/app/policies/registration_sync_datum_policy.rb b/app/policies/registration_sync_datum_policy.rb index 1504f0643..08513b86b 100644 --- a/app/policies/registration_sync_datum_policy.rb +++ b/app/policies/registration_sync_datum_policy.rb @@ -3,8 +3,12 @@ def people? allowed?(action: :people) end + def sync_statistics? + allowed?(action: :sync_statistics) + end + def synchronize? - allowed?(action: :people) + allowed?(action: :synchronize) end class Scope < PlannerPolicy::Scope diff --git a/config/routes.rb b/config/routes.rb index 277f1001f..5c4550ce6 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -205,6 +205,7 @@ resources :page_contents, path: 'page_content' # + get 'registration_sync_data/sync_statistics', to: 'registration_sync_data#sync_statistics' get 'registration_sync_data/synchronize', to: 'registration_sync_data#synchronize' resources :registration_sync_data, path: 'registration_sync_datum' do # This needs to work a bit different than the other sub relationships diff --git a/lib/tasks/rbac.rake b/lib/tasks/rbac.rake index 32efd5b7c..81126b4fe 100644 --- a/lib/tasks/rbac.rake +++ b/lib/tasks/rbac.rake @@ -310,7 +310,8 @@ namespace :rbac do "show": false, "update": false, "people": false, - "synchronize": false + "synchronize": false, + "sync_statistics": false } }) end @@ -660,7 +661,8 @@ namespace :rbac do "show": true, "update": false, "people": true, - "synchronize": false + "synchronize": false, + "sync_statistics": true } }) end @@ -1010,7 +1012,8 @@ namespace :rbac do "show": true, "update": true, "people": true, - "synchronize": true + "synchronize": true, + "sync_statistics": true }, "PersonSyncDatum": { "index": true, From e052fc5713f126573e89cd8ac7d3a5d4fab673a0 Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 5 Jun 2024 20:55:20 -0400 Subject: [PATCH 18/21] Include matched prson in reg sync datum --- .../registration_sync_data_controller.rb | 20 ++++++++++++------- app/models/registration_sync_datum.rb | 4 ++++ .../registration_sync_datum_serializer.rb | 11 +--------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/app/controllers/registration_sync_data_controller.rb b/app/controllers/registration_sync_data_controller.rb index 2d8bffbfb..a9f16e907 100644 --- a/app/controllers/registration_sync_data_controller.rb +++ b/app/controllers/registration_sync_data_controller.rb @@ -46,12 +46,18 @@ def people content_type: 'application/json' end - # by default get the data that is not already mapped to a person - def default_scope(query: nil) - return nil unless query - - # People that have a potential mapping and not already mapped - query.where('reg_id not in (select reg_id from people where reg_id is not null)') - .where('reg_id in (select reg_id from registration_map_counts)') + def serializer_includes + [ + :matched_person + ] end + + # # by default get the data that is not already mapped to a person + # def default_scope(query: nil) + # return nil unless query + + # # People that have a potential mapping and not already mapped + # query.where('reg_id not in (select reg_id from people where reg_id is not null)') + # .where('reg_id in (select reg_id from registration_map_counts)') + # end end diff --git a/app/models/registration_sync_datum.rb b/app/models/registration_sync_datum.rb index d804e15e1..c377e2700 100644 --- a/app/models/registration_sync_datum.rb +++ b/app/models/registration_sync_datum.rb @@ -33,6 +33,10 @@ class RegistrationSyncDatum < ApplicationRecord # limit the matches ...? # Add index of reg_id to people # where reg_id not in people.reg_id + has_one :matched_person, + class_name: 'Person', + foreign_key: 'reg_id', + primary_key: 'reg_id' has_many :people, through: :registration_sync_matches end diff --git a/app/serializers/registration_sync_datum_serializer.rb b/app/serializers/registration_sync_datum_serializer.rb index 376107f3b..b394d3a03 100644 --- a/app/serializers/registration_sync_datum_serializer.rb +++ b/app/serializers/registration_sync_datum_serializer.rb @@ -34,14 +34,5 @@ class RegistrationSyncDatumSerializer :badge_name, :raw_info - # The people that this data could be matched to - # has_many :people, serializer: PersonSerializer, - # links: { - # self: -> (object, params) { - # "#{params[:domain]}/registration_sync_datum/#{object.id}" - # }, - # related: -> (object, params) { - # "#{params[:domain]}/registration_sync_datum/#{object.id}/people" - # } - # } + has_one :matched_person, serializer: PersonSerializer end From 5441146097e33db1e5c223d7ef51b5963f41ecf1 Mon Sep 17 00:00:00 2001 From: Gail Terman Date: Wed, 5 Jun 2024 21:04:50 -0400 Subject: [PATCH 19/21] PLAN-995 remove reg link and add ticket number view --- app/javascript/profile/dl_person.vue | 23 ++++++++++++++++++++--- app/javascript/profile/person_details.vue | 5 +++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/app/javascript/profile/dl_person.vue b/app/javascript/profile/dl_person.vue index b273fb920..0f8e66ee7 100644 --- a/app/javascript/profile/dl_person.vue +++ b/app/javascript/profile/dl_person.vue @@ -9,9 +9,9 @@
Restricted - Yes - No - Not Specified + {{ yes(field) }} + {{ no(field) }} + {{ notSpecified(field) }} {{selected[field]}}
@@ -29,6 +29,9 @@ export default { props: { fields: { default: [] + }, + overrides: { + default: () => {} } }, mixins: [ @@ -38,5 +41,19 @@ export default { model, PROFILE_FIELD_LABELS }), + methods: { + getOverride(key, defaultText, field) { + return this.overrides?.[key]?.[field] ?? defaultText; + }, + notSpecified(field) { + return this.getOverride('null', 'Not Specified', field); + }, + yes(field) { + return this.getOverride('true', 'Yes', field); + }, + no(field) { + return this.getOverride('false', 'No', field); + } + } } diff --git a/app/javascript/profile/person_details.vue b/app/javascript/profile/person_details.vue index 0135090ac..a1fe2f829 100644 --- a/app/javascript/profile/person_details.vue +++ b/app/javascript/profile/person_details.vue @@ -4,8 +4,9 @@
Identity
- - + + +
From 07276235b67c7538e6f1e11e4aa382c25035e196 Mon Sep 17 00:00:00 2001 From: Gail Terman Date: Wed, 5 Jun 2024 23:44:18 -0400 Subject: [PATCH 20/21] PLAN-996 removal of link, and link confirmation --- app/javascript/components/icon_button.vue | 31 +++- app/javascript/constants/strings.js | 12 +- app/javascript/people/people_admin_tab.vue | 157 ++++++++++++------ .../people/person-edit-reg-number.vue | 106 +++++++----- app/javascript/profile/dl_person.vue | 6 +- 5 files changed, 202 insertions(+), 110 deletions(-) diff --git a/app/javascript/components/icon_button.vue b/app/javascript/components/icon_button.vue index d909f1ea7..bb802ec44 100644 --- a/app/javascript/components/icon_button.vue +++ b/app/javascript/components/icon_button.vue @@ -3,17 +3,19 @@ - - + + @@ -35,6 +37,10 @@ export default { type: String, required: false }, + variant: { + type: String, + default: 'primary' + }, disabledTooltip: { type: String, default: "This button is disabled" @@ -51,23 +57,30 @@ export default { type: String, default: "sm" }, + tooltip: { + type: String, + }, + modal: { + type: String, + } }, computed: { - variant() { + computedVariant() { switch(this.background) { case "none": return "link"; case "default": return "primary"; + case "danger": + return "danger"; } }, iconVariant() { - switch(this.variant) { + switch(this.computedVariant) { case "link": - return "primary"; - case "primary": { + return this.disabled ? undefined : this.variant; + case "primary": return undefined; - } } }, spanId() { diff --git a/app/javascript/constants/strings.js b/app/javascript/constants/strings.js index b53e4eec7..c008c8699 100644 --- a/app/javascript/constants/strings.js +++ b/app/javascript/constants/strings.js @@ -2,6 +2,7 @@ const twoLines = (line1, line2) => (h) => h('p', {}, [line1, h('br'), line2]); const errorMessage = (message) => (errorCode) => twoLines(message, `Error code: ${errorCode}`); const titleCase = (model) => `${model.substring(0, 1).toUpperCase()}${model.substring(1)}`; const nLines = (lines) => (h) => h('p', {}, lines.reduce((p, c) => [...p, c, h('br')], [])); +const bold = (text) => (h) => h('strong', {}, text); module.exports = { // login page @@ -274,6 +275,9 @@ module.exports = { global_diaspora: "Member of the global diaspora", non_anglophone: "Represent something other than a purely anglophone perspective", excluded_demographic_categories: "Participant's demographic categories that should not be discussed on panels that include them", + registered: "Registered", + registration_type: "Registration Type", + reg_attending_status: "Registration Attending Status", }, PERSON_SAVE_SUCCESS: "Profile record saved successfully", PERSON_NEVER_LOGGED_IN: "Never logged in", @@ -362,10 +366,10 @@ module.exports = { PROFILE_LINK_EXPLAINATION_1: "When you link your Planorama account to your registration account they are matched, and you will be asked to confirm the correct registration account.", PROFILE_LINK_EXPLAINATION_2: "You will continue to log in to Planorama using the password you created, not your registration login details.", PROFILE_LINK_EXPLAINATION_TITLE: "What does it mean to link to registration?", - REG_ID_SEARCH_PLACEHOLDER: "Search for a registration ID.", + REG_ID_SEARCH_PLACEHOLDER: "Search for a ticket number.", REG_ID_FOUND: "Found a match in the registration database", - REG_ID_NOT_FOUND: "No results found. This number may already be linked. Please check your input.", + REG_ID_NOT_FOUND: "No results found. Please check your input.", REG_ID_UNLINK_BUTTON: "Unlink Current Registration", - REG_ID_UNLINK_CONFIRMATION_TITLE: "Really really???", - REG_ID_UNLINK_CONFIRMATION_TEXT: "Are you sure, bozo", + REG_ID_UNLINK_CONFIRMATION_TITLE: "", + REG_ID_UNLINK_CONFIRMATION_TEXT: (name, number) => `This will unlink ${name} from Ticket Number ${number}.`, } diff --git a/app/javascript/people/people_admin_tab.vue b/app/javascript/people/people_admin_tab.vue index 972d35532..5f84b33e8 100644 --- a/app/javascript/people/people_admin_tab.vue +++ b/app/javascript/people/people_admin_tab.vue @@ -1,80 +1,137 @@ - + diff --git a/app/javascript/people/person-edit-reg-number.vue b/app/javascript/people/person-edit-reg-number.vue index 4c488a81f..97eb43505 100644 --- a/app/javascript/people/person-edit-reg-number.vue +++ b/app/javascript/people/person-edit-reg-number.vue @@ -1,20 +1,41 @@ @@ -32,6 +33,9 @@ export default { }, overrides: { default: () => {} + }, + nullText: { + default: "Not Specified" } }, mixins: [ @@ -46,7 +50,7 @@ export default { return this.overrides?.[key]?.[field] ?? defaultText; }, notSpecified(field) { - return this.getOverride('null', 'Not Specified', field); + return this.getOverride('null', this.nullText, field); }, yes(field) { return this.getOverride('true', 'Yes', field); From eeda978546fa4fc7e65e8ea2e82c36f6341aedb7 Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 6 Jun 2024 07:54:05 -0400 Subject: [PATCH 21/21] unique index for people.reg_id --- db/migrate/20240606115218_ensure_reg_id_unique.rb | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 db/migrate/20240606115218_ensure_reg_id_unique.rb diff --git a/db/migrate/20240606115218_ensure_reg_id_unique.rb b/db/migrate/20240606115218_ensure_reg_id_unique.rb new file mode 100644 index 000000000..6f1805d00 --- /dev/null +++ b/db/migrate/20240606115218_ensure_reg_id_unique.rb @@ -0,0 +1,5 @@ +class EnsureRegIdUnique < ActiveRecord::Migration[6.1] + def change + add_index :people, [:reg_id], unique: true, name: "idx_people_reg_id" + end +end